diff --git a/docs/api.md b/docs/api.md
index e103ae14a..97ccf4c3a 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -139,7 +139,7 @@ document. Paperless only reports PDF metadata at this point.
 
 ## Authorization
 
-The REST api provides four different forms of authentication.
+The REST api provides three different forms of authentication.
 
 1.  Basic authentication
 
@@ -177,12 +177,6 @@ The REST api provides four different forms of authentication.
 
     Tokens can also be managed in the Django admin.
 
-4.  Remote User authentication
-
-    If already setup (see
-    [configuration](configuration.md#PAPERLESS_ENABLE_HTTP_REMOTE_USER)),
-    you can authenticate against the API using Remote User auth.
-
 ## Searching for documents
 
 Full text searching is available on the `/api/documents/` endpoint. Two
diff --git a/src/paperless/auth.py b/src/paperless/auth.py
index 98e2a8b30..a23b01cb4 100644
--- a/src/paperless/auth.py
+++ b/src/paperless/auth.py
@@ -47,11 +47,3 @@ class HttpRemoteUserMiddleware(PersistentRemoteUserMiddleware):
     """
 
     header = settings.HTTP_REMOTE_USER_HEADER_NAME
-
-
-class PaperlessRemoteUserAuthentication(authentication.RemoteUserAuthentication):
-    """
-    REMOTE_USER authentication for DRF which overrides the default header.
-    """
-
-    header = settings.HTTP_REMOTE_USER_HEADER_NAME
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 54779006d..bc815d4d5 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -420,31 +420,19 @@ if AUTO_LOGIN_USERNAME:
     # regular login in case the provided user does not exist.
     MIDDLEWARE.insert(_index + 1, "paperless.auth.AutoLoginMiddleware")
 
+ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER")
+HTTP_REMOTE_USER_HEADER_NAME = os.getenv(
+    "PAPERLESS_HTTP_REMOTE_USER_HEADER_NAME",
+    "HTTP_REMOTE_USER",
+)
 
-def _parse_remote_user_settings() -> str:
-    global MIDDLEWARE, AUTHENTICATION_BACKENDS, REST_FRAMEWORK
-    enable = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER")
-    if enable:
-        MIDDLEWARE.append("paperless.auth.HttpRemoteUserMiddleware")
-        AUTHENTICATION_BACKENDS.insert(
-            0,
-            "django.contrib.auth.backends.RemoteUserBackend",
-        )
-        REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"].insert(
-            0,
-            "paperless.auth.PaperlessRemoteUserAuthentication",
-        )
-
-    header_name = os.getenv(
-        "PAPERLESS_HTTP_REMOTE_USER_HEADER_NAME",
-        "HTTP_REMOTE_USER",
+if ENABLE_HTTP_REMOTE_USER:
+    MIDDLEWARE.append("paperless.auth.HttpRemoteUserMiddleware")
+    AUTHENTICATION_BACKENDS.insert(0, "django.contrib.auth.backends.RemoteUserBackend")
+    REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"].append(
+        "rest_framework.authentication.RemoteUserAuthentication",
     )
 
-    return header_name
-
-
-HTTP_REMOTE_USER_HEADER_NAME = _parse_remote_user_settings()
-
 # X-Frame options for embedded PDF display:
 X_FRAME_OPTIONS = "ANY" if DEBUG else "SAMEORIGIN"
 
diff --git a/src/paperless/tests/test_remote_user.py b/src/paperless/tests/test_remote_user.py
deleted file mode 100644
index 194026e4d..000000000
--- a/src/paperless/tests/test_remote_user.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import os
-from unittest import mock
-
-from django.contrib.auth.models import User
-from rest_framework import status
-from rest_framework.test import APITestCase
-
-from documents.tests.utils import DirectoriesMixin
-from paperless.settings import _parse_remote_user_settings
-
-
-class TestRemoteUser(DirectoriesMixin, APITestCase):
-    def setUp(self):
-        super().setUp()
-
-        self.user = User.objects.create_superuser(
-            username="temp_admin",
-        )
-
-    def test_remote_user(self):
-        """
-        GIVEN:
-            - Configured user
-            - Remote user auth is enabled
-        WHEN:
-            - API call is made to get documents
-        THEN:
-            - Call succeeds
-        """
-
-        with mock.patch.dict(
-            os.environ,
-            {
-                "PAPERLESS_ENABLE_HTTP_REMOTE_USER": "True",
-            },
-        ):
-            _parse_remote_user_settings()
-
-            response = self.client.get("/api/documents/")
-
-            # 403 testing locally, 401 on ci...
-            self.assertIn(
-                response.status_code,
-                [status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN],
-            )
-
-            response = self.client.get(
-                "/api/documents/",
-                headers={
-                    "Remote-User": self.user.username,
-                },
-            )
-
-            self.assertEqual(response.status_code, status.HTTP_200_OK)
-
-    def test_remote_user_header_setting(self):
-        """
-        GIVEN:
-            - Remote user header name is set
-        WHEN:
-            - Settings are parsed
-        THEN:
-            - Correct header name is returned
-        """
-
-        with mock.patch.dict(
-            os.environ,
-            {
-                "PAPERLESS_ENABLE_HTTP_REMOTE_USER": "True",
-                "PAPERLESS_HTTP_REMOTE_USER_HEADER_NAME": "HTTP_FOO",
-            },
-        ):
-            header_name = _parse_remote_user_settings()
-
-            self.assertEqual(header_name, "HTTP_FOO")