Enhancement: system status report sanity check, simpler classifier check, styling updates (#9106)

src/documents/tests/test_api_status.py

@@ -1,18 +1,14 @@
 import os
-import tempfile
 from pathlib import Path
 from unittest import mock
 
+from celery import states
 from django.contrib.auth.models import User
 from django.test import override_settings
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from documents.classifier import ClassifierModelCorruptError
-from documents.classifier import DocumentClassifier
-from documents.classifier import load_classifier
-from documents.models import Document
-from documents.models import Tag
+from documents.models import PaperlessTask
 from paperless import version
 
 
@@ -193,7 +189,6 @@ class TestSystemStatus(APITestCase):
         self.assertEqual(response.data["tasks"]["index_status"], "ERROR")
         self.assertIsNotNone(response.data["tasks"]["index_error"])
 
-    @override_settings(DATA_DIR=Path("/tmp/does_not_exist/data/"))
     def test_system_status_classifier_ok(self):
         """
         GIVEN:
@@ -203,9 +198,11 @@ class TestSystemStatus(APITestCase):
         THEN:
             - The response contains an OK classifier status
         """
-        load_classifier()
-        test_classifier = DocumentClassifier()
-        test_classifier.save()
+        PaperlessTask.objects.create(
+            type=PaperlessTask.TaskType.SCHEDULED_TASK,
+            status=states.SUCCESS,
+            task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
+        )
         self.client.force_login(self.user)
         response = self.client.get(self.ENDPOINT)
         self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -215,73 +212,101 @@ class TestSystemStatus(APITestCase):
     def test_system_status_classifier_warning(self):
         """
         GIVEN:
-            - The classifier does not exist yet
-            - > 0 documents and tags with auto matching exist
+            - No classifier task is found
         WHEN:
             - The user requests the system status
         THEN:
-            - The response contains an WARNING classifier status
+            - The response contains a WARNING classifier status
         """
-        with override_settings(MODEL_FILE=Path("does_not_exist")):
-            Document.objects.create(
-                title="Test Document",
-            )
-            Tag.objects.create(name="Test Tag", matching_algorithm=Tag.MATCH_AUTO)
-            self.client.force_login(self.user)
-            response = self.client.get(self.ENDPOINT)
-            self.assertEqual(response.status_code, status.HTTP_200_OK)
-            self.assertEqual(response.data["tasks"]["classifier_status"], "WARNING")
-            self.assertIsNotNone(response.data["tasks"]["classifier_error"])
+        self.client.force_login(self.user)
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            response.data["tasks"]["classifier_status"],
+            "WARNING",
+        )
 
-    @mock.patch(
-        "documents.classifier.load_classifier",
-        side_effect=ClassifierModelCorruptError(),
-    )
-    def test_system_status_classifier_error(self, mock_load_classifier):
+    def test_system_status_classifier_error(self):
         """
         GIVEN:
-            - The classifier does exist but is corrupt
-            - > 0 documents and tags with auto matching exist
+            - An error occurred while loading the classifier
         WHEN:
             - The user requests the system status
         THEN:
             - The response contains an ERROR classifier status
         """
-        with (
-            tempfile.NamedTemporaryFile(
-                dir="/tmp",
-                delete=False,
-            ) as does_exist,
-            override_settings(MODEL_FILE=Path(does_exist.name)),
-        ):
-            Document.objects.create(
-                title="Test Document",
-            )
-            Tag.objects.create(
-                name="Test Tag",
-                matching_algorithm=Tag.MATCH_AUTO,
-            )
-            self.client.force_login(self.user)
-            response = self.client.get(self.ENDPOINT)
-            self.assertEqual(response.status_code, status.HTTP_200_OK)
-            self.assertEqual(
-                response.data["tasks"]["classifier_status"],
-                "ERROR",
-            )
-            self.assertIsNotNone(response.data["tasks"]["classifier_error"])
+        PaperlessTask.objects.create(
+            type=PaperlessTask.TaskType.SCHEDULED_TASK,
+            status=states.FAILURE,
+            task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
+            result="Classifier training failed",
+        )
+        self.client.force_login(self.user)
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            response.data["tasks"]["classifier_status"],
+            "ERROR",
+        )
+        self.assertIsNotNone(response.data["tasks"]["classifier_error"])
 
-    def test_system_status_classifier_ok_no_objects(self):
+    def test_system_status_sanity_check_ok(self):
         """
         GIVEN:
-            - The classifier does not exist (and should not)
-            - No documents nor objects with auto matching exist
+            - The sanity check is successful
         WHEN:
             - The user requests the system status
         THEN:
-            - The response contains an OK classifier status
+            - The response contains an OK sanity check status
         """
-        with override_settings(MODEL_FILE=Path("does_not_exist")):
-            self.client.force_login(self.user)
-            response = self.client.get(self.ENDPOINT)
-            self.assertEqual(response.status_code, status.HTTP_200_OK)
-            self.assertEqual(response.data["tasks"]["classifier_status"], "OK")
+        PaperlessTask.objects.create(
+            type=PaperlessTask.TaskType.SCHEDULED_TASK,
+            status=states.SUCCESS,
+            task_name=PaperlessTask.TaskName.CHECK_SANITY,
+        )
+        self.client.force_login(self.user)
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["tasks"]["sanity_check_status"], "OK")
+        self.assertIsNone(response.data["tasks"]["sanity_check_error"])
+
+    def test_system_status_sanity_check_warning(self):
+        """
+        GIVEN:
+            - No sanity check task is found
+        WHEN:
+            - The user requests the system status
+        THEN:
+            - The response contains a WARNING sanity check status
+        """
+        self.client.force_login(self.user)
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            response.data["tasks"]["sanity_check_status"],
+            "WARNING",
+        )
+
+    def test_system_status_sanity_check_error(self):
+        """
+        GIVEN:
+            - The sanity check failed
+        WHEN:
+            - The user requests the system status
+        THEN:
+            - The response contains an ERROR sanity check status
+        """
+        PaperlessTask.objects.create(
+            type=PaperlessTask.TaskType.SCHEDULED_TASK,
+            status=states.FAILURE,
+            task_name=PaperlessTask.TaskName.CHECK_SANITY,
+            result="5 issues found.",
+        )
+        self.client.force_login(self.user)
+        response = self.client.get(self.ENDPOINT)
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            response.data["tasks"]["sanity_check_status"],
+            "ERROR",
+        )
+        self.assertIsNotNone(response.data["tasks"]["sanity_check_error"])

src/documents/tests/test_api_tasks.py

@@ -1,4 +1,5 @@
 import uuid
+from unittest import mock
 
 import celery
 from django.contrib.auth.models import Permission
@@ -8,6 +9,7 @@ from rest_framework.test import APITestCase
 
 from documents.models import PaperlessTask
 from documents.tests.utils import DirectoriesMixin
+from documents.views import TasksViewSet
 
 
 class TestTasks(DirectoriesMixin, APITestCase):
@@ -130,7 +132,7 @@ class TestTasks(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
 
-        response = self.client.get(self.ENDPOINT)
+        response = self.client.get(self.ENDPOINT + "?acknowledged=false")
         self.assertEqual(len(response.data), 0)
 
     def test_tasks_owner_aware(self):
@@ -246,7 +248,7 @@ class TestTasks(DirectoriesMixin, APITestCase):
         PaperlessTask.objects.create(
             task_id=str(uuid.uuid4()),
             task_file_name="test.pdf",
-            task_name="documents.tasks.some_task",
+            task_name=PaperlessTask.TaskName.CONSUME_FILE,
             status=celery.states.SUCCESS,
         )
 
@@ -272,7 +274,7 @@ class TestTasks(DirectoriesMixin, APITestCase):
         PaperlessTask.objects.create(
             task_id=str(uuid.uuid4()),
             task_file_name="anothertest.pdf",
-            task_name="documents.tasks.some_task",
+            task_name=PaperlessTask.TaskName.CONSUME_FILE,
             status=celery.states.SUCCESS,
         )
 
@@ -309,3 +311,62 @@ class TestTasks(DirectoriesMixin, APITestCase):
         returned_data = response.data[0]
 
         self.assertEqual(returned_data["related_document"], "1234")
+
+    def test_run_train_classifier_task(self):
+        """
+        GIVEN:
+            - A superuser
+        WHEN:
+            - API call is made to run the train classifier task
+        THEN:
+            - The task is run
+        """
+        mock_train_classifier = mock.Mock(return_value="Task started")
+        TasksViewSet.TASK_AND_ARGS_BY_NAME = {
+            PaperlessTask.TaskName.TRAIN_CLASSIFIER: (
+                mock_train_classifier,
+                {"scheduled": False},
+            ),
+        }
+        response = self.client.post(
+            self.ENDPOINT + "run/",
+            {"task_name": PaperlessTask.TaskName.TRAIN_CLASSIFIER},
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data, {"result": "Task started"})
+        mock_train_classifier.assert_called_once_with(scheduled=False)
+
+        # mock error
+        mock_train_classifier.reset_mock()
+        mock_train_classifier.side_effect = Exception("Error")
+        response = self.client.post(
+            self.ENDPOINT + "run/",
+            {"task_name": PaperlessTask.TaskName.TRAIN_CLASSIFIER},
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
+        mock_train_classifier.assert_called_once_with(scheduled=False)
+
+    @mock.patch("documents.tasks.sanity_check")
+    def test_run_task_requires_superuser(self, mock_check_sanity):
+        """
+        GIVEN:
+            - A regular user
+        WHEN:
+            - API call is made to run a task
+        THEN:
+            - The task is not run
+        """
+        regular_user = User.objects.create_user(username="test")
+        regular_user.user_permissions.add(*Permission.objects.all())
+        self.client.logout()
+        self.client.force_authenticate(user=regular_user)
+
+        response = self.client.post(
+            self.ENDPOINT + "run/",
+            {"task_name": PaperlessTask.TaskName.CHECK_SANITY},
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+        mock_check_sanity.assert_not_called()

src/documents/tests/test_task_signals.py

@@ -68,7 +68,7 @@ class TestTaskSignalHandler(DirectoriesMixin, TestCase):
         self.assertIsNotNone(task)
         self.assertEqual(headers["id"], task.task_id)
         self.assertEqual("hello-999.pdf", task.task_file_name)
-        self.assertEqual("documents.tasks.consume_file", task.task_name)
+        self.assertEqual(PaperlessTask.TaskName.CONSUME_FILE, task.task_name)
         self.assertEqual(1, task.owner_id)
         self.assertEqual(celery.states.PENDING, task.status)
 

src/documents/tests/test_tasks.py

@@ -118,6 +118,19 @@ class TestSanityCheck(DirectoriesMixin, TestCase):
         self.assertRaises(SanityCheckFailedException, tasks.sanity_check)
         m.assert_called_once()
 
+    @mock.patch("documents.tasks.sanity_checker.check_sanity")
+    def test_sanity_check_error_no_raise(self, m):
+        messages = SanityCheckMessages()
+        messages.error(None, "Some error")
+        m.return_value = messages
+        # No exception should be raised
+        result = tasks.sanity_check(raise_on_error=False)
+        self.assertEqual(
+            result,
+            "Sanity check exited with errors. See log.",
+        )
+        m.assert_called_once()
+
     @mock.patch("documents.tasks.sanity_checker.check_sanity")
     def test_sanity_check_warning(self, m):
         messages = SanityCheckMessages()