Merge branch 'dev' into feature-websockets-status

src/documents/tests/test_api.py

@@ -590,6 +590,10 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertEqual(len(meta['original_metadata']), 0)
         self.assertGreater(len(meta['archive_metadata']), 0)
 
+    def test_get_metadata_invalid_doc(self):
+        response = self.client.get(f"/api/documents/34576/metadata/")
+        self.assertEqual(response.status_code, 404)
+
     def test_get_metadata_no_archive(self):
         doc = Document.objects.create(title="test", filename="file.pdf", mime_type="application/pdf")
 
@@ -605,6 +609,30 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertGreater(len(meta['original_metadata']), 0)
         self.assertIsNone(meta['archive_metadata'])
 
+    def test_get_empty_suggestions(self):
+        doc = Document.objects.create(title="test", mime_type="application/pdf")
+
+        response = self.client.get(f"/api/documents/{doc.pk}/suggestions/")
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data, {'correspondents': [], 'tags': [], 'document_types': []})
+
+    def test_get_suggestions_invalid_doc(self):
+        response = self.client.get(f"/api/documents/34676/suggestions/")
+        self.assertEqual(response.status_code, 404)
+
+    @mock.patch("documents.views.match_correspondents")
+    @mock.patch("documents.views.match_tags")
+    @mock.patch("documents.views.match_document_types")
+    def test_get_suggestions(self, match_document_types, match_tags, match_correspondents):
+        doc = Document.objects.create(title="test", mime_type="application/pdf", content="this is an invoice!")
+        match_tags.return_value = [Tag(id=56), Tag(id=123)]
+        match_document_types.return_value = [DocumentType(id=23)]
+        match_correspondents.return_value = [Correspondent(id=88), Correspondent(id=2)]
+
+        response = self.client.get(f"/api/documents/{doc.pk}/suggestions/")
+        self.assertEqual(response.data, {'correspondents': [88,2], 'tags': [56,123], 'document_types': [23]})
+
     def test_saved_views(self):
         u1 = User.objects.create_user("user1")
         u2 = User.objects.create_user("user2")

src/documents/tests/test_classifier.py

@@ -1,10 +1,13 @@
+import os
 import tempfile
+from pathlib import Path
 from time import sleep
 from unittest import mock
 
+from django.conf import settings
 from django.test import TestCase, override_settings
 
-from documents.classifier import DocumentClassifier, IncompatibleClassifierVersionError
+from documents.classifier import DocumentClassifier, IncompatibleClassifierVersionError, load_classifier
 from documents.models import Correspondent, Document, Tag, DocumentType
 from documents.tests.utils import DirectoriesMixin
 
@@ -235,3 +238,30 @@ class TestClassifier(DirectoriesMixin, TestCase):
         self.classifier.train()
         self.assertListEqual(self.classifier.predict_tags(doc1.content), [t1.pk])
         self.assertListEqual(self.classifier.predict_tags(doc2.content), [])
+
+    def test_load_classifier_not_exists(self):
+        self.assertFalse(os.path.exists(settings.MODEL_FILE))
+        self.assertIsNone(load_classifier())
+
+    @mock.patch("documents.classifier.DocumentClassifier.reload")
+    def test_load_classifier(self, reload):
+        Path(settings.MODEL_FILE).touch()
+        self.assertIsNotNone(load_classifier())
+
+    @mock.patch("documents.classifier.DocumentClassifier.reload")
+    def test_load_classifier_incompatible_version(self, reload):
+        Path(settings.MODEL_FILE).touch()
+        self.assertTrue(os.path.exists(settings.MODEL_FILE))
+
+        reload.side_effect = IncompatibleClassifierVersionError()
+        self.assertIsNone(load_classifier())
+        self.assertFalse(os.path.exists(settings.MODEL_FILE))
+
+    @mock.patch("documents.classifier.DocumentClassifier.reload")
+    def test_load_classifier_os_error(self, reload):
+        Path(settings.MODEL_FILE).touch()
+        self.assertTrue(os.path.exists(settings.MODEL_FILE))
+
+        reload.side_effect = OSError()
+        self.assertIsNone(load_classifier())
+        self.assertTrue(os.path.exists(settings.MODEL_FILE))

src/documents/tests/test_consumer.py

@@ -460,7 +460,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         self._assert_first_last_send_progress()
 
-    @mock.patch("documents.consumer.DocumentClassifier")
+    @mock.patch("documents.consumer.load_classifier")
     def testClassifyDocument(self, m):
         correspondent = Correspondent.objects.create(name="test")
         dtype = DocumentType.objects.create(name="test")

src/documents/tests/test_settings.py

@@ -20,7 +20,7 @@ class TestSettings(TestCase):
         self.assertEqual(default_threads, 1)
 
     def test_workers_threads(self):
-        for i in range(2, 64):
+        for i in range(1, 64):
             with mock.patch("paperless.settings.multiprocessing.cpu_count") as cpu_count:
                 cpu_count.return_value = i
 
@@ -31,4 +31,4 @@ class TestSettings(TestCase):
                 self.assertTrue(default_workers >= 1)
                 self.assertTrue(default_threads >= 1)
 
-                self.assertTrue(default_workers * default_threads < i, f"{i}")
+                self.assertTrue(default_workers * default_threads <= i, f"{i}")

src/documents/tests/test_tasks.py

@@ -1,11 +1,12 @@
-from datetime import datetime
+import os
 from unittest import mock
 
+from django.conf import settings
 from django.test import TestCase
 from django.utils import timezone
 
 from documents import tasks
-from documents.models import Document
+from documents.models import Document, Tag, Correspondent, DocumentType
 from documents.sanity_checker import SanityError, SanityFailedError
 from documents.tests.utils import DirectoriesMixin
 
@@ -22,8 +23,55 @@ class TestTasks(DirectoriesMixin, TestCase):
 
         tasks.index_optimize()
 
-    def test_train_classifier(self):
+    @mock.patch("documents.tasks.load_classifier")
+    def test_train_classifier_no_auto_matching(self, load_classifier):
         tasks.train_classifier()
+        load_classifier.assert_not_called()
+
+    @mock.patch("documents.tasks.load_classifier")
+    def test_train_classifier_with_auto_tag(self, load_classifier):
+        load_classifier.return_value = None
+        Tag.objects.create(matching_algorithm=Tag.MATCH_AUTO, name="test")
+        tasks.train_classifier()
+        load_classifier.assert_called_once()
+        self.assertFalse(os.path.isfile(settings.MODEL_FILE))
+
+    @mock.patch("documents.tasks.load_classifier")
+    def test_train_classifier_with_auto_type(self, load_classifier):
+        load_classifier.return_value = None
+        DocumentType.objects.create(matching_algorithm=Tag.MATCH_AUTO, name="test")
+        tasks.train_classifier()
+        load_classifier.assert_called_once()
+        self.assertFalse(os.path.isfile(settings.MODEL_FILE))
+
+    @mock.patch("documents.tasks.load_classifier")
+    def test_train_classifier_with_auto_correspondent(self, load_classifier):
+        load_classifier.return_value = None
+        Correspondent.objects.create(matching_algorithm=Tag.MATCH_AUTO, name="test")
+        tasks.train_classifier()
+        load_classifier.assert_called_once()
+        self.assertFalse(os.path.isfile(settings.MODEL_FILE))
+
+    def test_train_classifier(self):
+        c = Correspondent.objects.create(matching_algorithm=Tag.MATCH_AUTO, name="test")
+        doc = Document.objects.create(correspondent=c, content="test", title="test")
+        self.assertFalse(os.path.isfile(settings.MODEL_FILE))
+
+        tasks.train_classifier()
+        self.assertTrue(os.path.isfile(settings.MODEL_FILE))
+        mtime = os.stat(settings.MODEL_FILE).st_mtime
+
+        tasks.train_classifier()
+        self.assertTrue(os.path.isfile(settings.MODEL_FILE))
+        mtime2 = os.stat(settings.MODEL_FILE).st_mtime
+        self.assertEqual(mtime, mtime2)
+
+        doc.content = "test2"
+        doc.save()
+        tasks.train_classifier()
+        self.assertTrue(os.path.isfile(settings.MODEL_FILE))
+        mtime3 = os.stat(settings.MODEL_FILE).st_mtime
+        self.assertNotEqual(mtime2, mtime3)
 
     @mock.patch("documents.tasks.sanity_checker.check_sanity")
     def test_sanity_check(self, m):
@@ -35,7 +83,7 @@ class TestTasks(DirectoriesMixin, TestCase):
         self.assertRaises(SanityFailedError, tasks.sanity_check)
         m.assert_called_once()
 
-    def test_culk_update_documents(self):
+    def test_bulk_update_documents(self):
         doc1 = Document.objects.create(title="test", content="my document", checksum="wow", added=timezone.now(),
                                 created=timezone.now(), modified=timezone.now())