Merge branch 'dev' into feature-ocrmypdf

docs/extending.rst

@@ -53,7 +53,8 @@ them running.
 Testing and code style:
 
 *   Run ``pytest`` in the src/ directory to execute all tests. This also generates a HTML coverage
-    report.
+    report. When runnings test, paperless.conf is loaded as well. However: the tests rely on the default
+    configuration. This is not ideal. But for now, make sure no settings except for DEBUG are overridden when testing.
 *   Run ``pycodestyle`` to test your code for issues with the configured code style settings.
 
     .. note::

src/documents/consumer.py

@@ -193,6 +193,13 @@ class Consumer(LoggingMixin):
                             f.read()).hexdigest()
                         document.save()
 
+                # Afte performing all database operations and moving files
+                # into place, tell paperless where the file is.
+                document.filename = os.path.basename(document.source_path)
+                # Saving the document now will trigger the filename handling
+                # logic.
+                document.save()
+
                 # Delete the file only if it was successfully consumed
                 self.log("debug", "Deleting file {}".format(self.path))
                 os.unlink(self.path)
@@ -248,12 +255,6 @@ class Consumer(LoggingMixin):
 
         self.apply_overrides(document)
 
-        document.filename = generate_filename(document)
-
-        # We need to save the document twice, since we need the PK of the
-        # document in order to create its filename above.
-        document.save()
-
         return document
 
     def apply_overrides(self, document):

src/documents/signals/handlers.py

@@ -224,7 +224,11 @@ def update_filename_and_move_files(sender, instance, **kwargs):
     try:
         os.rename(old_path, new_path)
         instance.filename = new_filename
-        instance.save()
+        # Don't save here to prevent infinite recursion.
+        Document.objects.filter(pk=instance.pk).update(filename=new_filename)
+
+        logging.getLogger(__name__).debug(
+            f"Moved file {old_path} to {new_path}.")
 
     except OSError as e:
         instance.filename = old_filename

src/documents/tests/test_consumer.py

@@ -563,13 +563,34 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs")
 
-        print(document.source_path)
-        print("===")
-
         self.assertEqual(document.title, "new docs")
         self.assertEqual(document.correspondent.name, "Bank")
         self.assertEqual(document.filename, "bank/new-docs-0000001.pdf")
 
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    @mock.patch("documents.signals.handlers.generate_filename")
+    def testFilenameHandlingUnstableFormat(self, m):
+
+        filenames = ["this", "that", "now this", "i cant decide"]
+
+        def get_filename():
+            f = filenames.pop()
+            filenames.insert(0, f)
+            return f
+
+        m.side_effect = lambda f: get_filename()
+
+        filename = self.get_test_file()
+
+        Tag.objects.create(name="test", is_inbox_tag=True)
+
+        document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs")
+
+        self.assertEqual(document.title, "new docs")
+        self.assertEqual(document.correspondent.name, "Bank")
+        self.assertIsNotNone(os.path.isfile(document.title))
+        self.assertTrue(os.path.isfile(document.source_path))
+
     @mock.patch("documents.consumer.DocumentClassifier")
     def testClassifyDocument(self, m):
         correspondent = Correspondent.objects.create(name="test")

src/documents/tests/test_file_handling.py

@@ -1,14 +1,15 @@
 import os
 import shutil
 from pathlib import Path
+from unittest import mock
 from uuid import uuid4
 
 from django.conf import settings
+from django.db import DatabaseError
 from django.test import TestCase, override_settings
 
 from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories
 from ..models import Document, Correspondent
-from ..signals.handlers import update_filename_and_move_files
 
 
 class TestDate(TestCase):
@@ -133,18 +134,14 @@ class TestDate(TestCase):
         document.correspondent = Correspondent.objects.get_or_create(
             name="test")[0]
 
-        # This will cause save() to fail.
+        with mock.patch("documents.signals.handlers.Document.objects.filter") as m:
-        document.checksum = document1.checksum
+            m.side_effect = DatabaseError()
+            document.save()
 
-        # Assume saving the document initially works, this gets called.
+            # Check proper handling of files
-        # After renaming, an error occurs, and filename is not saved:
+            self.assertTrue(os.path.isfile(document.source_path))
-        # document should still be available at document.filename.
+            self.assertEqual(os.path.isfile(settings.MEDIA_ROOT + "/documents/originals/none/none-{:07d}.pdf".format(document.pk)), True)
-        update_filename_and_move_files(None, document)
+            self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk))
-
-        # Check proper handling of files
-        self.assertTrue(os.path.isfile(document.source_path))
-        self.assertEqual(os.path.isfile(settings.MEDIA_ROOT + "/documents/originals/none/none-{:07d}.pdf".format(document.pk)), True)
-        self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk))
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
     def test_document_delete(self):

src/documents/tests/test_management_decrypt.py

@@ -34,8 +34,8 @@ class TestDecryptDocuments(TestCase):
             PASSPHRASE="test"
         ).enable()
 
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "thumb", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg"))
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg"))
 
         Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
 

src/documents/tests/test_management_exporter.py

@@ -1,6 +1,7 @@
 import hashlib
 import json
 import os
+import shutil
 import tempfile
 
 from django.core.management import call_command
@@ -8,17 +9,19 @@ from django.test import TestCase, override_settings
 
 from documents.management.commands import document_exporter
 from documents.models import Document, Tag, DocumentType, Correspondent
+from documents.tests.utils import DirectoriesMixin
 
 
-class TestExporter(TestCase):
+class TestExporter(DirectoriesMixin, TestCase):
 
     @override_settings(
-        ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"),
-        THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"),
         PASSPHRASE="test"
     )
     def test_exporter(self):
-        file = os.path.join(os.path.dirname(__file__), "samples", "originals", "0000001.pdf")
+        shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
+        shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents"))
+
+        file = os.path.join(self.dirs.originals_dir, "0000001.pdf")
 
         with open(file, "rb") as f:
             checksum = hashlib.md5(f.read()).hexdigest()