diff --git a/docs/extending.rst b/docs/extending.rst index 29b08bf59..a0f14f2aa 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -53,7 +53,8 @@ them running. Testing and code style: * Run ``pytest`` in the src/ directory to execute all tests. This also generates a HTML coverage - report. + report. When runnings test, paperless.conf is loaded as well. However: the tests rely on the default + configuration. This is not ideal. But for now, make sure no settings except for DEBUG are overridden when testing. * Run ``pycodestyle`` to test your code for issues with the configured code style settings. .. note:: diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 7cdccc2e7..1accc423f 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -193,6 +193,13 @@ class Consumer(LoggingMixin): f.read()).hexdigest() document.save() + # Afte performing all database operations and moving files + # into place, tell paperless where the file is. + document.filename = os.path.basename(document.source_path) + # Saving the document now will trigger the filename handling + # logic. + document.save() + # Delete the file only if it was successfully consumed self.log("debug", "Deleting file {}".format(self.path)) os.unlink(self.path) @@ -248,12 +255,6 @@ class Consumer(LoggingMixin): self.apply_overrides(document) - document.filename = generate_filename(document) - - # We need to save the document twice, since we need the PK of the - # document in order to create its filename above. - document.save() - return document def apply_overrides(self, document): diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index e2d055fef..435143792 100755 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -224,7 +224,11 @@ def update_filename_and_move_files(sender, instance, **kwargs): try: os.rename(old_path, new_path) instance.filename = new_filename - instance.save() + # Don't save here to prevent infinite recursion. + Document.objects.filter(pk=instance.pk).update(filename=new_filename) + + logging.getLogger(__name__).debug( + f"Moved file {old_path} to {new_path}.") except OSError as e: instance.filename = old_filename diff --git a/src/documents/tests/samples/originals/0000001.pdf b/src/documents/tests/samples/documents/originals/0000001.pdf similarity index 100% rename from src/documents/tests/samples/originals/0000001.pdf rename to src/documents/tests/samples/documents/originals/0000001.pdf diff --git a/src/documents/tests/samples/originals/0000002.pdf.gpg b/src/documents/tests/samples/documents/originals/0000002.pdf.gpg similarity index 100% rename from src/documents/tests/samples/originals/0000002.pdf.gpg rename to src/documents/tests/samples/documents/originals/0000002.pdf.gpg diff --git a/src/documents/tests/samples/thumb/0000001.png b/src/documents/tests/samples/documents/thumbnails/0000001.png similarity index 100% rename from src/documents/tests/samples/thumb/0000001.png rename to src/documents/tests/samples/documents/thumbnails/0000001.png diff --git a/src/documents/tests/samples/thumb/0000002.png.gpg b/src/documents/tests/samples/documents/thumbnails/0000002.png.gpg similarity index 100% rename from src/documents/tests/samples/thumb/0000002.png.gpg rename to src/documents/tests/samples/documents/thumbnails/0000002.png.gpg diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index 61a98612d..cbf38ab6c 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -563,13 +563,34 @@ class TestConsumer(DirectoriesMixin, TestCase): document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs") - print(document.source_path) - print("===") - self.assertEqual(document.title, "new docs") self.assertEqual(document.correspondent.name, "Bank") self.assertEqual(document.filename, "bank/new-docs-0000001.pdf") + @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}") + @mock.patch("documents.signals.handlers.generate_filename") + def testFilenameHandlingUnstableFormat(self, m): + + filenames = ["this", "that", "now this", "i cant decide"] + + def get_filename(): + f = filenames.pop() + filenames.insert(0, f) + return f + + m.side_effect = lambda f: get_filename() + + filename = self.get_test_file() + + Tag.objects.create(name="test", is_inbox_tag=True) + + document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs") + + self.assertEqual(document.title, "new docs") + self.assertEqual(document.correspondent.name, "Bank") + self.assertIsNotNone(os.path.isfile(document.title)) + self.assertTrue(os.path.isfile(document.source_path)) + @mock.patch("documents.consumer.DocumentClassifier") def testClassifyDocument(self, m): correspondent = Correspondent.objects.create(name="test") diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 5ffd35f61..d799384e7 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -1,14 +1,15 @@ import os import shutil from pathlib import Path +from unittest import mock from uuid import uuid4 from django.conf import settings +from django.db import DatabaseError from django.test import TestCase, override_settings from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories from ..models import Document, Correspondent -from ..signals.handlers import update_filename_and_move_files class TestDate(TestCase): @@ -133,18 +134,14 @@ class TestDate(TestCase): document.correspondent = Correspondent.objects.get_or_create( name="test")[0] - # This will cause save() to fail. - document.checksum = document1.checksum + with mock.patch("documents.signals.handlers.Document.objects.filter") as m: + m.side_effect = DatabaseError() + document.save() - # Assume saving the document initially works, this gets called. - # After renaming, an error occurs, and filename is not saved: - # document should still be available at document.filename. - update_filename_and_move_files(None, document) - - # Check proper handling of files - self.assertTrue(os.path.isfile(document.source_path)) - self.assertEqual(os.path.isfile(settings.MEDIA_ROOT + "/documents/originals/none/none-{:07d}.pdf".format(document.pk)), True) - self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk)) + # Check proper handling of files + self.assertTrue(os.path.isfile(document.source_path)) + self.assertEqual(os.path.isfile(settings.MEDIA_ROOT + "/documents/originals/none/none-{:07d}.pdf".format(document.pk)), True) + self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk)) @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}") def test_document_delete(self): diff --git a/src/documents/tests/test_management_decrypt.py b/src/documents/tests/test_management_decrypt.py index 326276389..8f41e076f 100644 --- a/src/documents/tests/test_management_decrypt.py +++ b/src/documents/tests/test_management_decrypt.py @@ -34,8 +34,8 @@ class TestDecryptDocuments(TestCase): PASSPHRASE="test" ).enable() - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "thumb", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg")) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg")) Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index c8d1490d2..18171af75 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -1,6 +1,7 @@ import hashlib import json import os +import shutil import tempfile from django.core.management import call_command @@ -8,17 +9,19 @@ from django.test import TestCase, override_settings from documents.management.commands import document_exporter from documents.models import Document, Tag, DocumentType, Correspondent +from documents.tests.utils import DirectoriesMixin -class TestExporter(TestCase): +class TestExporter(DirectoriesMixin, TestCase): @override_settings( - ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"), - THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"), PASSPHRASE="test" ) def test_exporter(self): - file = os.path.join(os.path.dirname(__file__), "samples", "originals", "0000001.pdf") + shutil.rmtree(os.path.join(self.dirs.media_dir, "documents")) + shutil.copytree(os.path.join(os.path.dirname(__file__), "samples", "documents"), os.path.join(self.dirs.media_dir, "documents")) + + file = os.path.join(self.dirs.originals_dir, "0000001.pdf") with open(file, "rb") as f: checksum = hashlib.md5(f.read()).hexdigest()