diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 23d17abc9..f52dd5a7d 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -206,6 +206,8 @@ class Consumer(LoggingMixin): document.archive_checksum = hashlib.md5( f.read()).hexdigest() + # Don't save with the lock active. Saving will cause the file + # renaming logic to aquire the lock as well. document.save() # Delete the file only if it was successfully consumed diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index f0a74ca4f..6d407a7ab 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -1,5 +1,9 @@ import datetime +import hashlib import os +import random +import uuid +from concurrent.futures.thread import ThreadPoolExecutor from pathlib import Path from unittest import mock @@ -8,8 +12,10 @@ from django.db import DatabaseError from django.test import TestCase, override_settings from .utils import DirectoriesMixin -from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories +from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \ + generate_unique_filename from ..models import Document, Correspondent +from ..sanity_checker import check_sanity class TestFileHandling(DirectoriesMixin, TestCase): @@ -546,3 +552,42 @@ class TestFilenameGeneration(TestCase): def test_date(self): doc = Document.objects.create(title="does not matter", created=datetime.datetime(2020,5,21, 7,36,51, 153), mime_type="application/pdf", pk=2, checksum="2") self.assertEqual(generate_filename(doc), "2020-05-21.pdf") + + +def run(): + doc = Document.objects.create(checksum=str(uuid.uuid4()), title=str(uuid.uuid4()), content="wow") + doc.filename = generate_unique_filename(doc, settings.ORIGINALS_DIR) + Path(doc.thumbnail_path).touch() + with open(doc.source_path, "w") as f: + f.write(str(uuid.uuid4())) + with open(doc.source_path, "rb") as f: + doc.checksum = hashlib.md5(f.read()).hexdigest() + + with open(doc.archive_path, "w") as f: + f.write(str(uuid.uuid4())) + with open(doc.archive_path, "rb") as f: + doc.archive_checksum = hashlib.md5(f.read()).hexdigest() + + doc.save() + + for i in range(30): + doc.title = str(random.randrange(1, 5)) + doc.save() + + +class TestSuperMassive(DirectoriesMixin, TestCase): + + @override_settings(PAPERLESS_FILENAME_FORMAT="{title}") + def test_super_massive(self): + # try to save as many documents in parallel as possible. + # try to make the system fail. + + with ThreadPoolExecutor(max_workers=16) as executor: + results = [executor.submit(run) for i in range(16)] + + for r in results: + if r.exception(): + raise r.exception() + + # nope, everything still good. Thank you, lockfiles. + self.assertEqual(len(check_sanity()), 0)