From 13201dbfffc939a8a415a929325727b86ae842aa Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Sat, 10 Feb 2024 11:02:40 -0800 Subject: [PATCH] Ensure all creations of directories create the parents too (#5711) --- src/documents/consumer.py | 8 ++++---- src/documents/index.py | 2 +- .../management/commands/document_archiver.py | 3 +-- .../management/commands/document_exporter.py | 2 +- .../management/commands/document_importer.py | 6 +++--- src/documents/parsers.py | 2 +- src/documents/tests/test_date_parsing.py | 15 --------------- src/documents/tests/test_file_handling.py | 12 ++++++------ src/documents/views.py | 4 ++-- src/paperless/settings.py | 2 +- src/paperless_mail/mail.py | 4 ++-- 11 files changed, 22 insertions(+), 38 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 01b25edea..92461484c 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -233,10 +233,10 @@ class Consumer(LoggingMixin): """ Ensure all required directories exist before attempting to use them """ - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) - os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True) - os.makedirs(settings.ORIGINALS_DIR, exist_ok=True) - os.makedirs(settings.ARCHIVE_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) + settings.THUMBNAIL_DIR.mkdir(parents=True, exist_ok=True) + settings.ORIGINALS_DIR.mkdir(parents=True, exist_ok=True) + settings.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) def pre_check_asn_value(self): """ diff --git a/src/documents/index.py b/src/documents/index.py index 90bc15e7b..de651c13d 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -88,7 +88,7 @@ def open_index(recreate=False) -> FileIndex: logger.exception("Error while opening the index, recreating.") if not os.path.isdir(settings.INDEX_DIR): - os.makedirs(settings.INDEX_DIR, exist_ok=True) + settings.INDEX_DIR.mkdir(parents=True, exist_ok=True) return create_in(settings.INDEX_DIR, get_schema()) diff --git a/src/documents/management/commands/document_archiver.py b/src/documents/management/commands/document_archiver.py index 96b3b50ab..12677dd79 100644 --- a/src/documents/management/commands/document_archiver.py +++ b/src/documents/management/commands/document_archiver.py @@ -1,6 +1,5 @@ import logging import multiprocessing -import os import tqdm from django import db @@ -52,7 +51,7 @@ class Command(MultiProcessMixin, ProgressBarMixin, BaseCommand): self.handle_processes_mixin(**options) self.handle_progress_bar_mixin(**options) - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) overwrite = options["overwrite"] diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index 9bdbd1c51..faa42ba31 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -182,7 +182,7 @@ class Command(BaseCommand): if self.zip_export: self.original_target = self.target - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp_dir = tempfile.TemporaryDirectory( dir=settings.SCRATCH_DIR, prefix="paperless-export", diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index c166ec0cb..88ac2b91a 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -243,9 +243,9 @@ class Command(BaseCommand): ) from e def _import_files_from_manifest(self, progress_bar_disable): - os.makedirs(settings.ORIGINALS_DIR, exist_ok=True) - os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True) - os.makedirs(settings.ARCHIVE_DIR, exist_ok=True) + settings.ORIGINALS_DIR.mkdir(parents=True, exist_ok=True) + settings.THUMBNAIL_DIR.mkdir(parents=True, exist_ok=True) + settings.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) self.stdout.write("Copy files into paperless...") diff --git a/src/documents/parsers.py b/src/documents/parsers.py index b791d8322..a2b4677e6 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -322,7 +322,7 @@ class DocumentParser(LoggingMixin): super().__init__() self.logging_group = logging_group self.settings = self.get_settings() - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) self.tempdir = Path( tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR), ) diff --git a/src/documents/tests/test_date_parsing.py b/src/documents/tests/test_date_parsing.py index d4ea71be5..4c75e5f6b 100644 --- a/src/documents/tests/test_date_parsing.py +++ b/src/documents/tests/test_date_parsing.py @@ -1,7 +1,4 @@ import datetime -import os -import shutil -from uuid import uuid4 from dateutil import tz from django.conf import settings @@ -13,17 +10,6 @@ from documents.parsers import parse_date_generator class TestDate(TestCase): - SAMPLE_FILES = os.path.join( - os.path.dirname(__file__), - "../../paperless_tesseract/tests/samples", - ) - SCRATCH = f"/tmp/paperless-tests-{str(uuid4())[:8]}" - - def setUp(self): - os.makedirs(self.SCRATCH, exist_ok=True) - - def tearDown(self): - shutil.rmtree(self.SCRATCH) def test_date_format_1(self): text = "lorem ipsum 130218 lorem ipsum" @@ -93,7 +79,6 @@ class TestDate(TestCase): datetime.datetime(2020, 3, 1, 0, 0, tzinfo=tz.gettz(settings.TIME_ZONE)), ) - @override_settings(SCRATCH_DIR=SCRATCH) def test_date_format_9(self): text = "lorem ipsum\n27. Nullmonth 2020\nMärz 2020\nlorem ipsum" self.assertEqual( diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 225008b78..a924e377b 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -470,12 +470,12 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): def test_try_delete_empty_directories(self): # Create our working directory - tmp = os.path.join(settings.ORIGINALS_DIR, "test_delete_empty") - os.makedirs(tmp) + tmp: Path = settings.ORIGINALS_DIR / "test_delete_empty" + tmp.mkdir(exist_ok=True, parents=True) - os.makedirs(os.path.join(tmp, "notempty")) - Path(os.path.join(tmp, "notempty", "file")).touch() - os.makedirs(os.path.join(tmp, "notempty", "empty")) + (tmp / "notempty").mkdir(exist_ok=True, parents=True) + (tmp / "notempty" / "file").touch() + (tmp / "notempty" / "empty").mkdir(exist_ok=True, parents=True) delete_empty_directories( os.path.join(tmp, "notempty", "empty"), @@ -647,7 +647,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test existing_archive_file = os.path.join(settings.ARCHIVE_DIR, "none", "my_doc.pdf") Path(original).touch() Path(archive).touch() - os.makedirs(os.path.join(settings.ARCHIVE_DIR, "none")) + (settings.ARCHIVE_DIR / "none").mkdir(parents=True, exist_ok=True) Path(existing_archive_file).touch() doc = Document.objects.create( mime_type="application/pdf", diff --git a/src/documents/views.py b/src/documents/views.py index 3be7f4ec6..c73a8050b 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -904,7 +904,7 @@ class PostDocumentView(GenericAPIView): t = int(mktime(datetime.now().timetuple())) - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp_file_path = Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) / Path( pathvalidate.sanitize_filename(doc_name), @@ -1136,7 +1136,7 @@ class BulkDownloadView(GenericAPIView): content = serializer.validated_data.get("content") follow_filename_format = serializer.validated_data.get("follow_formatting") - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp = tempfile.NamedTemporaryFile( dir=settings.SCRATCH_DIR, suffix="-compressed-archive", diff --git a/src/paperless/settings.py b/src/paperless/settings.py index a9792db9f..163492b1e 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -689,7 +689,7 @@ USE_TZ = True setup_logging_queues() -os.makedirs(LOGGING_DIR, exist_ok=True) +LOGGING_DIR.mkdir(parents=True, exist_ok=True) LOGROTATE_MAX_SIZE = os.getenv("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024) LOGROTATE_MAX_BACKUPS = os.getenv("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 6514014ca..a4ff6478c 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -710,7 +710,7 @@ class MailAccountHandler(LoggingMixin): f"{message.subject} from {message.from_}", ) - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) temp_dir = Path( tempfile.mkdtemp( @@ -793,7 +793,7 @@ class MailAccountHandler(LoggingMixin): tag_ids, doc_type, ): - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) _, temp_filename = tempfile.mkstemp( prefix="paperless-mail-", dir=settings.SCRATCH_DIR,