Merge branch 'dev' into feature-permissions

src/documents/management/commands/document_consumer.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from fnmatch import filter
 from pathlib import Path
 from pathlib import PurePath
 from threading import Event
@@ -7,6 +8,7 @@ from threading import Thread
 from time import monotonic
 from time import sleep
 from typing import Final
+from typing import Set
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
@@ -25,15 +27,15 @@ except ImportError:  # pragma: nocover
 logger = logging.getLogger("paperless.management.consumer")
 
 
-def _tags_from_path(filepath):
-    """Walk up the directory tree from filepath to CONSUMPTION_DIR
-    and get or create Tag IDs for every directory.
+def _tags_from_path(filepath) -> Set[Tag]:
+    """
+    Walk up the directory tree from filepath to CONSUMPTION_DIR
+    and get or create Tag IDs for every directory.
+
+    Returns set of Tag models
     """
-    normalized_consumption_dir = os.path.abspath(
-        os.path.normpath(settings.CONSUMPTION_DIR),
-    )
     tag_ids = set()
-    path_parts = Path(filepath).relative_to(normalized_consumption_dir).parent.parts
+    path_parts = Path(filepath).relative_to(settings.CONSUMPTION_DIR).parent.parts
     for part in path_parts:
         tag_ids.add(
             Tag.objects.get_or_create(name__iexact=part, defaults={"name": part})[0].pk,
@@ -43,14 +45,41 @@ def _tags_from_path(filepath):
 
 
 def _is_ignored(filepath: str) -> bool:
-    normalized_consumption_dir = os.path.abspath(
-        os.path.normpath(settings.CONSUMPTION_DIR),
+    """
+    Checks if the given file should be ignored, based on configured
+    patterns.
+
+    Returns True if the file is ignored, False otherwise
+    """
+    filepath = os.path.abspath(
+        os.path.normpath(filepath),
     )
-    filepath_relative = PurePath(filepath).relative_to(normalized_consumption_dir)
-    return any(filepath_relative.match(p) for p in settings.CONSUMER_IGNORE_PATTERNS)
+
+    # Trim out the consume directory, leaving only filename and it's
+    # path relative to the consume directory
+    filepath_relative = PurePath(filepath).relative_to(settings.CONSUMPTION_DIR)
+
+    # March through the components of the path, including directories and the filename
+    # looking for anything matching
+    # foo/bar/baz/file.pdf -> (foo, bar, baz, file.pdf)
+    parts = []
+    for part in filepath_relative.parts:
+        # If the part is not the name (ie, it's a dir)
+        # Need to append the trailing slash or fnmatch doesn't match
+        # fnmatch("dir", "dir/*") == False
+        # fnmatch("dir/", "dir/*") == True
+        if part != filepath_relative.name:
+            part = part + "/"
+        parts.append(part)
+
+    for pattern in settings.CONSUMER_IGNORE_PATTERNS:
+        if len(filter(parts, pattern)):
+            return True
+
+    return False
 
 
-def _consume(filepath):
+def _consume(filepath: str) -> None:
     if os.path.isdir(filepath) or _is_ignored(filepath):
         return
 
@@ -103,7 +132,13 @@ def _consume(filepath):
         logger.exception("Error while consuming document")
 
 
-def _consume_wait_unmodified(file):
+def _consume_wait_unmodified(file: str) -> None:
+    """
+    Waits for the given file to appear unmodified based on file size
+    and modification time.  Will wait a configured number of seconds
+    and retry a configured number of times before either consuming or
+    giving up
+    """
     if _is_ignored(file):
         return
 

src/documents/tests/test_management_consumer.py

@@ -247,22 +247,85 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
 
     def test_is_ignored(self):
         test_paths = [
-            (os.path.join(self.dirs.consumption_dir, "foo.pdf"), False),
-            (os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"), False),
-            (os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"), True),
-            (
-                os.path.join(self.dirs.consumption_dir, "foo", ".DS_STORE", "bar.pdf"),
-                True,
-            ),
-            (os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"), True),
-            (os.path.join(self.dirs.consumption_dir, "._foo.pdf"), True),
-            (os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"), False),
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "foo.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    "foo",
+                    ".DS_STORE",
+                    "bar.pdf",
+                ),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    ".DS_STORE",
+                    "foo",
+                    "bar.pdf",
+                ),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".stfolder.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    ".stversions",
+                    "foo.pdf",
+                ),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".stversions.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "._foo.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "my_foo.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    "@eaDir",
+                    "SYNO@.fileindexdb",
+                    "_1jk.fnm",
+                ),
+                "ignore": True,
+            },
         ]
-        for file_path, expected_ignored in test_paths:
+        for test_setup in test_paths:
+            filepath = test_setup["path"]
+            expected_ignored_result = test_setup["ignore"]
             self.assertEqual(
-                expected_ignored,
-                document_consumer._is_ignored(file_path),
-                f'_is_ignored("{file_path}") != {expected_ignored}',
+                expected_ignored_result,
+                document_consumer._is_ignored(filepath),
+                f'_is_ignored("{filepath}") != {expected_ignored_result}',
             )
 
     @mock.patch("documents.management.commands.document_consumer.open")

src/locale/nl_NL/LC_MESSAGES/django.po

@@ -3,7 +3,7 @@ msgstr ""
 "Project-Id-Version: paperless-ngx\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2022-11-09 21:50+0000\n"
-"PO-Revision-Date: 2023-01-23 12:37\n"
+"PO-Revision-Date: 2023-01-27 19:22\n"
 "Last-Translator: \n"
 "Language-Team: Dutch\n"
 "Language: nl_NL\n"
@@ -368,15 +368,15 @@ msgstr "heeft tags in"
 
 #: documents/models.py:410
 msgid "ASN greater than"
-msgstr ""
+msgstr "ASN groter dan"
 
 #: documents/models.py:411
 msgid "ASN less than"
-msgstr ""
+msgstr "ASN kleiner dan"
 
 #: documents/models.py:412
 msgid "storage path is"
-msgstr ""
+msgstr "opslagpad is"
 
 #: documents/models.py:422
 msgid "rule type"
@@ -396,99 +396,99 @@ msgstr "filterregels"
 
 #: documents/models.py:536
 msgid "Task ID"
-msgstr ""
+msgstr "Taak ID"
 
 #: documents/models.py:537
 msgid "Celery ID for the Task that was run"
-msgstr ""
+msgstr "Celery ID voor de taak die werd uitgevoerd"
 
 #: documents/models.py:542
 msgid "Acknowledged"
-msgstr ""
+msgstr "Bevestigd"
 
 #: documents/models.py:543
 msgid "If the task is acknowledged via the frontend or API"
-msgstr ""
+msgstr "Of de taak is bevestigd via de frontend of de API"
 
 #: documents/models.py:549 documents/models.py:556
 msgid "Task Name"
-msgstr ""
+msgstr "Taaknaam"
 
 #: documents/models.py:550
 msgid "Name of the file which the Task was run for"
-msgstr ""
+msgstr "Naam van het bestand waarvoor de taak werd uitgevoerd"
 
 #: documents/models.py:557
 msgid "Name of the Task which was run"
-msgstr ""
+msgstr "Naam van de uitgevoerde taak"
 
 #: documents/models.py:562
 msgid "Task Positional Arguments"
-msgstr ""
+msgstr "Positionele argumenten voor taak"
 
 #: documents/models.py:564
 msgid "JSON representation of the positional arguments used with the task"
-msgstr ""
+msgstr "JSON weergave van de positionele argumenten die gebruikt worden voor de taak"
 
 #: documents/models.py:569
 msgid "Task Named Arguments"
-msgstr ""
+msgstr "Argumenten met naam voor taak"
 
 #: documents/models.py:571
 msgid "JSON representation of the named arguments used with the task"
-msgstr ""
+msgstr "JSON weergave van de argumenten met naam die gebruikt worden voor de taak"
 
 #: documents/models.py:578
 msgid "Task State"
-msgstr ""
+msgstr "Taakstatus"
 
 #: documents/models.py:579
 msgid "Current state of the task being run"
-msgstr ""
+msgstr "Huidige status van de taak die wordt uitgevoerd"
 
 #: documents/models.py:584
 msgid "Created DateTime"
-msgstr ""
+msgstr "Aangemaakt DateTime"
 
 #: documents/models.py:585
 msgid "Datetime field when the task result was created in UTC"
-msgstr ""
+msgstr "Datetime veld wanneer het resultaat van de taak werd aangemaakt in UTC"
 
 #: documents/models.py:590
 msgid "Started DateTime"
-msgstr ""
+msgstr "Gestart DateTime"
 
 #: documents/models.py:591
 msgid "Datetime field when the task was started in UTC"
-msgstr ""
+msgstr "Datetime veld wanneer de taak werd gestart in UTC"
 
 #: documents/models.py:596
 msgid "Completed DateTime"
-msgstr ""
+msgstr "Voltooid DateTime"
 
 #: documents/models.py:597
 msgid "Datetime field when the task was completed in UTC"
-msgstr ""
+msgstr "Datetime veld wanneer de taak werd voltooid in UTC"
 
 #: documents/models.py:602
 msgid "Result Data"
-msgstr ""
+msgstr "Resultaatgegevens"
 
 #: documents/models.py:604
 msgid "The data returned by the task"
-msgstr ""
+msgstr "Gegevens geretourneerd door de taak"
 
 #: documents/models.py:613
 msgid "Comment for the document"
-msgstr ""
+msgstr "Commentaar op het document"
 
 #: documents/models.py:642
 msgid "comment"
-msgstr ""
+msgstr "opmerking"
 
 #: documents/models.py:643
 msgid "comments"
-msgstr ""
+msgstr "opmerkingen"
 
 #: documents/serialisers.py:72
 #, python-format

src/paperless/settings.py

@@ -676,7 +676,7 @@ CONSUMER_IGNORE_PATTERNS = list(
     json.loads(
         os.getenv(
             "PAPERLESS_CONSUMER_IGNORE_PATTERNS",
-            '[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]',  # noqa: E501
+            '[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]',  # noqa: E501
         ),
     ),
 )

src/paperless/version.py

@@ -1,7 +1,7 @@
 from typing import Final
 from typing import Tuple
 
-__version__: Final[Tuple[int, int, int]] = (1, 12, 2)
+__version__: Final[Tuple[int, int, int]] = (1, 13, 0)
 # Version string like X.Y.Z
 __full_version_str__: Final[str] = ".".join(map(str, __version__))
 # Version string like X.Y