Merge pull request #721 from paperless-ngx/bug-fix-date-ignore

Fix Ignore Date Parsing

src/documents/tests/test_consumer.py

@@ -1,3 +1,4 @@
+import datetime
 import os
 import re
 import shutil
@@ -5,6 +6,8 @@ import tempfile
 from unittest import mock
 from unittest.mock import MagicMock
 
+from dateutil import tz
+
 try:
     import zoneinfo
 except ImportError:
@@ -502,7 +505,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         self.assertRaisesMessage(
             ConsumerError,
-            "sample.pdf: The following error occured while consuming sample.pdf: NO.",
+            "sample.pdf: The following error occurred while consuming sample.pdf: NO.",
             self.consumer.try_consume_file,
             filename,
         )
@@ -654,6 +657,127 @@ class TestConsumer(DirectoriesMixin, TestCase):
         sanity_check()
 
 
+@mock.patch("documents.consumer.magic.from_file", fake_magic_from_file)
+class TestConsumerCreatedDate(DirectoriesMixin, TestCase):
+    def setUp(self):
+        super(TestConsumerCreatedDate, self).setUp()
+
+        # this prevents websocket message reports during testing.
+        patcher = mock.patch("documents.consumer.Consumer._send_progress")
+        self._send_progress = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        self.consumer = Consumer()
+
+    def test_consume_date_from_content(self):
+        """
+        GIVEN:
+            - File content with date in DMY (default) format
+
+        THEN:
+            - Should parse the date from the file content
+        """
+        src = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "documents",
+            "originals",
+            "0000005.pdf",
+        )
+        dst = os.path.join(self.dirs.scratch_dir, "sample.pdf")
+        shutil.copy(src, dst)
+
+        document = self.consumer.try_consume_file(dst)
+
+        self.assertEqual(
+            document.created,
+            datetime.datetime(1996, 2, 20, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
+    @override_settings(FILENAME_DATE_ORDER="YMD")
+    def test_consume_date_from_filename(self):
+        """
+        GIVEN:
+            - File content with date in DMY (default) format
+            - Filename with date in YMD format
+
+        THEN:
+            - Should parse the date from the filename
+        """
+        src = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "documents",
+            "originals",
+            "0000005.pdf",
+        )
+        dst = os.path.join(self.dirs.scratch_dir, "Scan - 2022-02-01.pdf")
+        shutil.copy(src, dst)
+
+        document = self.consumer.try_consume_file(dst)
+
+        self.assertEqual(
+            document.created,
+            datetime.datetime(2022, 2, 1, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
+    def test_consume_date_filename_date_use_content(self):
+        """
+        GIVEN:
+            - File content with date in DMY (default) format
+            - Filename date parsing disabled
+            - Filename with date in YMD format
+
+        THEN:
+            - Should parse the date from the content
+        """
+        src = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "documents",
+            "originals",
+            "0000005.pdf",
+        )
+        dst = os.path.join(self.dirs.scratch_dir, "Scan - 2022-02-01.pdf")
+        shutil.copy(src, dst)
+
+        document = self.consumer.try_consume_file(dst)
+
+        self.assertEqual(
+            document.created,
+            datetime.datetime(1996, 2, 20, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
+    @override_settings(
+        IGNORE_DATES=(datetime.date(2010, 12, 13), datetime.date(2011, 11, 12)),
+    )
+    def test_consume_date_use_content_with_ignore(self):
+        """
+        GIVEN:
+            - File content with dates in DMY (default) format
+            - File content includes ignored dates
+
+        THEN:
+            - Should parse the date from the filename
+        """
+        src = os.path.join(
+            os.path.dirname(__file__),
+            "samples",
+            "documents",
+            "originals",
+            "0000006.pdf",
+        )
+        dst = os.path.join(self.dirs.scratch_dir, "0000006.pdf")
+        shutil.copy(src, dst)
+
+        document = self.consumer.try_consume_file(dst)
+
+        self.assertEqual(
+            document.created,
+            datetime.datetime(1997, 2, 20, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
+
 class PreConsumeTestCase(TestCase):
     @mock.patch("documents.consumer.Popen")
     @override_settings(PRE_CONSUME_SCRIPT=None)

src/documents/tests/test_date_parsing.py

@@ -8,6 +8,7 @@ from django.conf import settings
 from django.test import override_settings
 from django.test import TestCase
 from documents.parsers import parse_date
+from paperless.settings import DATE_ORDER
 
 
 class TestDate(TestCase):
@@ -160,19 +161,112 @@ class TestDate(TestCase):
     def test_crazy_date_with_spaces(self, *args):
         self.assertIsNone(parse_date("", "20 408000l 2475"))
 
+    @override_settings(FILENAME_DATE_ORDER="YMD")
+    def test_filename_date_parse_valid_ymd(self, *args):
+        """
+        GIVEN:
+            - Date parsing from the filename is enabled
+            - Filename date format is with Year Month Day (YMD)
+            - Filename contains date matching the format
+
+        THEN:
+            - Should parse the date from the filename
+        """
+        self.assertEqual(
+            parse_date("/tmp/Scan-2022-04-01.pdf", "No date in here"),
+            datetime.datetime(2022, 4, 1, 0, 0, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
+    @override_settings(FILENAME_DATE_ORDER="DMY")
+    def test_filename_date_parse_valid_dmy(self, *args):
+        """
+        GIVEN:
+            - Date parsing from the filename is enabled
+            - Filename date format is with Day Month Year (DMY)
+            - Filename contains date matching the format
+
+        THEN:
+            - Should parse the date from the filename
+        """
+        self.assertEqual(
+            parse_date("/tmp/Scan-10.01.2021.pdf", "No date in here"),
+            datetime.datetime(2021, 1, 10, 0, 0, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
     @override_settings(FILENAME_DATE_ORDER="YMD")
     def test_filename_date_parse_invalid(self, *args):
+        """
+        GIVEN:
+            - Date parsing from the filename is enabled
+            - Filename includes no date
+            - File content includes no date
+
+        THEN:
+            - No date is parsed
+        """
         self.assertIsNone(
             parse_date("/tmp/20 408000l 2475 - test.pdf", "No date in here"),
         )
 
+    @override_settings(
+        FILENAME_DATE_ORDER="YMD",
+        IGNORE_DATES=(datetime.date(2022, 4, 1),),
+    )
+    def test_filename_date_ignored_use_content(self, *args):
+        """
+        GIVEN:
+            - Date parsing from the filename is enabled
+            - Filename date format is with Day Month Year (YMD)
+            - Date order is Day Month Year (DMY, the default)
+            - Filename contains date matching the format
+            - Filename date is an ignored date
+            - File content includes a date
+
+        THEN:
+            - Should parse the date from the content not filename
+        """
+        self.assertEqual(
+            parse_date("/tmp/Scan-2022-04-01.pdf", "The matching date is 24.03.2022"),
+            datetime.datetime(2022, 3, 24, 0, 0, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
     @override_settings(
         IGNORE_DATES=(datetime.date(2019, 11, 3), datetime.date(2020, 1, 17)),
     )
-    def test_ignored_dates(self, *args):
+    def test_ignored_dates_default_order(self, *args):
+        """
+        GIVEN:
+            - Ignore dates have been set
+            - File content includes ignored dates
+            - File content includes 1 non-ignored date
+
+        THEN:
+            - Should parse the date non-ignored date from content
+        """
         text = "lorem ipsum 110319, 20200117 and lorem 13.02.2018 lorem " "ipsum"
-        date = parse_date("", text)
         self.assertEqual(
-            date,
+            parse_date("", text),
+            datetime.datetime(2018, 2, 13, 0, 0, tzinfo=tz.gettz(settings.TIME_ZONE)),
+        )
+
+    @override_settings(
+        IGNORE_DATES=(datetime.date(2019, 11, 3), datetime.date(2020, 1, 17)),
+        DATE_ORDER="YMD",
+    )
+    def test_ignored_dates_order_ymd(self, *args):
+        """
+        GIVEN:
+            - Ignore dates have been set
+            - Date order is Year Month Date (YMD)
+            - File content includes ignored dates
+            - File content includes 1 non-ignored date
+
+        THEN:
+            - Should parse the date non-ignored date from content
+        """
+        text = "lorem ipsum 190311, 20200117 and lorem 13.02.2018 lorem " "ipsum"
+
+        self.assertEqual(
+            parse_date("", text),
             datetime.datetime(2018, 2, 13, 0, 0, tzinfo=tz.gettz(settings.TIME_ZONE)),
         )