Merge branch 'ENH_filename_date_parsing' of https://github.com/jat255/paperless into jat255-ENH_filename_date_parsing

docs/guesswork.rst

@@ -43,6 +43,16 @@ These however wouldn't work:
 * ``Some Company Name, Invoice 2016-01-01, money, invoices.pdf``
 * ``Another Company- Letter of Reference.jpg``
 
+Do I have to be so strict about naming?
+---------------------------------------
+Rather than using the strict document naming rules, one can also set the option
+``PAPERLESS_FILENAME_DATE_ORDER`` in ``paperless.conf`` to any date order
+that is accepted by dateparser_. Doing so will cause ``paperless`` to default
+to any date format that is found in the title, instead of a date pulled from
+the document's text, without requiring the strict formatting of the document
+filename as described above.
+
+.. _dateparser: https://github.com/scrapinghub/dateparser/blob/v0.7.0/docs/usage.rst#settings
 
 .. _guesswork-content:
 

paperless.conf.example

@@ -127,6 +127,14 @@ PAPERLESS_DEBUG="false"
 # "true", the document will instead be opened in the browser, if possible.
 #PAPERLESS_INLINE_DOC="false"
 
+# By default, paperless will check the document text for document date information.
+# Uncomment the line below to enable checking the document filename for date
+# information. The date order can be set to any option as specified in
+# https://dateparser.readthedocs.io/en/latest/#settings. The filename will be
+# checked first, and if nothing is found, the document text will be checked
+# as normal.
+#PAPERLESS_FILENAME_DATE_ORDER="YMD"
+
 #
 # The following values use sensible defaults for modern systems, but if you're
 # running Paperless on a low-resource device (like a Raspberry Pi), modifying

src/documents/parsers.py

@@ -14,14 +14,18 @@ from django.utils import timezone
 # - XX.YY.ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
 # - XX/YY/ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
 # - XX-YY-ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
+# - ZZZZ.XX.YY with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
+# - ZZZZ/XX/YY with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
+# - ZZZZ-XX-YY with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
 # - XX. MONTH ZZZZ with XX being 1 or 2 and ZZZZ being 2 or 4 digits
 # - MONTH ZZZZ, with ZZZZ being 4 digits
 # - MONTH XX, ZZZZ with XX being 1 or 2 and ZZZZ being 4 digits
 DATE_REGEX = re.compile(
-    r'\b([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})\b|' +
+    r'(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|' +
-    r'\b([0-9]{1,2}[\. ]+[^ ]{3,9} ([0-9]{4}|[0-9]{2}))\b|' +
+    r'(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|' +
-    r'\b([^\W\d_]{3,9} [0-9]{1,2}, ([0-9]{4}))\b|' +
+    r'(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[^ ]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|' +
-    r'\b([^\W\d_]{3,9} [0-9]{4})\b'
+    r'(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{1,2}, ([0-9]{4}))(\b|(?=([_-])))|' +
+    r'(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{4})(\b|(?=([_-])))'
 )
 
 
@@ -37,6 +41,7 @@ class DocumentParser:
 
     SCRATCH = settings.SCRATCH_DIR
     DATE_ORDER = settings.DATE_ORDER
+    FILENAME_DATE_ORDER = settings.FILENAME_DATE_ORDER
     OPTIPNG = settings.OPTIPNG_BINARY
 
     def __init__(self, path):
@@ -75,30 +80,53 @@ class DocumentParser:
         Returns the date of the document.
         """
 
+        def __parser__(ds, date_order):
+            """
+            Call dateparser.parse with a particular date ordering
+            """
+            return dateparser.parse(ds,
+                                    settings={"DATE_ORDER": date_order,
+                                              "PREFER_DAY_OF_MONTH": "first",
+                                              "RETURN_AS_TIMEZONE_AWARE":
+                                                  True})
         date = None
         date_string = None
 
+        next_year = timezone.now().year + 5  # Arbitrary 5 year future limit
+        title = os.path.basename(self.document_path)
+
+        # if filename date parsing is enabled, search there first:
+        if self.FILENAME_DATE_ORDER:
+            self.log("info", "Checking document title for date")
+            for m in re.finditer(DATE_REGEX, title):
+                date_string = m.group(0)
+
+                try:
+                    date = __parser__(date_string, self.FILENAME_DATE_ORDER)
+                except TypeError:
+                    # Skip all matches that do not parse to a proper date
+                    continue
+
+                if date is not None and next_year > date.year > 1900:
+                    self.log("info",
+                             "Detected document date {} based on string {} "
+                             "from document title"
+                             "".format(date.isoformat(), date_string))
+                    return date
+
         try:
+            # getting text after checking filename will save time if only
+            # looking at the filename instead of the whole text
             text = self.get_text()
         except ParseError:
             return None
 
-        next_year = timezone.now().year + 5  # Arbitrary 5 year future limit
+        # Iterate through all regex matches in text and try to parse the date
-
-        # Iterate through all regex matches and try to parse the date
         for m in re.finditer(DATE_REGEX, text):
-
             date_string = m.group(0)
 
             try:
-                date = dateparser.parse(
+                date = __parser__(date_string, self.DATE_ORDER)
-                    date_string,
-                    settings={
-                        "DATE_ORDER": self.DATE_ORDER,
-                        "PREFER_DAY_OF_MONTH": "first",
-                        "RETURN_AS_TIMEZONE_AWARE": True
-                    }
-                )
             except TypeError:
                 # Skip all matches that do not parse to a proper date
                 continue

src/paperless/settings.py

@@ -306,6 +306,7 @@ FY_END = os.getenv("PAPERLESS_FINANCIAL_YEAR_END")
 
 # Specify the default date order (for autodetected dates)
 DATE_ORDER = os.getenv("PAPERLESS_DATE_ORDER", "DMY")
+FILENAME_DATE_ORDER = os.getenv("PAPERLESS_FILENAME_DATE_ORDER")
 
 # Specify for how many years a correspondent is considered recent. Recent
 # correspondents will be shown in a separate "Recent correspondents" filter as

src/paperless_tesseract/tests/test_date.py

@@ -8,6 +8,7 @@ from dateutil import tz
 from django.test import TestCase, override_settings
 
 from ..parsers import RasterisedDocumentParser
+from django.conf import settings
 
 
 class TestDate(TestCase):
@@ -59,9 +60,11 @@ class TestDate(TestCase):
         input_file = os.path.join(self.SAMPLE_FILES, "")
         document = RasterisedDocumentParser(input_file)
         document._text = "lorem ipsum 13.02.2018 lorem ipsum"
+        date = document.get_date()
         self.assertEqual(
-            document.get_date(),
+            date,
-            datetime.datetime(2018, 2, 13, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 2, 13, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -72,10 +75,13 @@ class TestDate(TestCase):
         input_file = os.path.join(self.SAMPLE_FILES, "")
         document = RasterisedDocumentParser(input_file)
         document._text = (
-            "lorem ipsum 130218, 2018, 20180213 and 13.02.2018 lorem ipsum")
+            "lorem ipsum 130218, 2018, 20180213 and lorem 13.02.2018 lorem "
+            "ipsum")
+        date = document.get_date()
         self.assertEqual(
-            document.get_date(),
+            date,
-            datetime.datetime(2018, 2, 13, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 2, 13, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -110,9 +116,11 @@ class TestDate(TestCase):
             "März 2019\n"
             "lorem ipsum"
         )
+        date = document.get_date()
         self.assertEqual(
-            document.get_date(),
+            date,
-            datetime.datetime(2019, 3, 1, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2019, 3, 1, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -134,7 +142,8 @@ class TestDate(TestCase):
                           "März 2020")
         self.assertEqual(document.get_date(),
                          datetime.datetime(2020, 3, 1, 0, 0,
-                                           tzinfo=tz.tzutc()))
+                                           tzinfo=tz.gettz(
+                                               settings.TIME_ZONE)))
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
@@ -149,7 +158,8 @@ class TestDate(TestCase):
                           "lorem ipsum")
         self.assertEqual(document.get_date(),
                          datetime.datetime(2020, 3, 1, 0, 0,
-                                           tzinfo=tz.tzutc()))
+                                           tzinfo=tz.gettz(
+                                               settings.TIME_ZONE)))
 
     @mock.patch(
         "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
@@ -158,11 +168,14 @@ class TestDate(TestCase):
     def test_get_text_1_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_1.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
+        date = document.get_date()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
-            document.get_date(),
+            date,
-            datetime.datetime(2018, 4, 1, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 4, 1, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -172,11 +185,13 @@ class TestDate(TestCase):
     def test_get_text_1_png(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_1.png")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), False)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 4, 1, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 4, 1, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -186,11 +201,13 @@ class TestDate(TestCase):
     def test_get_text_2_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_2.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2013, 2, 1, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2013, 2, 1, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -200,11 +217,13 @@ class TestDate(TestCase):
     def test_get_text_2_png(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_2.png")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), False)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2013, 2, 1, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2013, 2, 1, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -215,11 +234,13 @@ class TestDate(TestCase):
     def test_get_text_3_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_3.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 10, 5, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 10, 5, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -230,11 +251,13 @@ class TestDate(TestCase):
     def test_get_text_3_png(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_3.png")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), False)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 10, 5, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 10, 5, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -245,11 +268,13 @@ class TestDate(TestCase):
     def test_get_text_4_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_4.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 10, 5, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 10, 5, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -260,11 +285,13 @@ class TestDate(TestCase):
     def test_get_text_4_png(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_4.png")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), False)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 10, 5, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 10, 5, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -274,11 +301,13 @@ class TestDate(TestCase):
     def test_get_text_5_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_5.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 12, 17, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 12, 17, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -288,11 +317,13 @@ class TestDate(TestCase):
     def test_get_text_5_png(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_5.png")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), False)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 12, 17, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 12, 17, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -307,7 +338,8 @@ class TestDate(TestCase):
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 12, 17, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 12, 17, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -322,7 +354,8 @@ class TestDate(TestCase):
         self.assertEqual(document._is_ocred(), False)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 12, 17, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 12, 17, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -332,6 +365,7 @@ class TestDate(TestCase):
     def test_get_text_6_pdf_eu(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_6.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(document.get_date(), None)
@@ -343,6 +377,7 @@ class TestDate(TestCase):
     def test_get_text_6_png_eu(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_6.png")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), False)
         self.assertEqual(document.get_date(), None)
@@ -354,11 +389,13 @@ class TestDate(TestCase):
     def test_get_text_7_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_7.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2018, 4, 1, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2018, 4, 1, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -368,11 +405,13 @@ class TestDate(TestCase):
     def test_get_text_8_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_8.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2017, 12, 31, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2017, 12, 31, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(
@@ -382,11 +421,82 @@ class TestDate(TestCase):
     def test_get_text_9_pdf(self):
         input_file = os.path.join(self.SAMPLE_FILES, "tests_date_9.pdf")
         document = RasterisedDocumentParser(input_file)
+        document.DATE_ORDER = 'DMY'
         document.get_text()
         self.assertEqual(document._is_ocred(), True)
         self.assertEqual(
             document.get_date(),
-            datetime.datetime(2017, 12, 31, 0, 0, tzinfo=tz.tzutc())
+            datetime.datetime(2017, 12, 31, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
+        )
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_filename_date_1_pdf(self):
+        input_file = os.path.join(self.SAMPLE_FILES,
+                                  "tests_date_in_filename_2018-03-20_1.pdf")
+        document = RasterisedDocumentParser(input_file)
+        document.FILENAME_DATE_ORDER = 'YMD'
+        document.get_text()
+        date = document.get_date()
+        self.assertEqual(document._is_ocred(), True)
+        self.assertEqual(
+            date,
+            datetime.datetime(2018, 3, 20, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
+        )
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_filename_date_1_png(self):
+        input_file = os.path.join(self.SAMPLE_FILES,
+                                  "tests_date_in_filename_2018-03-20_1.png")
+        document = RasterisedDocumentParser(input_file)
+        document.FILENAME_DATE_ORDER = 'YMD'
+        date = document.get_date()
+        self.assertEqual(document._is_ocred(), False)
+        self.assertEqual(
+            date,
+            datetime.datetime(2018, 3, 20, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
+        )
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_filename_date_2_pdf(self):
+        input_file = os.path.join(self.SAMPLE_FILES,
+                                  "2013-12-11_tests_date_in_filename_2.pdf")
+        document = RasterisedDocumentParser(input_file)
+        document.FILENAME_DATE_ORDER = 'YMD'
+        date = document.get_date()
+        self.assertEqual(document._is_ocred(), True)
+        self.assertEqual(
+            date,
+            datetime.datetime(2013, 12, 11, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
+        )
+
+    @mock.patch(
+        "paperless_tesseract.parsers.RasterisedDocumentParser.SCRATCH",
+        SCRATCH
+    )
+    def test_filename_date_2_png(self):
+        input_file = os.path.join(self.SAMPLE_FILES,
+                                  "2013-12-11_tests_date_in_filename_2.png")
+        document = RasterisedDocumentParser(input_file)
+        document.FILENAME_DATE_ORDER = 'YMD'
+        date = document.get_date()
+        self.assertEqual(document._is_ocred(), False)
+        self.assertEqual(
+            date,
+            datetime.datetime(2013, 12, 11, 0, 0,
+                              tzinfo=tz.gettz(settings.TIME_ZONE))
         )
 
     @mock.patch(