From b140935843b1ef11f25870be40eec468e3fdd9bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Wolf-Bastian=20P=C3=B6ttner?= <bastian@poettner.de>
Date: Sun, 28 Jan 2018 19:09:52 +0100
Subject: [PATCH] Add support for a heuristic that extracts the document date
 from its text

---
 requirements.txt                   |  1 +
 src/documents/consumer.py          |  8 +++++---
 src/documents/parsers.py           |  6 ++++++
 src/paperless/settings.py          |  3 +++
 src/paperless_tesseract/parsers.py | 25 +++++++++++++++++++++++++
 5 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 96336725b..fa857c677 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,7 @@ python-dateutil>=2.6.0
 python-dotenv>=0.6.2
 python-gnupg>=0.3.9
 pytz>=2016.10
+dateparser>=0.6.0
 gunicorn==19.7.1
 
 # For the tests
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index b4f300400..b7f7f61e7 100644
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -118,12 +118,14 @@ class Consumer(object):
 
             parsed_document = parser_class(doc)
             thumbnail = parsed_document.get_thumbnail()
+            date = parsed_document.get_date()
 
             try:
                 document = self._store(
                     parsed_document.get_text(),
                     doc,
-                    thumbnail
+                    thumbnail,
+                    date
                 )
             except ParseError as e:
 
@@ -174,7 +176,7 @@ class Consumer(object):
         return sorted(
             options, key=lambda _: _["weight"], reverse=True)[0]["parser"]
 
-    def _store(self, text, doc, thumbnail):
+    def _store(self, text, doc, thumbnail, date):
 
         file_info = FileInfo.from_path(doc)
 
@@ -182,7 +184,7 @@ class Consumer(object):
 
         self.log("debug", "Saving record to database")
 
-        created = file_info.created or timezone.make_aware(
+        created = file_info.created or date or timezone.make_aware(
                     datetime.datetime.fromtimestamp(stats.st_mtime))
 
         with open(doc, "rb") as f:
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index 9f63cbbcd..c11501487 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -35,6 +35,12 @@ class DocumentParser(object):
         """
         raise NotImplementedError()
 
+    def get_date(self):
+        """
+        Returns the date of the document.
+        """
+        raise NotImplementedError()
+
     def log(self, level, message):
         getattr(self.logger, level)(message, extra={
             "group": self.logging_group
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 696b0ddbd..3dad91e86 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -258,3 +258,6 @@ PAPERLESS_LIST_PER_PAGE = int(os.getenv("PAPERLESS_LIST_PER_PAGE", 100))
 
 FY_START = os.getenv("PAPERLESS_FINANCIAL_YEAR_START")
 FY_END = os.getenv("PAPERLESS_FINANCIAL_YEAR_END")
+
+# Specify the default date order (for autodetected dates)
+DATE_ORDER = os.getenv("PAPERLESS_DATE_ORDER", "DMY")
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index 0c5d039e1..abadbd3a5 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -3,6 +3,7 @@ import os
 import re
 import subprocess
 from multiprocessing.pool import Pool
+import dateparser
 
 import langdetect
 import pyocr
@@ -30,6 +31,7 @@ class RasterisedDocumentParser(DocumentParser):
     DENSITY = settings.CONVERT_DENSITY if settings.CONVERT_DENSITY else 300
     THREADS = int(settings.OCR_THREADS) if settings.OCR_THREADS else None
     UNPAPER = settings.UNPAPER_BINARY
+    DATE_ORDER = settings.DATE_ORDER
     DEFAULT_OCR_LANGUAGE = settings.OCR_LANGUAGE
 
     def get_thumbnail(self):
@@ -175,6 +177,29 @@ class RasterisedDocumentParser(DocumentParser):
         text += self._ocr(imgs[middle + 1:], self.DEFAULT_OCR_LANGUAGE)
         return text
 
+    def get_date(self):
+        text = self.get_text()
+
+        # This regular expression will try to find dates in the document at
+        # hand and will match the following formats:
+        # - XX.YY.ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
+        # - XX/YY/ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
+        # - XX-YY-ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits
+        # - XX. MONTH ZZZZ with XX being 1 or 2 and ZZZZ being 2 or 4 digits
+        # - MONTH ZZZZ
+        m = re.search(
+            r'\b([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})\b|' +
+            r'\b([0-9]{1,2}\. [^ ]{3,9} ([0-9]{4}|[0-9]{2}))\b|' +
+            r'\b([^ ]{3,9} [0-9]{4})\b', text)
+
+        if m is None:
+            return None
+
+        return dateparser.parse(m.group(0),
+                                settings={'DATE_ORDER': self.DATE_ORDER,
+                                          'PREFER_DAY_OF_MONTH': 'first',
+                                          'RETURN_AS_TIMEZONE_AWARE': True})
+
 
 def run_convert(*args):