Merge branch 'dev' into celery-tasks

src/documents/parsers.py

@@ -20,13 +20,16 @@ from django.utils import timezone
 # - XX. MONTH ZZZZ with XX being 1 or 2 and ZZZZ being 2 or 4 digits
 # - MONTH ZZZZ, with ZZZZ being 4 digits
 # - MONTH XX, ZZZZ with XX being 1 or 2 and ZZZZ being 4 digits
+from documents.loggers import LoggingMixin
 from documents.signals import document_consumer_declaration
 
+# TODO: isnt there a date parsing library for this?
+
 DATE_REGEX = re.compile(
-    r'(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|' +  # NOQA: E501
-    r'(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|' +  # NOQA: E501
-    r'(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[^ ]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|' +  # NOQA: E501
-    r'(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{1,2}, ([0-9]{4}))(\b|(?=([_-])))|' +
+    r'(\b|(?!=([_-])))([0-9]{1,2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{4}|[0-9]{2})(\b|(?=([_-])))|'   # NOQA: E501
+    r'(\b|(?!=([_-])))([0-9]{4}|[0-9]{2})[\.\/-]([0-9]{1,2})[\.\/-]([0-9]{1,2})(\b|(?=([_-])))|'   # NOQA: E501
+    r'(\b|(?!=([_-])))([0-9]{1,2}[\. ]+[^ ]{3,9} ([0-9]{4}|[0-9]{2}))(\b|(?=([_-])))|'   # NOQA: E501
+    r'(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{1,2}, ([0-9]{4}))(\b|(?=([_-])))|'
     r'(\b|(?!=([_-])))([^\W\d_]{3,9} [0-9]{4})(\b|(?=([_-])))'
 )
 
@@ -39,17 +42,16 @@ def get_parser_class(doc):
     Determine the appropriate parser class based on the file
     """
 
-    parsers = []
-    for response in document_consumer_declaration.send(None):
-        parsers.append(response[1])
-
-    #TODO: add a check that checks parser availability.
-
     options = []
-    for parser in parsers:
-        result = parser(doc)
-        if result:
-            options.append(result)
+
+    # Sein letzter Befehl war: KOMMT! Und sie kamen. Alle. Sogar die Parser.
+
+    for response in document_consumer_declaration.send(None):
+        parser_declaration = response[1]
+        parser_test = parser_declaration["test"]
+
+        if parser_test(doc):
+            options.append(parser_declaration)
 
     if not options:
         return None
@@ -59,7 +61,7 @@ def get_parser_class(doc):
         options, key=lambda _: _["weight"], reverse=True)[0]["parser"]
 
 
-def run_convert(input, output, density=None, scale=None, alpha=None, strip=False, trim=False, type=None, depth=None, extra=None, logging_group=None):
+def run_convert(input_file, output_file, density=None, scale=None, alpha=None, strip=False, trim=False, type=None, depth=None, extra=None, logging_group=None):
     environment = os.environ.copy()
     if settings.CONVERT_MEMORY_LIMIT:
         environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
@@ -74,7 +76,7 @@ def run_convert(input, output, density=None, scale=None, alpha=None, strip=False
     args += ['-trim'] if trim else []
     args += ['-type', str(type)] if type else []
     args += ['-depth', str(depth)] if depth else []
-    args += [input, output]
+    args += [input_file, output_file]
 
     logger.debug("Execute: " + " ".join(args), extra={'group': logging_group})
 
@@ -100,17 +102,17 @@ class ParseError(Exception):
     pass
 
 
-class DocumentParser:
+class DocumentParser(LoggingMixin):
     """
     Subclass this to make your own parser.  Have a look at
     `paperless_tesseract.parsers` for inspiration.
     """
 
     def __init__(self, path, logging_group, progress_callback):
+        super().__init__()
+        self.logging_group = logging_group
         self.document_path = path
         self.tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR)
-        self.logger = logging.getLogger(__name__)
-        self.logging_group = logging_group
         self.progress_callback = progress_callback
 
     def get_thumbnail(self):
@@ -121,16 +123,19 @@ class DocumentParser:
 
     def optimise_thumbnail(self, in_path):
 
-        out_path = os.path.join(self.tempdir, "optipng.png")
+        if settings.OPTIMIZE_THUMBNAILS:
+            out_path = os.path.join(self.tempdir, "optipng.png")
 
-        args = (settings.OPTIPNG_BINARY, "-silent", "-o5", in_path, "-out", out_path)
+            args = (settings.OPTIPNG_BINARY, "-silent", "-o5", in_path, "-out", out_path)
 
-        self.log('debug', 'Execute: ' + " ".join(args))
+            self.log('debug', 'Execute: ' + " ".join(args))
 
-        if not subprocess.Popen(args).wait() == 0:
-            raise ParseError("Optipng failed at {}".format(args))
+            if not subprocess.Popen(args).wait() == 0:
+                raise ParseError("Optipng failed at {}".format(args))
 
-        return out_path
+            return out_path
+        else:
+            return in_path
 
     def get_optimised_thumbnail(self):
         return self.optimise_thumbnail(self.get_thumbnail())
@@ -222,11 +227,6 @@ class DocumentParser:
 
         return date
 
-    def log(self, level, message):
-        getattr(self.logger, level)(message, extra={
-            "group": self.logging_group
-        })
-
     def cleanup(self):
         self.log("debug", "Deleting directory {}".format(self.tempdir))
         shutil.rmtree(self.tempdir)