Merge branch 'feature-ocrmypdf' into dev

src/documents/consumer.py

@@ -6,6 +6,7 @@ import os
 import magic
 from django.conf import settings
 from django.db import transaction
+from django.db.models import Q
 from django.utils import timezone
 
 from .classifier import DocumentClassifier, IncompatibleClassifierVersionError
@@ -13,7 +14,7 @@ from .file_handling import create_source_path_directory
 from .loggers import LoggingMixin
 from .models import Document, FileInfo, Correspondent, DocumentType, Tag
 from .parsers import ParseError, get_parser_class_for_mime_type, \
-    get_supported_file_extensions
+    get_supported_file_extensions, parse_date
 from .signals import (
     document_consumption_finished,
     document_consumption_started
@@ -58,7 +59,7 @@ class Consumer(LoggingMixin):
     def pre_check_duplicate(self):
         with open(self.path, "rb") as f:
             checksum = hashlib.md5(f.read()).hexdigest()
-        if Document.objects.filter(checksum=checksum).exists():
+        if Document.objects.filter(Q(checksum=checksum) | Q(archive_checksum=checksum)).exists():  # NOQA: E501
             if settings.CONSUMER_DELETE_DUPLICATES:
                 os.unlink(self.path)
             raise ConsumerError(
@@ -69,6 +70,7 @@ class Consumer(LoggingMixin):
         os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
         os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True)
         os.makedirs(settings.ORIGINALS_DIR, exist_ok=True)
+        os.makedirs(settings.ARCHIVE_DIR, exist_ok=True)
 
     def try_consume_file(self,
                          path,
@@ -124,7 +126,7 @@ class Consumer(LoggingMixin):
 
         # This doesn't parse the document yet, but gives us a parser.
 
-        document_parser = parser_class(self.path, self.logging_group)
+        document_parser = parser_class(self.logging_group)
 
         # However, this already created working directories which we have to
         # clean up.
@@ -132,13 +134,24 @@ class Consumer(LoggingMixin):
         # Parse the document. This may take some time.
 
         try:
-            self.log("debug", f"Generating thumbnail for {self.filename}...")
-            thumbnail = document_parser.get_optimised_thumbnail()
             self.log("debug", "Parsing {}...".format(self.filename))
+            document_parser.parse(self.path, mime_type)
+
+            self.log("debug", f"Generating thumbnail for {self.filename}...")
+            thumbnail = document_parser.get_optimised_thumbnail(
+                self.path, mime_type)
+
             text = document_parser.get_text()
             date = document_parser.get_date()
+            if not date:
+                date = parse_date(self.filename, text)
+            archive_path = document_parser.get_archive_path()
+
         except ParseError as e:
             document_parser.cleanup()
+            self.log(
+                "error",
+                f"Error while consuming document {self.filename}: {e}")
             raise ConsumerError(e)
 
         # Prepare the document classifier.
@@ -180,9 +193,24 @@ class Consumer(LoggingMixin):
                 # After everything is in the database, copy the files into
                 # place. If this fails, we'll also rollback the transaction.
 
+                # TODO: not required, since this is done by the file handling
+                #  logic
                 create_source_path_directory(document.source_path)
-                self._write(document, self.path, document.source_path)
-                self._write(document, thumbnail, document.thumbnail_path)
+
+                self._write(document.storage_type,
+                            self.path, document.source_path)
+
+                self._write(document.storage_type,
+                            thumbnail, document.thumbnail_path)
+
+                if archive_path and os.path.isfile(archive_path):
+                    self._write(document.storage_type,
+                                archive_path, document.archive_path)
+
+                    with open(archive_path, 'rb') as f:
+                        document.archive_checksum = hashlib.md5(
+                            f.read()).hexdigest()
+                        document.save()
 
                 # Afte performing all database operations and moving files
                 # into place, tell paperless where the file is.
@@ -195,6 +223,11 @@ class Consumer(LoggingMixin):
                 self.log("debug", "Deleting file {}".format(self.path))
                 os.unlink(self.path)
         except Exception as e:
+            self.log(
+                "error",
+                f"The following error occured while consuming "
+                f"{self.filename}: {e}"
+            )
             raise ConsumerError(e)
         finally:
             document_parser.cleanup()
@@ -259,7 +292,7 @@ class Consumer(LoggingMixin):
             for tag_id in self.override_tag_ids:
                 document.tags.add(Tag.objects.get(pk=tag_id))
 
-    def _write(self, document, source, target):
+    def _write(self, storage_type, source, target):
         with open(source, "rb") as read_file:
             with open(target, "wb") as write_file:
                 write_file.write(read_file.read())

src/documents/file_handling.py

@@ -10,10 +10,13 @@ def create_source_path_directory(source_path):
     os.makedirs(os.path.dirname(source_path), exist_ok=True)
 
 
-def delete_empty_directories(directory):
+def delete_empty_directories(directory, root):
+    if not os.path.isdir(directory):
+        return
+
     # Go up in the directory hierarchy and try to delete all directories
     directory = os.path.normpath(directory)
-    root = os.path.normpath(settings.ORIGINALS_DIR)
+    root = os.path.normpath(root)
 
     if not directory.startswith(root + os.path.sep):
         # don't do anything outside our originals folder.
@@ -101,3 +104,8 @@ def generate_filename(doc):
         filename += ".gpg"
 
     return filename
+
+
+def archive_name_from_filename(filename):
+
+    return os.path.splitext(filename)[0] + ".pdf"

src/documents/management/commands/document_archiver.py

@@ -0,0 +1,89 @@
+import hashlib
+import multiprocessing
+
+import logging
+import os
+import shutil
+import uuid
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from whoosh.writing import AsyncWriter
+
+from documents.models import Document
+from ... import index
+from ...mixins import Renderable
+from ...parsers import get_parser_class_for_mime_type
+
+
+def handle_document(document):
+    mime_type = document.mime_type
+
+    parser_class = get_parser_class_for_mime_type(mime_type)
+
+    parser = parser_class(logging_group=uuid.uuid4())
+    parser.parse(document.source_path, mime_type)
+    if parser.get_archive_path():
+        shutil.copy(parser.get_archive_path(), document.archive_path)
+        with document.archive_file as f:
+            document.archive_checksum = hashlib.md5(f.read()).hexdigest()
+    else:
+        logging.getLogger(__name__).warning(
+            f"Parser {parser} did not produce an archived document "
+            f"for {document.file_name}"
+        )
+
+    if parser.get_text():
+        document.content = parser.get_text()
+    document.save()
+
+    parser.cleanup()
+
+
+class Command(Renderable, BaseCommand):
+
+    help = """
+        Using the current classification model, assigns correspondents, tags
+        and document types to all documents, effectively allowing you to
+        back-tag all previously indexed documents with metadata created (or
+        modified) after their initial import.
+    """.replace("    ", "")
+
+    def __init__(self, *args, **kwargs):
+        self.verbosity = 0
+        BaseCommand.__init__(self, *args, **kwargs)
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-f", "--overwrite",
+            default=False,
+            action="store_true",
+            help="Recreates the archived document for documents that already "
+                 "have an archived version."
+        )
+
+    def handle(self, *args, **options):
+
+        os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
+
+        overwrite = options["overwrite"]
+
+        documents = Document.objects.all()
+
+        documents_to_process = filter(
+            lambda d: overwrite or not os.path.exists(d.archive_path),
+            documents
+        )
+
+        with multiprocessing.Pool(processes=settings.TASK_WORKERS) as pool:
+            list(
+                pool.imap(
+                    handle_document,
+                    list(documents_to_process)
+                )
+            )
+
+        ix = index.open_index()
+        with AsyncWriter(ix) as writer:
+            for d in documents_to_process:
+                index.update_document(writer, d)

src/documents/management/commands/document_exporter.py

@@ -7,7 +7,8 @@ from django.core import serializers
 from django.core.management.base import BaseCommand, CommandError
 
 from documents.models import Document, Correspondent, Tag, DocumentType
-from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME
+from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME, \
+    EXPORTER_ARCHIVE_NAME
 from paperless.db import GnuPG
 from ...mixins import Renderable
 
@@ -54,7 +55,6 @@ class Command(Renderable, BaseCommand):
             document = document_map[document_dict["pk"]]
 
             unique_filename = f"{document.pk:07}_{document.file_name}"
-
             file_target = os.path.join(self.target, unique_filename)
 
             thumbnail_name = unique_filename + "-thumbnail.png"
@@ -63,6 +63,14 @@ class Command(Renderable, BaseCommand):
             document_dict[EXPORTER_FILE_NAME] = unique_filename
             document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_name
 
+            if os.path.exists(document.archive_path):
+                archive_name = \
+                    f"{document.pk:07}_archive_{document.archive_file_name}"
+                archive_target = os.path.join(self.target, archive_name)
+                document_dict[EXPORTER_ARCHIVE_NAME] = archive_name
+            else:
+                archive_target = None
+
             print(f"Exporting: {file_target}")
 
             t = int(time.mktime(document.created.timetuple()))
@@ -76,11 +84,18 @@ class Command(Renderable, BaseCommand):
                     f.write(GnuPG.decrypted(document.thumbnail_file))
                     os.utime(thumbnail_target, times=(t, t))
 
+                if archive_target:
+                    with open(archive_target, "wb") as f:
+                        f.write(GnuPG.decrypted(document.archive_path))
+                        os.utime(archive_target, times=(t, t))
             else:
 
                 shutil.copy(document.source_path, file_target)
                 shutil.copy(document.thumbnail_path, thumbnail_target)
 
+                if archive_target:
+                    shutil.copy(document.archive_path, archive_target)
+
         manifest += json.loads(
             serializers.serialize("json", Correspondent.objects.all()))
 

src/documents/management/commands/document_importer.py

@@ -7,8 +7,8 @@ from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
 
 from documents.models import Document
-from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME
-from paperless.db import GnuPG
+from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME, \
+    EXPORTER_ARCHIVE_NAME
 from ...file_handling import generate_filename, create_source_path_directory
 from ...mixins import Renderable
 
@@ -79,23 +79,41 @@ class Command(Renderable, BaseCommand):
                     'appear to be in the source directory.'.format(doc_file)
                 )
 
+            if EXPORTER_ARCHIVE_NAME in record:
+                archive_file = record[EXPORTER_ARCHIVE_NAME]
+                if not os.path.exists(os.path.join(self.source, archive_file)):
+                    raise CommandError(
+                        f"The manifest file refers to {archive_file} which "
+                        f"does not appear to be in the source directory."
+                    )
+
     def _import_files_from_manifest(self):
 
-        storage_type = Document.STORAGE_TYPE_UNENCRYPTED
+        os.makedirs(settings.ORIGINALS_DIR, exist_ok=True)
+        os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True)
+        os.makedirs(settings.ARCHIVE_DIR, exist_ok=True)
 
         for record in self.manifest:
 
             if not record["model"] == "documents.document":
                 continue
 
-            doc_file = record[EXPORTER_FILE_NAME]
-            thumb_file = record[EXPORTER_THUMBNAIL_NAME]
             document = Document.objects.get(pk=record["pk"])
 
+            doc_file = record[EXPORTER_FILE_NAME]
             document_path = os.path.join(self.source, doc_file)
+
+            thumb_file = record[EXPORTER_THUMBNAIL_NAME]
             thumbnail_path = os.path.join(self.source, thumb_file)
 
-            document.storage_type = storage_type
+            if EXPORTER_ARCHIVE_NAME in record:
+                archive_file = record[EXPORTER_ARCHIVE_NAME]
+                archive_path = os.path.join(self.source, archive_file)
+            else:
+                archive_path = None
+
+            document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
+
             document.filename = generate_filename(document)
 
             if os.path.isfile(document.source_path):
@@ -106,5 +124,7 @@ class Command(Renderable, BaseCommand):
             print(f"Moving {document_path} to {document.source_path}")
             shutil.copy(document_path, document.source_path)
             shutil.copy(thumbnail_path, document.thumbnail_path)
+            if archive_path:
+                shutil.copy(archive_path, document.archive_path)
 
             document.save()

src/documents/migrations/1005_checksums.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.3 on 2020-11-29 00:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '1004_sanity_check_schedule'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='document',
+            name='archive_checksum',
+            field=models.CharField(blank=True, editable=False, help_text='The checksum of the archived document.', max_length=32, null=True),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='checksum',
+            field=models.CharField(editable=False, help_text='The checksum of the original document.', max_length=32, unique=True),
+        ),
+    ]

src/documents/models.py

@@ -11,6 +11,7 @@ from django.db import models
 from django.utils import timezone
 from django.utils.text import slugify
 
+from documents.file_handling import archive_name_from_filename
 from documents.parsers import get_default_file_extension
 
 
@@ -158,9 +159,15 @@ class Document(models.Model):
         max_length=32,
         editable=False,
         unique=True,
-        help_text="The checksum of the original document (before it was "
-                  "encrypted).  We use this to prevent duplicate document "
-                  "imports."
+        help_text="The checksum of the original document."
+    )
+
+    archive_checksum = models.CharField(
+        max_length=32,
+        editable=False,
+        blank=True,
+        null=True,
+        help_text="The checksum of the archived document."
     )
 
     created = models.DateTimeField(
@@ -225,10 +232,30 @@ class Document(models.Model):
     def source_file(self):
         return open(self.source_path, "rb")
 
+    @property
+    def archive_path(self):
+        if self.filename:
+            fname = archive_name_from_filename(self.filename)
+        else:
+            fname = "{:07}.pdf".format(self.pk)
+
+        return os.path.join(
+            settings.ARCHIVE_DIR,
+            fname
+        )
+
+    @property
+    def archive_file(self):
+        return open(self.archive_path, "rb")
+
     @property
     def file_name(self):
         return slugify(str(self)) + self.file_type
 
+    @property
+    def archive_file_name(self):
+        return slugify(str(self)) + ".pdf"
+
     @property
     def file_type(self):
         return get_default_file_extension(self.mime_type)

src/documents/parsers.py

@@ -131,21 +131,59 @@ def run_convert(input_file,
         raise ParseError("Convert failed at {}".format(args))
 
 
-def run_unpaper(pnm, logging_group=None):
-    pnm_out = pnm.replace(".pnm", ".unpaper.pnm")
+def parse_date(filename, text):
+    """
+    Returns the date of the document.
+    """
 
-    command_args = (settings.UNPAPER_BINARY, "--overwrite", "--quiet", pnm,
-                    pnm_out)
+    def __parser(ds, date_order):
+        """
+        Call dateparser.parse with a particular date ordering
+        """
+        return dateparser.parse(
+            ds,
+            settings={
+                "DATE_ORDER": date_order,
+                "PREFER_DAY_OF_MONTH": "first",
+                "RETURN_AS_TIMEZONE_AWARE":
+                True
+            }
+        )
 
-    logger.debug(f"Execute: {' '.join(command_args)}",
-                 extra={'group': logging_group})
+    date = None
 
-    if not subprocess.Popen(command_args,
-                            stdout=subprocess.DEVNULL,
-                            stderr=subprocess.DEVNULL).wait() == 0:
-        raise ParseError(f"Unpaper failed at {command_args}")
+    next_year = timezone.now().year + 5  # Arbitrary 5 year future limit
 
-    return pnm_out
+    # if filename date parsing is enabled, search there first:
+    if settings.FILENAME_DATE_ORDER:
+        for m in re.finditer(DATE_REGEX, filename):
+            date_string = m.group(0)
+
+            try:
+                date = __parser(date_string, settings.FILENAME_DATE_ORDER)
+            except (TypeError, ValueError):
+                # Skip all matches that do not parse to a proper date
+                continue
+
+            if date is not None and next_year > date.year > 1900:
+                return date
+
+    # Iterate through all regex matches in text and try to parse the date
+    for m in re.finditer(DATE_REGEX, text):
+        date_string = m.group(0)
+
+        try:
+            date = __parser(date_string, settings.DATE_ORDER)
+        except (TypeError, ValueError):
+            # Skip all matches that do not parse to a proper date
+            continue
+
+        if date is not None and next_year > date.year > 1900:
+            break
+        else:
+            date = None
+
+    return date
 
 
 class ParseError(Exception):
@@ -158,26 +196,35 @@ class DocumentParser(LoggingMixin):
     `paperless_tesseract.parsers` for inspiration.
     """
 
-    def __init__(self, path, logging_group):
+    def __init__(self, logging_group):
         super().__init__()
         self.logging_group = logging_group
-        self.document_path = path
         self.tempdir = tempfile.mkdtemp(
             prefix="paperless-", dir=settings.SCRATCH_DIR)
 
-    def get_thumbnail(self):
+        self.archive_path = None
+        self.text = None
+        self.date = None
+
+    def parse(self, document_path, mime_type):
+        raise NotImplementedError()
+
+    def get_archive_path(self):
+        return self.archive_path
+
+    def get_thumbnail(self, document_path, mime_type):
         """
         Returns the path to a file we can use as a thumbnail for this document.
         """
         raise NotImplementedError()
 
-    def optimise_thumbnail(self, in_path):
-
+    def get_optimised_thumbnail(self, document_path, mime_type):
+        thumbnail = self.get_thumbnail(document_path, mime_type)
         if settings.OPTIMIZE_THUMBNAILS:
-            out_path = os.path.join(self.tempdir, "optipng.png")
+            out_path = os.path.join(self.tempdir, "thumb_optipng.png")
 
             args = (settings.OPTIPNG_BINARY,
-                    "-silent", "-o5", in_path, "-out", out_path)
+                    "-silent", "-o5", thumbnail, "-out", out_path)
 
             self.log('debug', f"Execute: {' '.join(args)}")
 
@@ -186,97 +233,13 @@ class DocumentParser(LoggingMixin):
 
             return out_path
         else:
-            return in_path
-
-    def get_optimised_thumbnail(self):
-        return self.optimise_thumbnail(self.get_thumbnail())
+            return thumbnail
 
     def get_text(self):
-        """
-        Returns the text from the document and only the text.
-        """
-        raise NotImplementedError()
+        return self.text
 
     def get_date(self):
-        """
-        Returns the date of the document.
-        """
-
-        def __parser(ds, date_order):
-            """
-            Call dateparser.parse with a particular date ordering
-            """
-            return dateparser.parse(
-                ds,
-                settings={
-                    "DATE_ORDER": date_order,
-                    "PREFER_DAY_OF_MONTH": "first",
-                    "RETURN_AS_TIMEZONE_AWARE":
-                    True
-                }
-            )
-
-        date = None
-        date_string = None
-
-        next_year = timezone.now().year + 5  # Arbitrary 5 year future limit
-        title = os.path.basename(self.document_path)
-
-        # if filename date parsing is enabled, search there first:
-        if settings.FILENAME_DATE_ORDER:
-            self.log("info", "Checking document title for date")
-            for m in re.finditer(DATE_REGEX, title):
-                date_string = m.group(0)
-
-                try:
-                    date = __parser(date_string, settings.FILENAME_DATE_ORDER)
-                except (TypeError, ValueError):
-                    # Skip all matches that do not parse to a proper date
-                    continue
-
-                if date is not None and next_year > date.year > 1900:
-                    self.log(
-                        "info",
-                        "Detected document date {} based on string {} "
-                        "from document title"
-                        "".format(date.isoformat(), date_string)
-                    )
-                    return date
-
-        try:
-            # getting text after checking filename will save time if only
-            # looking at the filename instead of the whole text
-            text = self.get_text()
-        except ParseError:
-            return None
-
-        # Iterate through all regex matches in text and try to parse the date
-        for m in re.finditer(DATE_REGEX, text):
-            date_string = m.group(0)
-
-            try:
-                date = __parser(date_string, settings.DATE_ORDER)
-            except (TypeError, ValueError):
-                # Skip all matches that do not parse to a proper date
-                continue
-
-            if date is not None and next_year > date.year > 1900:
-                break
-            else:
-                date = None
-
-        if date is not None:
-            self.log(
-                "info",
-                "Detected document date {} based on string {}".format(
-                    date.isoformat(),
-                    date_string
-                )
-            )
-        else:
-            self.log("info", "Unable to detect date for document")
-
-        return date
+        return self.date
 
     def cleanup(self):
         self.log("debug", "Deleting directory {}".format(self.tempdir))

src/documents/sanity_checker.py

@@ -67,19 +67,34 @@ def check_sanity():
                 f"Original of document {doc.pk} does not exist."))
         else:
             present_files.remove(os.path.normpath(doc.source_path))
-            checksum = None
             try:
                 with doc.source_file as f:
                     checksum = hashlib.md5(f.read()).hexdigest()
             except OSError as e:
                 messages.append(SanityError(
                     f"Cannot read original file of document {doc.pk}: {e}"))
+            else:
+                if not checksum == doc.checksum:
+                    messages.append(SanityError(
+                        f"Checksum mismatch of document {doc.pk}. "
+                        f"Stored: {doc.checksum}, actual: {checksum}."
+                    ))
 
-            if checksum and not checksum == doc.checksum:
+        if os.path.isfile(doc.archive_path):
+            present_files.remove(os.path.normpath(doc.archive_path))
+            try:
+                with doc.archive_file as f:
+                    checksum = hashlib.md5(f.read()).hexdigest()
+            except OSError as e:
                 messages.append(SanityError(
-                    f"Checksum mismatch of document {doc.pk}. "
-                    f"Stored: {doc.checksum}, actual: {checksum}."
+                    f"Cannot read archive file of document {doc.pk}: {e}"
                 ))
+            else:
+                if not checksum == doc.archive_checksum:
+                    messages.append(SanityError(
+                        f"Checksum mismatch of archive {doc.pk}. "
+                        f"Stored: {doc.checksum}, actual: {checksum}."
+                    ))
 
         if not doc.content:
             messages.append(SanityWarning(

src/documents/settings.py

@@ -2,3 +2,4 @@
 # for exporting/importing commands
 EXPORTER_FILE_NAME = "__exported_file_name__"
 EXPORTER_THUMBNAIL_NAME = "__exported_thumbnail_name__"
+EXPORTER_ARCHIVE_NAME = "__exported_archive_name__"

src/documents/signals/handlers.py

@@ -13,7 +13,7 @@ from rest_framework.reverse import reverse
 
 from .. import index, matching
 from ..file_handling import delete_empty_directories, generate_filename, \
-    create_source_path_directory
+    create_source_path_directory, archive_name_from_filename
 from ..models import Document, Tag
 
 
@@ -169,13 +169,46 @@ def run_post_consume_script(sender, document, **kwargs):
 
 @receiver(models.signals.post_delete, sender=Document)
 def cleanup_document_deletion(sender, instance, using, **kwargs):
-    for f in (instance.source_path, instance.thumbnail_path):
-        try:
-            os.unlink(f)
-        except FileNotFoundError:
-            pass  # The file's already gone, so we're cool with it.
+    for f in (instance.source_path,
+              instance.archive_path,
+              instance.thumbnail_path):
+        if os.path.isfile(f):
+            try:
+                os.unlink(f)
+                logging.getLogger(__name__).debug(
+                    f"Deleted file {f}.")
+            except OSError as e:
+                logging.getLogger(__name__).warning(
+                    f"While deleting document {instance.file_name}, the file "
+                    f"{f} could not be deleted: {e}"
+                )
 
-    delete_empty_directories(os.path.dirname(instance.source_path))
+    delete_empty_directories(
+        os.path.dirname(instance.source_path),
+        root=settings.ORIGINALS_DIR
+    )
+
+    delete_empty_directories(
+        os.path.dirname(instance.archive_path),
+        root=settings.ARCHIVE_DIR
+    )
+
+
+def validate_move(instance, old_path, new_path):
+    if not os.path.isfile(old_path):
+        # Can't do anything if the old file does not exist anymore.
+        logging.getLogger(__name__).fatal(
+            f"Document {str(instance)}: File {old_path} has gone.")
+        return False
+
+    if os.path.isfile(new_path):
+        # Can't do anything if the new file already exists. Skip updating file.
+        logging.getLogger(__name__).warning(
+            f"Document {str(instance)}: Cannot rename file "
+            f"since target path {new_path} already exists.")
+        return False
+
+    return True
 
 
 @receiver(models.signals.m2m_changed, sender=Document.tags.through)
@@ -183,55 +216,90 @@ def cleanup_document_deletion(sender, instance, using, **kwargs):
 def update_filename_and_move_files(sender, instance, **kwargs):
 
     if not instance.filename:
-        # Can't update the filename if there is not filename to begin with
-        # This happens after the consumer creates a new document.
-        # The PK needs to be set first by saving the document once. When this
-        # happens, the file is not yet in the ORIGINALS_DIR, and thus can't be
-        # renamed anyway. In all other cases, instance.filename will be set.
+        # Can't update the filename if there is no filename to begin with
+        # This happens when the consumer creates a new document.
+        # The document is modified and saved multiple times, and only after
+        # everything is done (i.e., the generated filename is final),
+        # filename will be set to the location where the consumer has put
+        # the file.
+        #
+        # This will in turn cause this logic to move the file where it belongs.
         return
 
     old_filename = instance.filename
-    old_path = instance.source_path
     new_filename = generate_filename(instance)
 
     if new_filename == instance.filename:
         # Don't do anything if its the same.
         return
 
-    new_path = os.path.join(settings.ORIGINALS_DIR, new_filename)
+    old_source_path = instance.source_path
+    new_source_path = os.path.join(settings.ORIGINALS_DIR, new_filename)
 
-    if not os.path.isfile(old_path):
-        # Can't do anything if the old file does not exist anymore.
-        logging.getLogger(__name__).fatal(
-            f"Document {str(instance)}: File {old_path} has gone.")
+    if not validate_move(instance, old_source_path, new_source_path):
         return
 
-    if os.path.isfile(new_path):
-        # Can't do anything if the new file already exists. Skip updating file.
-        logging.getLogger(__name__).warning(
-            f"Document {str(instance)}: Cannot rename file "
-            f"since target path {new_path} already exists.")
-        return
+    # archive files are optional, archive checksum tells us if we have one,
+    # since this is None for documents without archived files.
+    if instance.archive_checksum:
+        new_archive_filename = archive_name_from_filename(new_filename)
+        old_archive_path = instance.archive_path
+        new_archive_path = os.path.join(settings.ARCHIVE_DIR,
+                                        new_archive_filename)
 
-    create_source_path_directory(new_path)
+        if not validate_move(instance, old_archive_path, new_archive_path):
+            return
+
+        create_source_path_directory(new_archive_path)
+    else:
+        old_archive_path = None
+        new_archive_path = None
+
+    create_source_path_directory(new_source_path)
 
     try:
-        os.rename(old_path, new_path)
+        os.rename(old_source_path, new_source_path)
+        if instance.archive_checksum:
+            os.rename(old_archive_path, new_archive_path)
         instance.filename = new_filename
         # Don't save here to prevent infinite recursion.
         Document.objects.filter(pk=instance.pk).update(filename=new_filename)
 
         logging.getLogger(__name__).debug(
-            f"Moved file {old_path} to {new_path}.")
+            f"Moved file {old_source_path} to {new_source_path}.")
+
+        logging.getLogger(__name__).debug(
+            f"Moved file {old_archive_path} to {new_archive_path}.")
 
     except OSError as e:
         instance.filename = old_filename
+        # this happens when we can't move a file. If that's the case for the
+        # archive file, we try our best to revert the changes.
+        try:
+            os.rename(new_source_path, old_source_path)
+            os.rename(new_archive_path, old_archive_path)
+        except:
+            # This is fine, since:
+            # A: if we managed to move source from A to B, we will also manage
+            #  to move it from B to A. If not, we have a serious issue
+            #  that's going to get caught by the santiy checker.
+            #  all files remain in place and will never be overwritten,
+            #  so this is not the end of the world.
+            # B: if moving the orignal file failed, nothing has changed anyway.
+            pass
     except DatabaseError as e:
-        os.rename(new_path, old_path)
+        os.rename(new_source_path, old_source_path)
+        if instance.archive_checksum:
+            os.rename(new_archive_path, old_archive_path)
         instance.filename = old_filename
 
-    if not os.path.isfile(old_path):
-        delete_empty_directories(os.path.dirname(old_path))
+    if not os.path.isfile(old_source_path):
+        delete_empty_directories(os.path.dirname(old_source_path),
+                                 root=settings.ORIGINALS_DIR)
+
+    if old_archive_path and not os.path.isfile(old_archive_path):
+        delete_empty_directories(os.path.dirname(old_archive_path),
+                                 root=settings.ARCHIVE_DIR)
 
 
 def set_log_entry(sender, document=None, logging_group=None, **kwargs):

src/documents/tasks.py

@@ -12,7 +12,9 @@ from documents.sanity_checker import SanityFailedError
 
 
 def index_optimize():
-    index.open_index().optimize()
+    ix = index.open_index()
+    writer = AsyncWriter(ix)
+    writer.commit(optimize=True)
 
 
 def index_reindex():

src/documents/tests/test_api.py

@@ -100,6 +100,44 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, content_thumbnail)
 
+    def test_download_with_archive(self):
+
+        _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir)
+
+        content = b"This is a test"
+        content_archive = b"This is the same test but archived"
+
+        with open(filename, "wb") as f:
+            f.write(content)
+
+        filename = os.path.basename(filename)
+
+        doc = Document.objects.create(title="none", filename=filename,
+                                      mime_type="application/pdf")
+
+        with open(doc.archive_path, "wb") as f:
+            f.write(content_archive)
+
+        response = self.client.get('/api/documents/{}/download/'.format(doc.pk))
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.content, content_archive)
+
+        response = self.client.get('/api/documents/{}/download/?original=true'.format(doc.pk))
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.content, content)
+
+        response = self.client.get('/api/documents/{}/preview/'.format(doc.pk))
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.content, content_archive)
+
+        response = self.client.get('/api/documents/{}/preview/?original=true'.format(doc.pk))
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.content, content)
+
     def test_document_actions_not_existing_file(self):
 
         doc = Document.objects.create(title="none", filename=os.path.basename("asd"), mime_type="application/pdf")

src/documents/tests/test_consumer.py

@@ -1,5 +1,6 @@
 import os
 import re
+import shutil
 import tempfile
 from unittest import mock
 from unittest.mock import MagicMock
@@ -364,35 +365,36 @@ class TestFieldPermutations(TestCase):
 
 class DummyParser(DocumentParser):
 
-    def get_thumbnail(self):
+    def get_thumbnail(self, document_path, mime_type):
         # not important during tests
         raise NotImplementedError()
 
-    def __init__(self, path, logging_group, scratch_dir):
-        super(DummyParser, self).__init__(path, logging_group)
+    def __init__(self, logging_group, scratch_dir, archive_path):
+        super(DummyParser, self).__init__(logging_group)
         _, self.fake_thumb = tempfile.mkstemp(suffix=".png", dir=scratch_dir)
+        self.archive_path = archive_path
 
-    def get_optimised_thumbnail(self):
+    def get_optimised_thumbnail(self, document_path, mime_type):
         return self.fake_thumb
 
-    def get_text(self):
-        return "The Text"
+    def parse(self, document_path, mime_type):
+        self.text = "The Text"
 
 
 class FaultyParser(DocumentParser):
 
-    def get_thumbnail(self):
+    def get_thumbnail(self, document_path, mime_type):
         # not important during tests
         raise NotImplementedError()
 
-    def __init__(self, path, logging_group, scratch_dir):
-        super(FaultyParser, self).__init__(path, logging_group)
+    def __init__(self, logging_group, scratch_dir):
+        super(FaultyParser, self).__init__(logging_group)
         _, self.fake_thumb = tempfile.mkstemp(suffix=".png", dir=scratch_dir)
 
-    def get_optimised_thumbnail(self):
+    def get_optimised_thumbnail(self, document_path, mime_type):
         return self.fake_thumb
 
-    def get_text(self):
+    def parse(self, document_path, mime_type):
         raise ParseError("Does not compute.")
 
 
@@ -410,11 +412,11 @@ def fake_magic_from_file(file, mime=False):
 @mock.patch("documents.consumer.magic.from_file", fake_magic_from_file)
 class TestConsumer(DirectoriesMixin, TestCase):
 
-    def make_dummy_parser(self, path, logging_group):
-        return DummyParser(path, logging_group, self.dirs.scratch_dir)
+    def make_dummy_parser(self, logging_group):
+        return DummyParser(logging_group, self.dirs.scratch_dir, self.get_test_archive_file())
 
-    def make_faulty_parser(self, path, logging_group):
-        return FaultyParser(path, logging_group, self.dirs.scratch_dir)
+    def make_faulty_parser(self, logging_group):
+        return FaultyParser(logging_group, self.dirs.scratch_dir)
 
     def setUp(self):
         super(TestConsumer, self).setUp()
@@ -432,8 +434,16 @@ class TestConsumer(DirectoriesMixin, TestCase):
         self.consumer = Consumer()
 
     def get_test_file(self):
-        fd, f = tempfile.mkstemp(suffix=".pdf", dir=self.dirs.scratch_dir)
-        return f
+        src = os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000001.pdf")
+        dst = os.path.join(self.dirs.scratch_dir, "sample.pdf")
+        shutil.copy(src, dst)
+        return dst
+
+    def get_test_archive_file(self):
+        src = os.path.join(os.path.dirname(__file__), "samples", "documents", "archive", "0000001.pdf")
+        dst = os.path.join(self.dirs.scratch_dir, "sample_archive.pdf")
+        shutil.copy(src, dst)
+        return dst
 
     @override_settings(PAPERLESS_FILENAME_FORMAT=None)
     def testNormalOperation(self):
@@ -455,6 +465,13 @@ class TestConsumer(DirectoriesMixin, TestCase):
             document.thumbnail_path
         ))
 
+        self.assertTrue(os.path.isfile(
+            document.archive_path
+        ))
+
+        self.assertEqual(document.checksum, "42995833e01aea9b3edee44bbfdd7ce1")
+        self.assertEqual(document.archive_checksum, "62acb0bcbfbcaa62ca6ad3668e4e404b")
+
         self.assertFalse(os.path.isfile(filename))
 
     def testOverrideFilename(self):
@@ -502,7 +519,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         self.fail("Should throw exception")
 
-    def testDuplicates(self):
+    def testDuplicates1(self):
         self.consumer.try_consume_file(self.get_test_file())
 
         try:
@@ -513,6 +530,21 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         self.fail("Should throw exception")
 
+    def testDuplicates2(self):
+        self.consumer.try_consume_file(self.get_test_file())
+
+        try:
+            self.consumer.try_consume_file(self.get_test_archive_file())
+        except ConsumerError as e:
+            self.assertTrue(str(e).endswith("It is a duplicate."))
+            return
+
+        self.fail("Should throw exception")
+
+    def testDuplicates3(self):
+        self.consumer.try_consume_file(self.get_test_archive_file())
+        self.consumer.try_consume_file(self.get_test_file())
+
     @mock.patch("documents.parsers.document_consumer_declaration.send")
     def testNoParsers(self, m):
         m.return_value = []

src/documents/tests/test_date_parsing.py

@@ -0,0 +1,140 @@
+import datetime
+import os
+import shutil
+from unittest import mock
+from uuid import uuid4
+
+from dateutil import tz
+from django.conf import settings
+from django.test import TestCase, override_settings
+
+from documents.parsers import parse_date
+from paperless_tesseract.parsers import RasterisedDocumentParser
+
+
+class TestDate(TestCase):
+
+    SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "../../paperless_tesseract/tests/samples")
+    SCRATCH = "/tmp/paperless-tests-{}".format(str(uuid4())[:8])
+
+    def setUp(self):
+        os.makedirs(self.SCRATCH, exist_ok=True)
+
+    def tearDown(self):
+        shutil.rmtree(self.SCRATCH)
+
+    def test_date_format_1(self):
+        text = "lorem ipsum 130218 lorem ipsum"
+        self.assertEqual(parse_date("", text), None)
+
+    def test_date_format_2(self):
+        text = "lorem ipsum 2018 lorem ipsum"
+        self.assertEqual(parse_date("", text), None)
+
+    def test_date_format_3(self):
+        text = "lorem ipsum 20180213 lorem ipsum"
+        self.assertEqual(parse_date("", text), None)
+
+    def test_date_format_4(self):
+        text = "lorem ipsum 13.02.2018 lorem ipsum"
+        date = parse_date("", text)
+        self.assertEqual(
+            date,
+            datetime.datetime(
+                2018, 2, 13, 0, 0,
+                tzinfo=tz.gettz(settings.TIME_ZONE)
+            )
+        )
+
+    def test_date_format_5(self):
+        text = (
+            "lorem ipsum 130218, 2018, 20180213 and lorem 13.02.2018 lorem "
+            "ipsum"
+        )
+        date = parse_date("", text)
+        self.assertEqual(
+            date,
+            datetime.datetime(
+                2018, 2, 13, 0, 0,
+                tzinfo=tz.gettz(settings.TIME_ZONE)
+            )
+        )
+
+    def test_date_format_6(self):
+        text = (
+            "lorem ipsum\n"
+            "Wohnort\n"
+            "3100\n"
+            "IBAN\n"
+            "AT87 4534\n"
+            "1234\n"
+            "1234 5678\n"
+            "BIC\n"
+            "lorem ipsum"
+        )
+        self.assertEqual(parse_date("", text), None)
+
+    def test_date_format_7(self):
+        text = (
+            "lorem ipsum\n"
+            "März 2019\n"
+            "lorem ipsum"
+        )
+        date = parse_date("", text)
+        self.assertEqual(
+            date,
+            datetime.datetime(
+                2019, 3, 1, 0, 0,
+                tzinfo=tz.gettz(settings.TIME_ZONE)
+            )
+        )
+
+    def test_date_format_8(self):
+        text = (
+            "lorem ipsum\n"
+            "Wohnort\n"
+            "3100\n"
+            "IBAN\n"
+            "AT87 4534\n"
+            "1234\n"
+            "1234 5678\n"
+            "BIC\n"
+            "lorem ipsum\n"
+            "März 2020"
+        )
+        self.assertEqual(
+            parse_date("", text),
+            datetime.datetime(
+                2020, 3, 1, 0, 0,
+                tzinfo=tz.gettz(settings.TIME_ZONE)
+            )
+        )
+
+    @override_settings(SCRATCH_DIR=SCRATCH)
+    def test_date_format_9(self):
+        text = (
+            "lorem ipsum\n"
+            "27. Nullmonth 2020\n"
+            "März 2020\n"
+            "lorem ipsum"
+        )
+        self.assertEqual(
+            parse_date("", text),
+            datetime.datetime(
+                2020, 3, 1, 0, 0,
+                tzinfo=tz.gettz(settings.TIME_ZONE)
+            )
+        )
+
+    def test_crazy_date_past(self, *args):
+        self.assertIsNone(parse_date("", "01-07-0590 00:00:00"))
+
+    def test_crazy_date_future(self, *args):
+        self.assertIsNone(parse_date("", "01-07-2350 00:00:00"))
+
+    def test_crazy_date_with_spaces(self, *args):
+        self.assertIsNone(parse_date("", "20 408000l 2475"))
+
+    @override_settings(FILENAME_DATE_ORDER="YMD")
+    def test_filename_date_parse_invalid(self, *args):
+        self.assertIsNone(parse_date("/tmp/20 408000l 2475 - test.pdf", "No date in here"))

src/documents/tests/test_document_model.py

@@ -1,12 +1,29 @@
+import os
+import shutil
+import tempfile
+from pathlib import Path
 from unittest import mock
 
-from django.test import TestCase
+from django.test import TestCase, override_settings
 
 from ..models import Document, Correspondent
 
 
 class TestDocument(TestCase):
 
+    def setUp(self) -> None:
+        self.originals_dir = tempfile.mkdtemp()
+        self.thumb_dir = tempfile.mkdtemp()
+
+        override_settings(
+            ORIGINALS_DIR=self.originals_dir,
+            THUMBNAIL_DIR=self.thumb_dir,
+        ).enable()
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.originals_dir)
+        shutil.rmtree(self.thumb_dir)
+
     def test_file_deletion(self):
         document = Document.objects.create(
             correspondent=Correspondent.objects.create(name="Test0"),
@@ -19,6 +36,9 @@ class TestDocument(TestCase):
         file_path = document.source_path
         thumb_path = document.thumbnail_path
 
+        Path(file_path).touch()
+        Path(thumb_path).touch()
+
         with mock.patch("documents.signals.handlers.os.unlink") as mock_unlink:
             document.delete()
             mock_unlink.assert_any_call(file_path)

src/documents/tests/test_file_handling.py

@@ -2,32 +2,17 @@ import os
 import shutil
 from pathlib import Path
 from unittest import mock
-from uuid import uuid4
 
 from django.conf import settings
 from django.db import DatabaseError
 from django.test import TestCase, override_settings
 
+from .utils import DirectoriesMixin
 from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories
 from ..models import Document, Correspondent
 
 
-class TestDate(TestCase):
-    deletion_list = []
-
-    def add_to_deletion_list(self, dirname):
-        self.deletion_list.append(dirname)
-
-    def setUp(self):
-        folder = "/tmp/paperless-tests-{}".format(str(uuid4())[:8])
-        os.makedirs(folder + "/documents/originals")
-        override_settings(MEDIA_ROOT=folder).enable()
-        override_settings(ORIGINALS_DIR=folder + "/documents/originals").enable()
-        self.add_to_deletion_list(folder)
-
-    def tearDown(self):
-        for dirname in self.deletion_list:
-            shutil.rmtree(dirname, ignore_errors=True)
+class TestFileHandling(DirectoriesMixin, TestCase):
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="")
     def test_generate_source_filename(self):
@@ -104,7 +89,7 @@ class TestDate(TestCase):
         document.save()
 
         # Check proper handling of files
-        self.assertEqual(os.path.isfile(settings.MEDIA_ROOT + "/documents/originals/none/none-{:07d}.pdf".format(document.pk)), True)
+        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none-{:07d}.pdf".format(document.pk)), True)
         self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk))
 
         os.chmod(settings.ORIGINALS_DIR + "/none", 0o777)
@@ -140,7 +125,7 @@ class TestDate(TestCase):
 
             # Check proper handling of files
             self.assertTrue(os.path.isfile(document.source_path))
-            self.assertEqual(os.path.isfile(settings.MEDIA_ROOT + "/documents/originals/none/none-{:07d}.pdf".format(document.pk)), True)
+            self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none-{:07d}.pdf".format(document.pk)), True)
             self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk))
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@@ -196,8 +181,8 @@ class TestDate(TestCase):
         document.save()
 
         # Check proper handling of files
-        self.assertEqual(os.path.isdir(settings.MEDIA_ROOT + "/documents/originals/test"), True)
-        self.assertEqual(os.path.isdir(settings.MEDIA_ROOT + "/documents/originals/none"), True)
+        self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/test"), True)
+        self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), True)
         self.assertTrue(os.path.isfile(important_file))
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
@@ -315,13 +300,12 @@ class TestDate(TestCase):
         # Create our working directory
         tmp = os.path.join(settings.ORIGINALS_DIR, "test_delete_empty")
         os.makedirs(tmp)
-        self.add_to_deletion_list(tmp)
 
         os.makedirs(os.path.join(tmp, "notempty"))
         Path(os.path.join(tmp, "notempty", "file")).touch()
         os.makedirs(os.path.join(tmp, "notempty", "empty"))
 
-        delete_empty_directories(os.path.join(tmp, "notempty", "empty"))
+        delete_empty_directories(os.path.join(tmp, "notempty", "empty"), root=settings.ORIGINALS_DIR)
         self.assertEqual(os.path.isdir(os.path.join(tmp, "notempty")), True)
         self.assertEqual(os.path.isfile(
             os.path.join(tmp, "notempty", "file")), True)
@@ -345,3 +329,159 @@ class TestDate(TestCase):
         document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
 
         self.assertEqual(generate_filename(document), "0000001.pdf")
+
+
+class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT=None)
+    def test_create_no_format(self):
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        Path(archive).touch()
+        doc = Document.objects.create(mime_type="application/pdf", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertTrue(os.path.isfile(original))
+        self.assertTrue(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    def test_create_with_format(self):
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        Path(archive).touch()
+        doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertFalse(os.path.isfile(original))
+        self.assertFalse(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))
+        self.assertEqual(doc.source_path, os.path.join(settings.ORIGINALS_DIR, "none", "my_doc-0000001.pdf"))
+        self.assertEqual(doc.archive_path, os.path.join(settings.ARCHIVE_DIR, "none", "my_doc-0000001.pdf"))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    def test_move_archive_gone(self):
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        #Path(archive).touch()
+        doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertTrue(os.path.isfile(original))
+        self.assertFalse(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertFalse(os.path.isfile(doc.archive_path))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    def test_move_archive_exists(self):
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        Path(archive).touch()
+        os.makedirs(os.path.join(settings.ARCHIVE_DIR, "none"))
+        Path(os.path.join(settings.ARCHIVE_DIR, "none", "my_doc-0000001.pdf")).touch()
+        doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertTrue(os.path.isfile(original))
+        self.assertTrue(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    @mock.patch("documents.signals.handlers.os.rename")
+    def test_move_archive_error(self, m):
+
+        def fake_rename(src, dst):
+            if "archive" in src:
+                raise OSError()
+            else:
+                os.remove(src)
+                Path(dst).touch()
+
+        m.side_effect = fake_rename
+
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        Path(archive).touch()
+        doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertTrue(os.path.isfile(original))
+        self.assertTrue(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    def test_move_file_gone(self):
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        #Path(original).touch()
+        Path(archive).touch()
+        doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertFalse(os.path.isfile(original))
+        self.assertTrue(os.path.isfile(archive))
+        self.assertFalse(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    @mock.patch("documents.signals.handlers.os.rename")
+    def test_move_file_error(self, m):
+
+        def fake_rename(src, dst):
+            if "original" in src:
+                raise OSError()
+            else:
+                os.remove(src)
+                Path(dst).touch()
+
+        m.side_effect = fake_rename
+
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        Path(archive).touch()
+        doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertTrue(os.path.isfile(original))
+        self.assertTrue(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))
+
+    def test_archive_deleted(self):
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        Path(archive).touch()
+        doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+
+        self.assertTrue(os.path.isfile(original))
+        self.assertTrue(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))
+
+        doc.delete()
+
+        self.assertFalse(os.path.isfile(original))
+        self.assertFalse(os.path.isfile(archive))
+        self.assertFalse(os.path.isfile(doc.source_path))
+        self.assertFalse(os.path.isfile(doc.archive_path))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
+    def test_database_error(self):
+
+        original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
+        archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
+        Path(original).touch()
+        Path(archive).touch()
+        doc = Document(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
+        with mock.patch("documents.signals.handlers.Document.objects.filter") as m:
+            m.side_effect = DatabaseError()
+            doc.save()
+
+        self.assertTrue(os.path.isfile(original))
+        self.assertTrue(os.path.isfile(archive))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc.archive_path))

src/documents/tests/test_management_archiver.py

@@ -0,0 +1,42 @@
+import filecmp
+import os
+import shutil
+
+from django.core.management import call_command
+from django.test import TestCase
+
+from documents.management.commands.document_archiver import handle_document
+from documents.models import Document
+from documents.tests.utils import DirectoriesMixin
+
+
+sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
+
+
+class TestArchiver(DirectoriesMixin, TestCase):
+
+    def make_models(self):
+        self.d1 = Document.objects.create(checksum="A", title="A", content="first document", pk=1, mime_type="application/pdf")
+        #self.d2 = Document.objects.create(checksum="B", title="B", content="second document")
+        #self.d3 = Document.objects.create(checksum="C", title="C", content="unrelated document")
+
+    def test_archiver(self):
+
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
+        self.make_models()
+
+        call_command('document_archiver')
+
+    def test_handle_document(self):
+
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
+        self.make_models()
+
+        handle_document(self.d1)
+
+        doc = Document.objects.get(id=self.d1.id)
+
+        self.assertIsNotNone(doc.checksum)
+        self.assertTrue(os.path.isfile(doc.archive_path))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(filecmp.cmp(sample_file, doc.source_path))

src/documents/tests/test_management_exporter.py

@@ -23,10 +23,7 @@ class TestExporter(DirectoriesMixin, TestCase):
 
         file = os.path.join(self.dirs.originals_dir, "0000001.pdf")
 
-        with open(file, "rb") as f:
-            checksum = hashlib.md5(f.read()).hexdigest()
-
-        Document.objects.create(checksum=checksum, title="wow", filename="0000001.pdf", id=1, mime_type="application/pdf")
+        Document.objects.create(checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", id=1, mime_type="application/pdf")
         Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
         Tag.objects.create(name="t")
         DocumentType.objects.create(name="dt")
@@ -51,6 +48,14 @@ class TestExporter(DirectoriesMixin, TestCase):
                     checksum = hashlib.md5(f.read()).hexdigest()
                 self.assertEqual(checksum, element['fields']['checksum'])
 
+                if document_exporter.EXPORTER_ARCHIVE_NAME in element:
+                    fname = os.path.join(target, element[document_exporter.EXPORTER_ARCHIVE_NAME])
+                    self.assertTrue(os.path.exists(fname))
+
+                    with open(fname, "rb") as f:
+                        checksum = hashlib.md5(f.read()).hexdigest()
+                    self.assertEqual(checksum, element['fields']['archive_checksum'])
+
         Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", id=3, mime_type="application/pdf")
 
         self.assertRaises(FileNotFoundError, call_command, 'document_exporter', target)

src/documents/tests/test_parsers.py

@@ -1,11 +1,13 @@
 import os
+import shutil
+import tempfile
 from tempfile import TemporaryDirectory
 from unittest import mock
 
-from django.test import TestCase
+from django.test import TestCase, override_settings
 
 from documents.parsers import get_parser_class, get_supported_file_extensions, get_default_file_extension, \
-    get_parser_class_for_mime_type
+    get_parser_class_for_mime_type, DocumentParser
 from paperless_tesseract.parsers import RasterisedDocumentParser
 from paperless_text.parsers import TextDocumentParser
 
@@ -66,6 +68,38 @@ class TestParserDiscovery(TestCase):
             )
 
 
+def fake_get_thumbnail(self, path, mimetype):
+    return os.path.join(os.path.dirname(__file__), "examples", "no-text.png")
+
+
+class TestBaseParser(TestCase):
+
+    def setUp(self) -> None:
+
+        self.scratch = tempfile.mkdtemp()
+        override_settings(
+            SCRATCH_DIR=self.scratch
+        ).enable()
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.scratch)
+
+    @mock.patch("documents.parsers.DocumentParser.get_thumbnail", fake_get_thumbnail)
+    @override_settings(OPTIMIZE_THUMBNAILS=True)
+    def test_get_optimised_thumbnail(self):
+        parser = DocumentParser(None)
+
+        parser.get_optimised_thumbnail("any", "not important")
+
+    @mock.patch("documents.parsers.DocumentParser.get_thumbnail", fake_get_thumbnail)
+    @override_settings(OPTIMIZE_THUMBNAILS=False)
+    def test_get_optimised_thumb_disabled(self):
+        parser = DocumentParser(None)
+
+        path = parser.get_optimised_thumbnail("any", "not important")
+        self.assertEqual(path, fake_get_thumbnail(None, None, None))
+
+
 class TestParserAvailability(TestCase):
 
     def test_file_extensions(self):

src/documents/tests/utils.py

@@ -17,10 +17,12 @@ def setup_directories():
     dirs.index_dir = os.path.join(dirs.data_dir, "index")
     dirs.originals_dir = os.path.join(dirs.media_dir, "documents", "originals")
     dirs.thumbnail_dir = os.path.join(dirs.media_dir, "documents", "thumbnails")
+    dirs.archive_dir = os.path.join(dirs.media_dir, "documents", "archive")
 
     os.makedirs(dirs.index_dir, exist_ok=True)
     os.makedirs(dirs.originals_dir, exist_ok=True)
     os.makedirs(dirs.thumbnail_dir, exist_ok=True)
+    os.makedirs(dirs.archive_dir, exist_ok=True)
 
     override_settings(
         DATA_DIR=dirs.data_dir,
@@ -28,6 +30,7 @@ def setup_directories():
         MEDIA_ROOT=dirs.media_dir,
         ORIGINALS_DIR=dirs.originals_dir,
         THUMBNAIL_DIR=dirs.thumbnail_dir,
+        ARCHIVE_DIR=dirs.archive_dir,
         CONSUMPTION_DIR=dirs.consumption_dir,
         INDEX_DIR=dirs.index_dir,
         MODEL_FILE=os.path.join(dirs.data_dir, "classification_model.pickle")

src/documents/views.py

@@ -1,3 +1,5 @@
+import os
+
 from django.db.models import Count, Max
 from django.http import HttpResponse, HttpResponseBadRequest, Http404
 from django.views.decorators.cache import cache_control
@@ -126,17 +128,30 @@ class DocumentViewSet(RetrieveModelMixin,
         index.remove_document_from_index(self.get_object())
         return super(DocumentViewSet, self).destroy(request, *args, **kwargs)
 
-    def file_response(self, pk, disposition):
+    @staticmethod
+    def original_requested(request):
+        return (
+            'original' in request.query_params and
+            request.query_params['original'] == 'true'
+        )
+
+    def file_response(self, pk, request, disposition):
         doc = Document.objects.get(id=pk)
-
-        if doc.storage_type == Document.STORAGE_TYPE_UNENCRYPTED:
-            file_handle = doc.source_file
+        if not self.original_requested(request) and os.path.isfile(doc.archive_path):  # NOQA: E501
+            file_handle = doc.archive_file
+            filename = doc.archive_file_name
+            mime_type = 'application/pdf'
         else:
-            file_handle = GnuPG.decrypted(doc.source_file)
+            file_handle = doc.source_file
+            filename = doc.file_name
+            mime_type = doc.mime_type
 
-        response = HttpResponse(file_handle, content_type=doc.mime_type)
+        if doc.storage_type == Document.STORAGE_TYPE_GPG:
+            file_handle = GnuPG.decrypted(file_handle)
+
+        response = HttpResponse(file_handle, content_type=mime_type)
         response["Content-Disposition"] = '{}; filename="{}"'.format(
-            disposition, doc.file_name)
+            disposition, filename)
         return response
 
     @action(methods=['post'], detail=False)
@@ -157,6 +172,8 @@ class DocumentViewSet(RetrieveModelMixin,
                 "paperless__checksum": doc.checksum,
                 "paperless__mime_type": doc.mime_type,
                 "paperless__filename": doc.filename,
+                "paperless__has_archive_version":
+                    os.path.isfile(doc.archive_path)
             })
         except Document.DoesNotExist:
             raise Http404()
@@ -164,7 +181,8 @@ class DocumentViewSet(RetrieveModelMixin,
     @action(methods=['get'], detail=True)
     def preview(self, request, pk=None):
         try:
-            response = self.file_response(pk, "inline")
+            response = self.file_response(
+                pk, request, "inline")
             return response
         except (FileNotFoundError, Document.DoesNotExist):
             raise Http404()
@@ -181,7 +199,8 @@ class DocumentViewSet(RetrieveModelMixin,
     @action(methods=['get'], detail=True)
     def download(self, request, pk=None):
         try:
-            return self.file_response(pk, "attachment")
+            return self.file_response(
+                pk, request, "attachment")
         except (FileNotFoundError, Document.DoesNotExist):
             raise Http404()
 

src/paperless/checks.py

@@ -57,7 +57,6 @@ def binaries_check(app_configs, **kwargs):
     binaries = (
         settings.CONVERT_BINARY,
         settings.OPTIPNG_BINARY,
-        settings.UNPAPER_BINARY,
         "tesseract"
     )
 

src/paperless/settings.py

@@ -49,6 +49,7 @@ STATIC_ROOT = os.getenv("PAPERLESS_STATICDIR", os.path.join(BASE_DIR, "..", "sta
 
 MEDIA_ROOT = os.getenv('PAPERLESS_MEDIA_ROOT', os.path.join(BASE_DIR, "..", "media"))
 ORIGINALS_DIR = os.path.join(MEDIA_ROOT, "documents", "originals")
+ARCHIVE_DIR = os.path.join(MEDIA_ROOT, "documents", "archive")
 THUMBNAIL_DIR = os.path.join(MEDIA_ROOT, "documents", "thumbnails")
 
 DATA_DIR = os.getenv('PAPERLESS_DATA_DIR', os.path.join(BASE_DIR, "..", "data"))
@@ -348,9 +349,17 @@ OCR_PAGES = int(os.getenv('PAPERLESS_OCR_PAGES', 0))
 # documents.  It should be a 3-letter language code consistent with ISO 639.
 OCR_LANGUAGE = os.getenv("PAPERLESS_OCR_LANGUAGE", "eng")
 
+# OCRmyPDF --output-type options are available.
+# TODO: validate this setting.
+OCR_OUTPUT_TYPE = os.getenv("PAPERLESS_OCR_OUTPUT_TYPE", "pdfa")
 
-# OCR all documents?
-OCR_ALWAYS = __get_boolean("PAPERLESS_OCR_ALWAYS", "false")
+# skip. redo, force
+# TODO: validate this.
+OCR_MODE = os.getenv("PAPERLESS_OCR_MODE", "skip")
+
+OCR_IMAGE_DPI = os.getenv("PAPERLESS_OCR_IMAGE_DPI")
+
+OCR_USER_ARGS = os.getenv("PAPERLESS_OCR_USER_ARGS", "{}")
 
 # GNUPG needs a home directory for some reason
 GNUPG_HOME = os.getenv("HOME", "/tmp")
@@ -359,11 +368,10 @@ GNUPG_HOME = os.getenv("HOME", "/tmp")
 CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert")
 CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
 CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
-CONVERT_DENSITY = int(os.getenv("PAPERLESS_CONVERT_DENSITY", 300))
 
 GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
+
 OPTIPNG_BINARY = os.getenv("PAPERLESS_OPTIPNG_BINARY", "optipng")
-UNPAPER_BINARY = os.getenv("PAPERLESS_UNPAPER_BINARY", "unpaper")
 
 
 # Pre-2.x versions of Paperless stored your documents locally with GPG

src/paperless_tesseract/checks.py

@@ -14,12 +14,21 @@ def get_tesseract_langs():
 
 @register()
 def check_default_language_available(app_configs, **kwargs):
-    langs = get_tesseract_langs()
+    installed_langs = get_tesseract_langs()
 
-    if settings.OCR_LANGUAGE not in langs:
-        return [Error(
-            f"The default ocr language {settings.OCR_LANGUAGE} is "
-            f"not installed. Paperless cannot OCR your documents "
-            f"without it. Please fix PAPERLESS_OCR_LANGUAGE.")]
-    else:
-        return []
+    if not settings.OCR_LANGUAGE:
+        return [Warning(
+            "No OCR language has been specified with PAPERLESS_OCR_LANGUAGE. "
+            "This means that tesseract will fallback to english."
+        )]
+
+    specified_langs = settings.OCR_LANGUAGE.split("+")
+
+    for lang in specified_langs:
+        if lang not in installed_langs:
+            return [Error(
+                f"The selected ocr language {lang} is "
+                f"not installed. Paperless cannot OCR your documents "
+                f"without it. Please fix PAPERLESS_OCR_LANGUAGE.")]
+
+    return []

src/paperless_tesseract/parsers.py

@@ -1,23 +1,15 @@
-import itertools
+import json
 import os
 import re
 import subprocess
-from multiprocessing.pool import ThreadPool
 
-import langdetect
+import ocrmypdf
 import pdftotext
-import pyocr
 from PIL import Image
 from django.conf import settings
-from pyocr import PyocrException
+from ocrmypdf import InputFileError
 
-from documents.parsers import DocumentParser, ParseError, run_unpaper, \
-    run_convert
-from .languages import ISO639
-
-
-class OCRError(Exception):
-    pass
+from documents.parsers import DocumentParser, ParseError, run_convert
 
 
 class RasterisedDocumentParser(DocumentParser):
@@ -26,11 +18,7 @@ class RasterisedDocumentParser(DocumentParser):
     image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.)
     """
 
-    def __init__(self, path, logging_group):
-        super().__init__(path, logging_group)
-        self._text = None
-
-    def get_thumbnail(self):
+    def get_thumbnail(self, document_path, mime_type):
         """
         The thumbnail of a PDF is just a 500px wide image of the first page.
         """
@@ -44,7 +32,7 @@ class RasterisedDocumentParser(DocumentParser):
                         alpha="remove",
                         strip=True,
                         trim=True,
-                        input_file="{}[0]".format(self.document_path),
+                        input_file="{}[0]".format(document_path),
                         output_file=out_path,
                         logging_group=self.logging_group)
         except ParseError:
@@ -59,7 +47,7 @@ class RasterisedDocumentParser(DocumentParser):
                    "-q",
                    "-sDEVICE=pngalpha",
                    "-o", gs_out_path,
-                   self.document_path]
+                   document_path]
             if not subprocess.Popen(cmd).wait() == 0:
                 raise ParseError("Thumbnail (gs) failed at {}".format(cmd))
             # then run convert on the output from gs
@@ -74,169 +62,126 @@ class RasterisedDocumentParser(DocumentParser):
 
         return out_path
 
-    def _is_ocred(self):
-
-        # Extract text from PDF using pdftotext
-        text = get_text_from_pdf(self.document_path)
-
-        # We assume, that a PDF with at least 50 characters contains text
-        # (so no OCR required)
-        return len(text) > 50
-
-    def get_text(self):
-
-        if self._text is not None:
-            return self._text
-
-        if not settings.OCR_ALWAYS and self._is_ocred():
-            self.log("debug", "Skipping OCR, using Text from PDF")
-            self._text = get_text_from_pdf(self.document_path)
-            return self._text
-
-        images = self._get_greyscale()
-
-        if not images:
-            raise ParseError("Empty document, nothing to do.")
+    def is_image(self, mime_type):
+        return mime_type in [
+            "image/png",
+            "image/jpeg"
+        ]
 
+    def get_dpi(self, image):
         try:
-
-            sample_page_index = int(len(images) / 2)
-            self.log(
-                "debug",
-                f"Attempting language detection on page "
-                f"{sample_page_index + 1} of {len(images)}...")
-
-            sample_page_text = self._ocr([images[sample_page_index]],
-                                         settings.OCR_LANGUAGE)[0]
-            guessed_language = self._guess_language(sample_page_text)
-
-            if not guessed_language or guessed_language not in ISO639:
-                self.log("warning", "Language detection failed.")
-                ocr_pages = self._complete_ocr_default_language(
-                    images, sample_page_index, sample_page_text)
-
-            elif ISO639[guessed_language] == settings.OCR_LANGUAGE:
-                self.log(
-                    "debug",
-                    f"Detected language: {guessed_language} "
-                    f"(default language)")
-                ocr_pages = self._complete_ocr_default_language(
-                    images, sample_page_index, sample_page_text)
-
-            elif not ISO639[guessed_language] in pyocr.get_available_tools()[0].get_available_languages():  # NOQA: E501
-                self.log(
-                    "warning",
-                    f"Detected language {guessed_language} is not available "
-                    f"on this system.")
-                ocr_pages = self._complete_ocr_default_language(
-                    images, sample_page_index, sample_page_text)
-
-            else:
-                self.log("debug", f"Detected language: {guessed_language}")
-                ocr_pages = self._ocr(images, ISO639[guessed_language])
-
-            self.log("debug", "OCR completed.")
-            self._text = strip_excess_whitespace(" ".join(ocr_pages))
-            return self._text
-
-        except OCRError as e:
-            raise ParseError(e)
-
-    def _get_greyscale(self):
-        """
-        Greyscale images are easier for Tesseract to OCR
-        """
-
-        # Convert PDF to multiple PNMs
-        input_file = self.document_path
-
-        if settings.OCR_PAGES == 1:
-            input_file += "[0]"
-        elif settings.OCR_PAGES > 1:
-            input_file += f"[0-{settings.OCR_PAGES - 1}]"
-
-        self.log(
-            "debug",
-            f"Converting document {input_file} into greyscale images")
-
-        output_files = os.path.join(self.tempdir, "convert-%04d.pnm")
-
-        run_convert(density=settings.CONVERT_DENSITY,
-                    depth="8",
-                    type="grayscale",
-                    input_file=input_file,
-                    output_file=output_files,
-                    logging_group=self.logging_group)
-
-        # Get a list of converted images
-        pnms = []
-        for f in os.listdir(self.tempdir):
-            if f.endswith(".pnm"):
-                pnms.append(os.path.join(self.tempdir, f))
-
-        self.log("debug", f"Running unpaper on {len(pnms)} pages...")
-
-        # Run unpaper in parallel on converted images
-        with ThreadPool(processes=settings.THREADS_PER_WORKER) as pool:
-            pnms = pool.map(run_unpaper, pnms)
-
-        return sorted(filter(lambda __: os.path.isfile(__), pnms))
-
-    def _guess_language(self, text):
-        try:
-            guess = langdetect.detect(text)
-            return guess
+            with Image.open(image) as im:
+                x, y = im.info['dpi']
+                return x
         except Exception as e:
-            self.log('warning', f"Language detection failed with: {e}")
+            self.log(
+                'warning',
+                f"Error while getting DPI from image {image}: {e}")
             return None
 
-    def _ocr(self, imgs, lang):
-        self.log(
-            "debug",
-            f"Performing OCR on {len(imgs)} page(s) with language {lang}")
-        with ThreadPool(processes=settings.THREADS_PER_WORKER) as pool:
-            r = pool.map(image_to_string, itertools.product(imgs, [lang]))
-            return r
+    def parse(self, document_path, mime_type):
+        if settings.OCR_MODE == "skip_noarchive":
+            text = get_text_from_pdf(document_path)
+            if text and len(text) > 50:
+                self.text = text
+                return
 
-    def _complete_ocr_default_language(self,
-                                       images,
-                                       sample_page_index,
-                                       sample_page):
-        images_copy = list(images)
-        del images_copy[sample_page_index]
-        if images_copy:
-            self.log('debug', "Continuing ocr with default language.")
-            ocr_pages = self._ocr(images_copy, settings.OCR_LANGUAGE)
-            ocr_pages.insert(sample_page_index, sample_page)
-            return ocr_pages
-        else:
-            return [sample_page]
+        archive_path = os.path.join(self.tempdir, "archive.pdf")
+
+        ocr_args = {
+            'input_file': document_path,
+            'output_file': archive_path,
+            'use_threads': True,
+            'jobs': settings.THREADS_PER_WORKER,
+            'language': settings.OCR_LANGUAGE,
+            'output_type': settings.OCR_OUTPUT_TYPE,
+            'progress_bar': False,
+            'clean': True
+        }
+
+        if settings.OCR_PAGES > 0:
+            ocr_args['pages'] = f"1-{settings.OCR_PAGES}"
+
+        if settings.OCR_MODE in ['skip', 'skip_noarchive']:
+            ocr_args['skip_text'] = True
+        elif settings.OCR_MODE == 'redo':
+            ocr_args['redo_ocr'] = True
+        elif settings.OCR_MODE == 'force':
+            ocr_args['force_ocr'] = True
+
+        if self.is_image(mime_type):
+            dpi = self.get_dpi(document_path)
+            if dpi:
+                self.log(
+                    "debug",
+                    f"Detected DPI for image {document_path}: {dpi}"
+                )
+                ocr_args['image_dpi'] = dpi
+            elif settings.OCR_IMAGE_DPI:
+                ocr_args['image_dpi'] = settings.OCR_IMAGE_DPI
+            else:
+                raise ParseError(
+                    f"Cannot produce archive PDF for image {document_path}, "
+                    f"no DPI information is present in this image and "
+                    f"OCR_IMAGE_DPI is not set.")
+
+        if settings.OCR_USER_ARGS:
+            try:
+                user_args = json.loads(settings.OCR_USER_ARGS)
+                ocr_args = {**ocr_args, **user_args}
+            except Exception as e:
+                self.log(
+                    "warning",
+                    f"There is an issue with PAPERLESS_OCR_USER_ARGS, so "
+                    f"they will not be used: {e}")
+
+        # This forces tesseract to use one core per page.
+        os.environ['OMP_THREAD_LIMIT'] = "1"
+
+        try:
+            self.log("debug",
+                     f"Calling OCRmyPDF with {str(ocr_args)}")
+            ocrmypdf.ocr(**ocr_args)
+            # success! announce results
+            self.archive_path = archive_path
+            self.text = get_text_from_pdf(archive_path)
+
+        except InputFileError as e:
+            # This happens with some PDFs when used with the redo_ocr option.
+            # This is not the end of the world, we'll just use what we already
+            # have in the document.
+            self.text = get_text_from_pdf(document_path)
+            # Also, no archived file.
+            if not self.text:
+                # However, if we don't have anything, fail:
+                raise ParseError(e)
+
+        except Exception as e:
+            # Anything else is probably serious.
+            raise ParseError(e)
+
+        if not self.text:
+            # This may happen for files that don't have any text.
+            self.log(
+                'warning',
+                f"Document {document_path} does not have any text."
+                f"This is probably an error or you tried to add an image "
+                f"without text.")
+            self.text = ""
 
 
 def strip_excess_whitespace(text):
+    if not text:
+        return None
+
     collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
     no_leading_whitespace = re.sub(
         r"([\n\r]+)([^\S\n\r]+)", '\\1', collapsed_spaces)
     no_trailing_whitespace = re.sub(
         r"([^\S\n\r]+)$", '', no_leading_whitespace)
-    return no_trailing_whitespace
 
-
-def image_to_string(args):
-    img, lang = args
-    ocr = pyocr.get_available_tools()[0]
-    with Image.open(img) as f:
-        if ocr.can_detect_orientation():
-            try:
-                orientation = ocr.detect_orientation(f, lang=lang)
-                f = f.rotate(orientation["angle"], expand=1)
-            except Exception:
-                # Rotation not possible, ignore
-                pass
-        try:
-            return ocr.image_to_string(f, lang=lang)
-        except PyocrException as e:
-            raise OCRError(e)
+    # TODO: this needs a rework
+    return no_trailing_whitespace.strip()
 
 
 def get_text_from_pdf(pdf_file):
@@ -245,6 +190,9 @@ def get_text_from_pdf(pdf_file):
         try:
             pdf = pdftotext.PDF(f)
         except pdftotext.Error:
-            return ""
+            # might not be a PDF file
+            return None
 
-    return "\n".join(pdf)
+    text = "\n".join(pdf)
+
+    return strip_excess_whitespace(text)

src/paperless_tesseract/tests/test_date.py

@@ -1,193 +0,0 @@
-import datetime
-import os
-import shutil
-from unittest import mock
-from uuid import uuid4
-
-from dateutil import tz
-from django.conf import settings
-from django.test import TestCase, override_settings
-
-from ..parsers import RasterisedDocumentParser
-
-
-class TestDate(TestCase):
-
-    SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "samples")
-    SCRATCH = "/tmp/paperless-tests-{}".format(str(uuid4())[:8])
-
-    def setUp(self):
-        os.makedirs(self.SCRATCH, exist_ok=True)
-
-    def tearDown(self):
-        shutil.rmtree(self.SCRATCH)
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_1(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = "lorem ipsum 130218 lorem ipsum"
-        self.assertEqual(document.get_date(), None)
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_2(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = "lorem ipsum 2018 lorem ipsum"
-        self.assertEqual(document.get_date(), None)
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_3(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = "lorem ipsum 20180213 lorem ipsum"
-        self.assertEqual(document.get_date(), None)
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_4(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = "lorem ipsum 13.02.2018 lorem ipsum"
-        date = document.get_date()
-        self.assertEqual(
-            date,
-            datetime.datetime(
-                2018, 2, 13, 0, 0,
-                tzinfo=tz.gettz(settings.TIME_ZONE)
-            )
-        )
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_5(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = (
-            "lorem ipsum 130218, 2018, 20180213 and lorem 13.02.2018 lorem "
-            "ipsum"
-        )
-        date = document.get_date()
-        self.assertEqual(
-            date,
-            datetime.datetime(
-                2018, 2, 13, 0, 0,
-                tzinfo=tz.gettz(settings.TIME_ZONE)
-            )
-        )
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_6(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = (
-            "lorem ipsum\n"
-            "Wohnort\n"
-            "3100\n"
-            "IBAN\n"
-            "AT87 4534\n"
-            "1234\n"
-            "1234 5678\n"
-            "BIC\n"
-            "lorem ipsum"
-        )
-        self.assertEqual(document.get_date(), None)
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_7(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = (
-            "lorem ipsum\n"
-            "März 2019\n"
-            "lorem ipsum"
-        )
-        date = document.get_date()
-        self.assertEqual(
-            date,
-            datetime.datetime(
-                2019, 3, 1, 0, 0,
-                tzinfo=tz.gettz(settings.TIME_ZONE)
-            )
-        )
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_8(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = (
-            "lorem ipsum\n"
-            "Wohnort\n"
-            "3100\n"
-            "IBAN\n"
-            "AT87 4534\n"
-            "1234\n"
-            "1234 5678\n"
-            "BIC\n"
-            "lorem ipsum\n"
-            "März 2020"
-        )
-        self.assertEqual(
-            document.get_date(),
-            datetime.datetime(
-                2020, 3, 1, 0, 0,
-                tzinfo=tz.gettz(settings.TIME_ZONE)
-            )
-        )
-
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_date_format_9(self):
-        input_file = os.path.join(self.SAMPLE_FILES, "")
-        document = RasterisedDocumentParser(input_file, None)
-        document._text = (
-            "lorem ipsum\n"
-            "27. Nullmonth 2020\n"
-            "März 2020\n"
-            "lorem ipsum"
-        )
-        self.assertEqual(
-            document.get_date(),
-            datetime.datetime(
-                2020, 3, 1, 0, 0,
-                tzinfo=tz.gettz(settings.TIME_ZONE)
-            )
-        )
-
-    @mock.patch(
-        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
-        return_value="01-07-0590 00:00:00"
-    )
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_crazy_date_past(self, *args):
-        document = RasterisedDocumentParser("/dev/null", None)
-        document.get_text()
-        self.assertIsNone(document.get_date())
-
-    @mock.patch(
-        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
-        return_value="01-07-2350 00:00:00"
-    )
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_crazy_date_future(self, *args):
-        document = RasterisedDocumentParser("/dev/null", None)
-        document.get_text()
-        self.assertIsNone(document.get_date())
-
-    @mock.patch(
-        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
-        return_value="20 408000l 2475"
-    )
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_crazy_date_with_spaces(self, *args):
-        document = RasterisedDocumentParser("/dev/null", None)
-        document.get_text()
-        self.assertIsNone(document.get_date())
-
-    @mock.patch(
-        "paperless_tesseract.parsers.RasterisedDocumentParser.get_text",
-        return_value="No date in here"
-    )
-    @override_settings(FILENAME_DATE_ORDER="YMD")
-    @override_settings(SCRATCH_DIR=SCRATCH)
-    def test_filename_date_parse_invalid(self, *args):
-        document = RasterisedDocumentParser("/tmp/20 408000l 2475 - test.pdf", None)
-        document.get_text()
-        self.assertIsNone(document.get_date())

src/paperless_tesseract/tests/test_ocr.py

@@ -1,76 +0,0 @@
-import os
-from unittest import mock, skipIf
-
-import pyocr
-from django.test import TestCase
-from pyocr.libtesseract.tesseract_raw import \
-    TesseractError as OtherTesseractError
-
-from ..parsers import image_to_string, strip_excess_whitespace
-
-
-class FakeTesseract(object):
-
-    @staticmethod
-    def can_detect_orientation():
-        return True
-
-    @staticmethod
-    def detect_orientation(file_handle, lang):
-        raise OtherTesseractError("arbitrary status", "message")
-
-    @staticmethod
-    def image_to_string(file_handle, lang):
-        return "This is test text"
-
-
-class FakePyOcr(object):
-
-    @staticmethod
-    def get_available_tools():
-        return [FakeTesseract]
-
-
-class TestOCR(TestCase):
-
-    text_cases = [
-        ("simple     string", "simple string"),
-        (
-            "simple    newline\n   testing string",
-            "simple newline\ntesting string"
-        ),
-        (
-            "utf-8   строка с пробелами в конце  ",
-            "utf-8 строка с пробелами в конце"
-        )
-    ]
-
-    SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "samples")
-    TESSERACT_INSTALLED = bool(pyocr.get_available_tools())
-
-    def test_strip_excess_whitespace(self):
-        for source, result in self.text_cases:
-            actual_result = strip_excess_whitespace(source)
-            self.assertEqual(
-                result,
-                actual_result,
-                "strip_exceess_whitespace({}) != '{}', but '{}'".format(
-                    source,
-                    result,
-                    actual_result
-                )
-            )
-
-    @skipIf(not TESSERACT_INSTALLED, "Tesseract not installed. Skipping")
-    @mock.patch("paperless_tesseract.parsers.pyocr", FakePyOcr)
-    def test_image_to_string_with_text_free_page(self):
-        """
-        This test is sort of silly, since it's really just reproducing an odd
-        exception thrown by pyocr when it encounters a page with no text.
-        Actually running this test against an installation of Tesseract results
-        in a segmentation fault rooted somewhere deep inside pyocr where I
-        don't care to dig.  Regardless, if you run the consumer normally,
-        text-free pages are now handled correctly so long as we work around
-        this weird exception.
-        """
-        image_to_string([os.path.join(self.SAMPLE_FILES, "no-text.png"), "en"])

src/paperless_tesseract/tests/test_parser.py

@@ -1,46 +1,17 @@
 import os
-import shutil
-import tempfile
 import uuid
 from typing import ContextManager
 from unittest import mock
 
 from django.test import TestCase, override_settings
-from pyocr.error import TesseractError
 
 from documents.parsers import ParseError, run_convert
-from paperless_tesseract.parsers import RasterisedDocumentParser, get_text_from_pdf, image_to_string, OCRError
+from documents.tests.utils import DirectoriesMixin
+from paperless_tesseract.parsers import RasterisedDocumentParser, get_text_from_pdf, strip_excess_whitespace
 
 image_to_string_calls = []
 
 
-class FakeTesseract(object):
-
-    @staticmethod
-    def can_detect_orientation():
-        return True
-
-    @staticmethod
-    def detect_orientation(file_handle, lang):
-        raise TesseractError("arbitrary status", "message")
-
-    @staticmethod
-    def get_available_languages():
-        return ['eng', 'deu']
-
-    @staticmethod
-    def image_to_string(file_handle, lang):
-        image_to_string_calls.append((file_handle.name, lang))
-        return file_handle.read()
-
-
-class FakePyOcr(object):
-
-    @staticmethod
-    def get_available_tools():
-        return [FakeTesseract]
-
-
 def fake_convert(input_file, output_file, **kwargs):
     with open(input_file) as f:
         lines = f.readlines()
@@ -50,12 +21,6 @@ def fake_convert(input_file, output_file, **kwargs):
             f2.write(line.strip())
 
 
-def fake_unpaper(pnm):
-    output = pnm + ".unpaper.pnm"
-    shutil.copy(pnm, output)
-    return output
-
-
 class FakeImageFile(ContextManager):
     def __init__(self, fname):
         self.fname = fname
@@ -67,142 +32,50 @@ class FakeImageFile(ContextManager):
         return os.path.basename(self.fname)
 
 
-fake_image = FakeImageFile
 
 
-@mock.patch("paperless_tesseract.parsers.pyocr", FakePyOcr)
-@mock.patch("paperless_tesseract.parsers.run_convert", fake_convert)
-@mock.patch("paperless_tesseract.parsers.run_unpaper", fake_unpaper)
-@mock.patch("paperless_tesseract.parsers.Image.open", open)
-class TestRasterisedDocumentParser(TestCase):
+class TestParser(DirectoriesMixin, TestCase):
 
-    def setUp(self):
-        self.scratch = tempfile.mkdtemp()
+    def assertContainsStrings(self, content, strings):
+        # Asserts that all strings appear in content, in the given order.
+        indices = [content.index(s) for s in strings]
+        self.assertListEqual(indices, sorted(indices))
 
-        global image_to_string_calls
+    text_cases = [
+        ("simple     string", "simple string"),
+        (
+            "simple    newline\n   testing string",
+            "simple newline\ntesting string"
+        ),
+        (
+            "utf-8   строка с пробелами в конце  ",
+            "utf-8 строка с пробелами в конце"
+        )
+    ]
 
-        image_to_string_calls = []
-
-        override_settings(OCR_LANGUAGE="eng", SCRATCH_DIR=self.scratch).enable()
-
-    def tearDown(self):
-        shutil.rmtree(self.scratch)
-
-    def get_input_file(self, pages):
-        _, fname = tempfile.mkstemp(suffix=".pdf", dir=self.scratch)
-        with open(fname, "w") as f:
-            f.writelines([f"line {p}\n" for p in range(pages)])
-        return fname
-
-    @mock.patch("paperless_tesseract.parsers.langdetect.detect", lambda _: "en")
-    def test_parse_text_simple_language_match(self):
-        parser = RasterisedDocumentParser(self.get_input_file(1), uuid.uuid4())
-        text = parser.get_text()
-        self.assertEqual(text, "line 0")
-
-        self.assertListEqual([args[1] for args in image_to_string_calls], ["eng"])
-
-    @mock.patch("paperless_tesseract.parsers.langdetect.detect", lambda _: "en")
-    def test_parse_text_2_pages(self):
-        parser = RasterisedDocumentParser(self.get_input_file(2), uuid.uuid4())
-        text = parser.get_text()
-        self.assertEqual(text, "line 0 line 1")
-
-        self.assertListEqual([args[1] for args in image_to_string_calls], ["eng", "eng"])
-
-    @mock.patch("paperless_tesseract.parsers.langdetect.detect", lambda _: "en")
-    def test_parse_text_3_pages(self):
-        parser = RasterisedDocumentParser(self.get_input_file(3), uuid.uuid4())
-        text = parser.get_text()
-        self.assertEqual(text, "line 0 line 1 line 2")
-
-        self.assertListEqual([args[1] for args in image_to_string_calls], ["eng", "eng", "eng"])
-
-    @mock.patch("paperless_tesseract.parsers.langdetect.detect", lambda _: None)
-    def test_parse_text_lang_detect_failed(self):
-        parser = RasterisedDocumentParser(self.get_input_file(3), uuid.uuid4())
-        text = parser.get_text()
-        self.assertEqual(text, "line 0 line 1 line 2")
-
-        self.assertListEqual([args[1] for args in image_to_string_calls], ["eng", "eng", "eng"])
-
-    @mock.patch("paperless_tesseract.parsers.langdetect.detect", lambda _: "it")
-    def test_parse_text_lang_not_installed(self):
-        parser = RasterisedDocumentParser(self.get_input_file(4), uuid.uuid4())
-        text = parser.get_text()
-        self.assertEqual(text, "line 0 line 1 line 2 line 3")
-
-        self.assertListEqual([args[1] for args in image_to_string_calls], ["eng", "eng", "eng", "eng"])
-
-    @mock.patch("paperless_tesseract.parsers.langdetect.detect", lambda _: "de")
-    def test_parse_text_lang_mismatch(self):
-        parser = RasterisedDocumentParser(self.get_input_file(3), uuid.uuid4())
-        text = parser.get_text()
-        self.assertEqual(text, "line 0 line 1 line 2")
-
-        self.assertListEqual([args[1] for args in image_to_string_calls], ["eng", "deu", "deu", "deu"])
-
-    @mock.patch("paperless_tesseract.parsers.langdetect.detect", lambda _: "de")
-    def test_parse_empty_doc(self):
-        parser = RasterisedDocumentParser(self.get_input_file(0), uuid.uuid4())
-        try:
-            parser.get_text()
-        except ParseError as e:
-            self.assertEqual("Empty document, nothing to do.", str(e))
-        else:
-            self.fail("Should raise exception")
-
-
-class TestAuxilliaryFunctions(TestCase):
-
-    def setUp(self):
-        self.scratch = tempfile.mkdtemp()
-
-        override_settings(SCRATCH_DIR=self.scratch).enable()
-
-    def tearDown(self):
-        shutil.rmtree(self.scratch)
+    def test_strip_excess_whitespace(self):
+        for source, result in self.text_cases:
+            actual_result = strip_excess_whitespace(source)
+            self.assertEqual(
+                result,
+                actual_result,
+                "strip_exceess_whitespace({}) != '{}', but '{}'".format(
+                    source,
+                    result,
+                    actual_result
+                )
+            )
 
     SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "samples")
 
     def test_get_text_from_pdf(self):
-        text = get_text_from_pdf(os.path.join(self.SAMPLE_FILES, 'simple.pdf'))
+        text = get_text_from_pdf(os.path.join(self.SAMPLE_FILES, 'simple-digital.pdf'))
 
-        self.assertEqual(text.strip(), "This is a test document.")
-
-    def test_get_text_from_pdf_error(self):
-        text = get_text_from_pdf(os.path.join(self.SAMPLE_FILES, 'simple.png'))
-
-        self.assertEqual(text.strip(), "")
-
-    def test_image_to_string(self):
-        text = image_to_string((os.path.join(self.SAMPLE_FILES, 'simple.png'), "eng"))
-
-        self.assertEqual(text, "This is a test document.")
-
-    def test_image_to_string_language_unavailable(self):
-        try:
-            image_to_string((os.path.join(self.SAMPLE_FILES, 'simple.png'), "ita"))
-        except OCRError as e:
-            self.assertTrue("Failed loading language" in str(e))
-        else:
-            self.fail("Should raise exception")
-
-    @override_settings(OCR_ALWAYS=False)
-    @mock.patch("paperless_tesseract.parsers.get_text_from_pdf")
-    @mock.patch("paperless_tesseract.parsers.RasterisedDocumentParser._get_greyscale")
-    def test_is_ocred(self, m2, m):
-        parser = RasterisedDocumentParser("", uuid.uuid4())
-        m.return_value = "lots of text lots of text lots of text lots of text lots of text lots of text " \
-                         "lots of text lots of text lots of text lots of text lots of text lots of text " \
-                         "lots of text lots of text lots of text lots of text lots of text lots of text "
-        parser.get_text()
-        self.assertEqual(m.call_count, 2)
-        self.assertEqual(m2.call_count, 0)
+        self.assertContainsStrings(text.strip(), ["This is a test document."])
 
     def test_thumbnail(self):
-        parser = RasterisedDocumentParser(os.path.join(self.SAMPLE_FILES, 'simple.pdf'), uuid.uuid4())
-        parser.get_thumbnail()
+        parser = RasterisedDocumentParser(uuid.uuid4())
+        parser.get_thumbnail(os.path.join(self.SAMPLE_FILES, 'simple-digital.pdf'), "application/pdf")
         # dont really know how to test it, just call it and assert that it does not raise anything.
 
     @mock.patch("paperless_tesseract.parsers.run_convert")
@@ -216,6 +89,161 @@ class TestAuxilliaryFunctions(TestCase):
 
         m.side_effect = call_convert
 
-        parser = RasterisedDocumentParser(os.path.join(self.SAMPLE_FILES, 'simple.pdf'), uuid.uuid4())
-        parser.get_thumbnail()
+        parser = RasterisedDocumentParser(uuid.uuid4())
+        parser.get_thumbnail(os.path.join(self.SAMPLE_FILES, 'simple-digital.pdf'), "application/pdf")
         # dont really know how to test it, just call it and assert that it does not raise anything.
+
+    def test_get_dpi(self):
+        parser = RasterisedDocumentParser(None)
+
+        dpi = parser.get_dpi(os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"))
+        self.assertEqual(dpi, None)
+
+        dpi = parser.get_dpi(os.path.join(self.SAMPLE_FILES, "simple.png"))
+        self.assertEqual(dpi, 72)
+
+    def test_simple_digital(self):
+        parser = RasterisedDocumentParser(None)
+
+        parser.parse(os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"), "application/pdf")
+
+        self.assertTrue(os.path.isfile(parser.archive_path))
+
+        self.assertContainsStrings(parser.get_text(), ["This is a test document."])
+
+    def test_with_form(self):
+        parser = RasterisedDocumentParser(None)
+
+        parser.parse(os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf")
+
+        self.assertTrue(os.path.isfile(parser.archive_path))
+
+        self.assertContainsStrings(parser.get_text(), ["Please enter your name in here:", "This is a PDF document with a form."])
+
+    @override_settings(OCR_MODE="redo")
+    def test_with_form_error(self):
+        parser = RasterisedDocumentParser(None)
+
+        parser.parse(os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf")
+
+        self.assertIsNone(parser.archive_path)
+        self.assertContainsStrings(parser.get_text(), ["Please enter your name in here:", "This is a PDF document with a form."])
+
+    @override_settings(OCR_MODE="redo")
+    @mock.patch("paperless_tesseract.parsers.get_text_from_pdf", lambda _: None)
+    def test_with_form_error_notext(self):
+        parser = RasterisedDocumentParser(None)
+
+        def f():
+            parser.parse(os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf")
+
+        self.assertRaises(ParseError, f)
+
+    @override_settings(OCR_MODE="force")
+    def test_with_form_force(self):
+        parser = RasterisedDocumentParser(None)
+
+        parser.parse(os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf")
+
+        self.assertContainsStrings(parser.get_text(), ["Please enter your name in here:", "This is a PDF document with a form."])
+
+    def test_image_simple(self):
+        parser = RasterisedDocumentParser(None)
+
+        parser.parse(os.path.join(self.SAMPLE_FILES, "simple.png"), "image/png")
+
+        self.assertTrue(os.path.isfile(parser.archive_path))
+
+        self.assertContainsStrings(parser.get_text(), ["This is a test document."])
+
+    def test_image_simple_alpha_fail(self):
+        parser = RasterisedDocumentParser(None)
+
+        def f():
+            parser.parse(os.path.join(self.SAMPLE_FILES, "simple-alpha.png"), "image/png")
+
+        self.assertRaises(ParseError, f)
+
+
+    def test_image_no_dpi_fail(self):
+        parser = RasterisedDocumentParser(None)
+
+        def f():
+            parser.parse(os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"), "image/png")
+
+        self.assertRaises(ParseError, f)
+
+    @override_settings(OCR_IMAGE_DPI=72)
+    def test_image_no_dpi_default(self):
+        parser = RasterisedDocumentParser(None)
+
+        parser.parse(os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"), "image/png")
+
+        self.assertTrue(os.path.isfile(parser.archive_path))
+
+        self.assertContainsStrings(parser.get_text().lower(), ["this is a test document."])
+
+    def test_multi_page(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf")
+        self.assertTrue(os.path.isfile(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2", "page 3"])
+
+    @override_settings(OCR_PAGES=2, OCR_MODE="skip")
+    def test_multi_page_pages_skip(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf")
+        self.assertTrue(os.path.isfile(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2", "page 3"])
+
+    @override_settings(OCR_PAGES=2, OCR_MODE="redo")
+    def test_multi_page_pages_redo(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf")
+        self.assertTrue(os.path.isfile(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2", "page 3"])
+
+    @override_settings(OCR_PAGES=2, OCR_MODE="force")
+    def test_multi_page_pages_force(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf")
+        self.assertTrue(os.path.isfile(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2", "page 3"])
+
+    @override_settings(OOCR_MODE="skip")
+    def test_multi_page_analog_pages_skip(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf")
+        self.assertTrue(os.path.isfile(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2", "page 3"])
+
+    @override_settings(OCR_PAGES=2, OCR_MODE="redo")
+    def test_multi_page_analog_pages_redo(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf")
+        self.assertTrue(os.path.isfile(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2"])
+        self.assertFalse("page 3" in parser.get_text().lower())
+
+    @override_settings(OCR_PAGES=1, OCR_MODE="force")
+    def test_multi_page_analog_pages_force(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf")
+        self.assertTrue(os.path.isfile(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1"])
+        self.assertFalse("page 2" in parser.get_text().lower())
+        self.assertFalse("page 3" in parser.get_text().lower())
+
+    @override_settings(OCR_MODE="skip_noarchive")
+    def test_skip_noarchive_withtext(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf")
+        self.assertIsNone(parser.archive_path)
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2", "page 3"])
+
+    @override_settings(OCR_MODE="skip_noarchive")
+    def test_skip_noarchive_notext(self):
+        parser = RasterisedDocumentParser(None)
+        parser.parse(os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf")
+        self.assertTrue(os.path.join(parser.archive_path))
+        self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2", "page 3"])

src/paperless_text/parsers.py

@@ -11,11 +11,7 @@ class TextDocumentParser(DocumentParser):
     This parser directly parses a text document (.txt, .md, or .csv)
     """
 
-    def __init__(self, path, logging_group):
-        super().__init__(path, logging_group)
-        self._text = None
-
-    def get_thumbnail(self):
+    def get_thumbnail(self, document_path, mime_type):
         """
         The thumbnail of a text file is just a 500px wide image of the text
         rendered onto a letter-sized page.
@@ -46,7 +42,7 @@ class TextDocumentParser(DocumentParser):
             )
 
         def read_text():
-            with open(self.document_path, 'r') as src:
+            with open(document_path, 'r') as src:
                 lines = [line.strip() for line in src.readlines()]
                 text = "\n".join([line for line in lines[:n_lines]])
                 return text.replace('"', "'")
@@ -76,15 +72,9 @@ class TextDocumentParser(DocumentParser):
 
         return out_path
 
-    def get_text(self):
-
-        if self._text is not None:
-            return self._text
-
-        with open(self.document_path, 'r') as f:
-            self._text = f.read()
-
-        return self._text
+    def parse(self, document_path, mime_type):
+        with open(document_path, 'r') as f:
+            self.text = f.read()
 
 
 def run_command(*args):