Merge branch 'dev' into feature-websockets-status

src/documents/admin.py

@@ -4,7 +4,8 @@ from django.utils.safestring import mark_safe
 from whoosh.writing import AsyncWriter
 
 from . import index
-from .models import Correspondent, Document, DocumentType, Log, Tag
+from .models import Correspondent, Document, DocumentType, Log, Tag, \
+    SavedView, SavedViewFilterRule
 
 
 class CorrespondentAdmin(admin.ModelAdmin):
@@ -17,8 +18,6 @@ class CorrespondentAdmin(admin.ModelAdmin):
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
-    readonly_fields = ("slug",)
-
 
 class TagAdmin(admin.ModelAdmin):
 
@@ -31,8 +30,6 @@ class TagAdmin(admin.ModelAdmin):
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
-    readonly_fields = ("slug", )
-
 
 class DocumentTypeAdmin(admin.ModelAdmin):
 
@@ -44,13 +41,16 @@ class DocumentTypeAdmin(admin.ModelAdmin):
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
-    readonly_fields = ("slug",)
-
 
 class DocumentAdmin(admin.ModelAdmin):
 
     search_fields = ("correspondent__name", "title", "content", "tags__name")
-    readonly_fields = ("added", "mime_type", "storage_type", "filename")
+    readonly_fields = (
+        "added",
+        "modified",
+        "mime_type",
+        "storage_type",
+        "filename")
 
     list_display_links = ("title",)
 
@@ -69,7 +69,7 @@ class DocumentAdmin(admin.ModelAdmin):
 
     filter_horizontal = ("tags",)
 
-    ordering = ["-created", "correspondent"]
+    ordering = ["-created"]
 
     date_hierarchy = "created"
 
@@ -101,7 +101,7 @@ class DocumentAdmin(admin.ModelAdmin):
         for tag in obj.tags.all():
             r += self._html_tag(
                 "span",
-                tag.slug + ", "
+                tag.name + ", "
             )
         return r
 
@@ -132,8 +132,22 @@ class LogAdmin(admin.ModelAdmin):
     list_display_links = ("created", "message")
 
 
+class RuleInline(admin.TabularInline):
+    model = SavedViewFilterRule
+
+
+class SavedViewAdmin(admin.ModelAdmin):
+
+    list_display = ("name", "user")
+
+    inlines = [
+        RuleInline
+    ]
+
+
 admin.site.register(Correspondent, CorrespondentAdmin)
 admin.site.register(Tag, TagAdmin)
 admin.site.register(DocumentType, DocumentTypeAdmin)
 admin.site.register(Document, DocumentAdmin)
 admin.site.register(Log, LogAdmin)
+admin.site.register(SavedView, SavedViewAdmin)

src/documents/apps.py

@@ -1,34 +1,30 @@
 from django.apps import AppConfig
 
+from django.utils.translation import gettext_lazy as _
+
 
 class DocumentsConfig(AppConfig):
 
     name = "documents"
 
-    def ready(self):
+    verbose_name = _("Documents")
 
-        from .signals import document_consumption_started
+    def ready(self):
         from .signals import document_consumption_finished
         from .signals.handlers import (
             add_inbox_tags,
-            run_pre_consume_script,
-            run_post_consume_script,
             set_log_entry,
             set_correspondent,
             set_document_type,
             set_tags,
             add_to_index
-
         )
 
-        document_consumption_started.connect(run_pre_consume_script)
-
         document_consumption_finished.connect(add_inbox_tags)
         document_consumption_finished.connect(set_correspondent)
         document_consumption_finished.connect(set_document_type)
         document_consumption_finished.connect(set_tags)
         document_consumption_finished.connect(set_log_entry)
         document_consumption_finished.connect(add_to_index)
-        document_consumption_finished.connect(run_post_consume_script)
 
         AppConfig.ready(self)

src/documents/bulk_edit.py

@@ -0,0 +1,107 @@
+import itertools
+
+from django.db.models import Q
+from django_q.tasks import async_task
+from whoosh.writing import AsyncWriter
+
+from documents import index
+from documents.models import Document, Correspondent, DocumentType
+
+
+def set_correspondent(doc_ids, correspondent):
+    if correspondent:
+        correspondent = Correspondent.objects.get(id=correspondent)
+
+    qs = Document.objects.filter(
+        Q(id__in=doc_ids) & ~Q(correspondent=correspondent))
+    affected_docs = [doc.id for doc in qs]
+    qs.update(correspondent=correspondent)
+
+    async_task(
+        "documents.tasks.bulk_update_documents", document_ids=affected_docs)
+
+    return "OK"
+
+
+def set_document_type(doc_ids, document_type):
+    if document_type:
+        document_type = DocumentType.objects.get(id=document_type)
+
+    qs = Document.objects.filter(
+        Q(id__in=doc_ids) & ~Q(document_type=document_type))
+    affected_docs = [doc.id for doc in qs]
+    qs.update(document_type=document_type)
+
+    async_task(
+        "documents.tasks.bulk_update_documents", document_ids=affected_docs)
+
+    return "OK"
+
+
+def add_tag(doc_ids, tag):
+
+    qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(tags__id=tag))
+    affected_docs = [doc.id for doc in qs]
+
+    DocumentTagRelationship = Document.tags.through
+
+    DocumentTagRelationship.objects.bulk_create([
+        DocumentTagRelationship(
+            document_id=doc, tag_id=tag) for doc in affected_docs
+    ])
+
+    async_task(
+        "documents.tasks.bulk_update_documents", document_ids=affected_docs)
+
+    return "OK"
+
+
+def remove_tag(doc_ids, tag):
+
+    qs = Document.objects.filter(Q(id__in=doc_ids) & Q(tags__id=tag))
+    affected_docs = [doc.id for doc in qs]
+
+    DocumentTagRelationship = Document.tags.through
+
+    DocumentTagRelationship.objects.filter(
+        Q(document_id__in=affected_docs) &
+        Q(tag_id=tag)
+    ).delete()
+
+    async_task(
+        "documents.tasks.bulk_update_documents", document_ids=affected_docs)
+
+    return "OK"
+
+
+def modify_tags(doc_ids, add_tags, remove_tags):
+    qs = Document.objects.filter(id__in=doc_ids)
+    affected_docs = [doc.id for doc in qs]
+
+    DocumentTagRelationship = Document.tags.through
+
+    DocumentTagRelationship.objects.filter(
+        document_id__in=affected_docs,
+        tag_id__in=remove_tags,
+    ).delete()
+
+    DocumentTagRelationship.objects.bulk_create([DocumentTagRelationship(
+        document_id=doc, tag_id=tag) for (doc, tag) in itertools.product(
+        affected_docs, add_tags)
+    ], ignore_conflicts=True)
+
+    async_task(
+        "documents.tasks.bulk_update_documents", document_ids=affected_docs)
+
+    return "OK"
+
+
+def delete(doc_ids):
+    Document.objects.filter(id__in=doc_ids).delete()
+
+    ix = index.open_index()
+    with AsyncWriter(ix) as writer:
+        for id in doc_ids:
+            index.remove_document_by_id(writer, id)
+
+    return "OK"

src/documents/checks.py

@@ -2,6 +2,7 @@ import textwrap
 
 from django.conf import settings
 from django.core.checks import Error, register
+from django.core.exceptions import FieldError
 from django.db.utils import OperationalError, ProgrammingError
 
 from documents.signals import document_consumer_declaration
@@ -16,7 +17,7 @@ def changed_password_check(app_configs, **kwargs):
     try:
         encrypted_doc = Document.objects.filter(
             storage_type=Document.STORAGE_TYPE_GPG).first()
-    except (OperationalError, ProgrammingError):
+    except (OperationalError, ProgrammingError, FieldError):
         return []  # No documents table yet
 
     if encrypted_doc:
@@ -50,6 +51,6 @@ def parser_check(app_configs, **kwargs):
 
     if len(parsers) == 0:
         return [Error("No parsers found. This is a bug. The consumer won't be "
-                      "able to onsume any documents without parsers.")]
+                      "able to consume any documents without parsers.")]
     else:
         return []

src/documents/consumer.py

@@ -1,7 +1,7 @@
 import datetime
 import hashlib
-import logging
 import os
+from subprocess import Popen
 
 import magic
 from asgiref.sync import async_to_sync
@@ -10,13 +10,15 @@ from django.conf import settings
 from django.db import transaction
 from django.db.models import Q
 from django.utils import timezone
+from filelock import FileLock
+from rest_framework.reverse import reverse
 
 from .classifier import DocumentClassifier, IncompatibleClassifierVersionError
-from .file_handling import create_source_path_directory
+from .file_handling import create_source_path_directory, \
+    generate_unique_filename
 from .loggers import LoggingMixin
 from .models import Document, FileInfo, Correspondent, DocumentType, Tag
-from .parsers import ParseError, get_parser_class_for_mime_type, \
-    get_supported_file_extensions, parse_date
+from .parsers import ParseError, get_parser_class_for_mime_type, parse_date
 from .signals import (
     document_consumption_finished,
     document_consumption_started
@@ -61,6 +63,10 @@ class Consumer(LoggingMixin):
 
     def pre_check_file_exists(self):
         if not os.path.isfile(self.path):
+            self.log(
+                "error",
+                "Cannot consume {}: It is not a file.".format(self.path)
+            )
             self._fail("File not found")
 
     def pre_check_duplicate(self):
@@ -69,6 +75,10 @@ class Consumer(LoggingMixin):
         if Document.objects.filter(Q(checksum=checksum) | Q(archive_checksum=checksum)).exists():  # NOQA: E501
             if settings.CONSUMER_DELETE_DUPLICATES:
                 os.unlink(self.path)
+            self.log(
+                "error",
+                "Not consuming {}: It is a duplicate.".format(self.filename)
+            )
             self._fail("Document is a duplicate")
 
     def pre_check_directories(self):
@@ -77,6 +87,39 @@ class Consumer(LoggingMixin):
         os.makedirs(settings.ORIGINALS_DIR, exist_ok=True)
         os.makedirs(settings.ARCHIVE_DIR, exist_ok=True)
 
+    def run_pre_consume_script(self):
+        if not settings.PRE_CONSUME_SCRIPT:
+            return
+
+        try:
+            Popen((settings.PRE_CONSUME_SCRIPT, self.path)).wait()
+        except Exception as e:
+            raise ConsumerError(
+                f"Error while executing pre-consume script: {e}"
+            )
+
+    def run_post_consume_script(self, document):
+        if not settings.POST_CONSUME_SCRIPT:
+            return
+
+        try:
+            Popen((
+                settings.POST_CONSUME_SCRIPT,
+                str(document.pk),
+                document.get_public_filename(),
+                os.path.normpath(document.source_path),
+                os.path.normpath(document.thumbnail_path),
+                reverse("document-download", kwargs={"pk": document.pk}),
+                reverse("document-thumb", kwargs={"pk": document.pk}),
+                str(document.correspondent),
+                str(",".join(document.tags.all().values_list(
+                    "name", flat=True)))
+            )).wait()
+        except Exception as e:
+            raise ConsumerError(
+                f"Error while executing pre-consume script: {e}"
+            )
+
     def try_consume_file(self,
                          path,
                          override_filename=None,
@@ -109,19 +152,20 @@ class Consumer(LoggingMixin):
         self.pre_check_directories()
         self.pre_check_duplicate()
 
-        self.log("info", "Consuming {}".format(self.filename))
+        self.log("info", f"Consuming {self.filename}")
 
         # Determine the parser class.
 
         mime_type = magic.from_file(self.path, mime=True)
 
+        self.log("debug", f"Detected mime type: {mime_type}")
+
         parser_class = get_parser_class_for_mime_type(mime_type)
         if not parser_class:
-            self._fail("No parsers abvailable")
+            self._fail(f"Unsupported mime type {mime_type}")
         else:
             self.log("debug",
-                     f"Parser: {parser_class.__name__} "
-                     f"based on mime type {mime_type}")
+                     f"Parser: {parser_class.__name__}")
 
         # Notify all listeners that we're going to do some work.
 
@@ -131,6 +175,8 @@ class Consumer(LoggingMixin):
             logging_group=self.logging_group
         )
 
+        self.run_pre_consume_script()
+
         def progress_callback(current_progress, max_progress, message):
             # recalculate progress to be within 20 and 80
             p = int((current_progress / max_progress) * 50 + 20)
@@ -149,7 +195,7 @@ class Consumer(LoggingMixin):
             self._send_progress(self.filename, 20, 100, 'WORKING',
                                 'Parsing document...')
             self.log("debug", "Parsing {}...".format(self.filename))
-            document_parser.parse(self.path, mime_type)
+            document_parser.parse(self.path, mime_type, self.filename)
 
             self.log("debug", f"Generating thumbnail for {self.filename}...")
             self._send_progress(self.filename, 70, 100, 'WORKING',
@@ -181,9 +227,10 @@ class Consumer(LoggingMixin):
         try:
             classifier = DocumentClassifier()
             classifier.reload()
-        except (FileNotFoundError, IncompatibleClassifierVersionError) as e:
-            logging.getLogger(__name__).warning(
-                "Cannot classify documents: {}.".format(e))
+        except (OSError, EOFError, IncompatibleClassifierVersionError) as e:
+            self.log(
+                "warning",
+                f"Cannot classify documents: {e}.")
             classifier = None
         self._send_progress(self.filename, 95, 100, 'WORKING',
                             'Storing the document...')
@@ -211,36 +258,34 @@ class Consumer(LoggingMixin):
 
                 # After everything is in the database, copy the files into
                 # place. If this fails, we'll also rollback the transaction.
+                with FileLock(settings.MEDIA_LOCK):
+                    document.filename = generate_unique_filename(
+                        document, settings.ORIGINALS_DIR)
+                    create_source_path_directory(document.source_path)
 
-                # TODO: not required, since this is done by the file handling
-                #  logic
-                create_source_path_directory(document.source_path)
-
-                self._write(document.storage_type,
-                            self.path, document.source_path)
-
-                self._write(document.storage_type,
-                            thumbnail, document.thumbnail_path)
-
-                if archive_path and os.path.isfile(archive_path):
                     self._write(document.storage_type,
-                                archive_path, document.archive_path)
+                                self.path, document.source_path)
 
-                    with open(archive_path, 'rb') as f:
-                        document.archive_checksum = hashlib.md5(
-                            f.read()).hexdigest()
-                        document.save()
+                    self._write(document.storage_type,
+                                thumbnail, document.thumbnail_path)
 
-                # Afte performing all database operations and moving files
-                # into place, tell paperless where the file is.
-                document.filename = os.path.basename(document.source_path)
-                # Saving the document now will trigger the filename handling
-                # logic.
+                    if archive_path and os.path.isfile(archive_path):
+                        create_source_path_directory(document.archive_path)
+                        self._write(document.storage_type,
+                                    archive_path, document.archive_path)
+
+                        with open(archive_path, 'rb') as f:
+                            document.archive_checksum = hashlib.md5(
+                                f.read()).hexdigest()
+
+                # Don't save with the lock active. Saving will cause the file
+                # renaming logic to aquire the lock as well.
                 document.save()
 
                 # Delete the file only if it was successfully consumed
                 self.log("debug", "Deleting file {}".format(self.path))
                 os.unlink(self.path)
+
         except Exception as e:
             self.log(
                 "error",
@@ -251,6 +296,8 @@ class Consumer(LoggingMixin):
         finally:
             document_parser.cleanup()
 
+        self.run_post_consume_script(document)
+
         self.log(
             "info",
             "Document {} consumption finished".format(document)
@@ -278,8 +325,7 @@ class Consumer(LoggingMixin):
 
         with open(self.path, "rb") as f:
             document = Document.objects.create(
-                correspondent=file_info.correspondent,
-                title=file_info.title,
+                title=(self.override_title or file_info.title)[:127],
                 content=text,
                 mime_type=mime_type,
                 checksum=hashlib.md5(f.read()).hexdigest(),
@@ -288,20 +334,13 @@ class Consumer(LoggingMixin):
                 storage_type=storage_type
             )
 
-        relevant_tags = set(file_info.tags)
-        if relevant_tags:
-            tag_names = ", ".join([t.slug for t in relevant_tags])
-            self.log("debug", "Tagging with {}".format(tag_names))
-            document.tags.add(*relevant_tags)
-
         self.apply_overrides(document)
 
+        document.save()
+
         return document
 
     def apply_overrides(self, document):
-        if self.override_title:
-            document.title = self.override_title
-
         if self.override_correspondent_id:
             document.correspondent = Correspondent.objects.get(
                 pk=self.override_correspondent_id)

src/documents/file_handling.py

@@ -8,6 +8,12 @@ from django.conf import settings
 from django.template.defaultfilters import slugify
 
 
+class defaultdictNoStr(defaultdict):
+
+    def __str__(self):
+        raise ValueError("Don't use {tags} directly.")
+
+
 def create_source_path_directory(source_path):
     os.makedirs(os.path.dirname(source_path), exist_ok=True)
 
@@ -70,13 +76,35 @@ def many_to_dictionary(field):
     return mydictionary
 
 
-def generate_filename(doc):
+def generate_unique_filename(doc, root):
+    counter = 0
+
+    while True:
+        new_filename = generate_filename(doc, counter)
+        if new_filename == doc.filename:
+            # still the same as before.
+            return new_filename
+
+        if os.path.exists(os.path.join(root, new_filename)):
+            counter += 1
+        else:
+            return new_filename
+
+
+def generate_filename(doc, counter=0):
     path = ""
 
     try:
         if settings.PAPERLESS_FILENAME_FORMAT is not None:
-            tags = defaultdict(lambda: slugify(None),
-                               many_to_dictionary(doc.tags))
+            tags = defaultdictNoStr(lambda: slugify(None),
+                                    many_to_dictionary(doc.tags))
+
+            tag_list = pathvalidate.sanitize_filename(
+                ",".join(sorted(
+                    [tag.name for tag in doc.tags.all()]
+                )),
+                replacement_text="-"
+            )
 
             if doc.correspondent:
                 correspondent = pathvalidate.sanitize_filename(
@@ -99,24 +127,28 @@ def generate_filename(doc):
                 document_type=document_type,
                 created=datetime.date.isoformat(doc.created),
                 created_year=doc.created.year if doc.created else "none",
-                created_month=doc.created.month if doc.created else "none",
-                created_day=doc.created.day if doc.created else "none",
+                created_month=f"{doc.created.month:02}" if doc.created else "none",  # NOQA: E501
+                created_day=f"{doc.created.day:02}" if doc.created else "none",
                 added=datetime.date.isoformat(doc.added),
                 added_year=doc.added.year if doc.added else "none",
-                added_month=doc.added.month if doc.added else "none",
-                added_day=doc.added.day if doc.added else "none",
+                added_month=f"{doc.added.month:02}" if doc.added else "none",
+                added_day=f"{doc.added.day:02}" if doc.added else "none",
                 tags=tags,
-            )
+                tag_list=tag_list
+            ).strip()
+
+            path = path.strip(os.sep)
+
     except (ValueError, KeyError, IndexError):
         logging.getLogger(__name__).warning(
             f"Invalid PAPERLESS_FILENAME_FORMAT: "
             f"{settings.PAPERLESS_FILENAME_FORMAT}, falling back to default")
 
-    # Always append the primary key to guarantee uniqueness of filename
+    counter_str = f"_{counter:02}" if counter else ""
     if len(path) > 0:
-        filename = "%s-%07i%s" % (path, doc.pk, doc.file_type)
+        filename = f"{path}{counter_str}{doc.file_type}"
     else:
-        filename = "%07i%s" % (doc.pk, doc.file_type)
+        filename = f"{doc.pk:07}{counter_str}{doc.file_type}"
 
     # Append .gpg for encrypted files
     if doc.storage_type == doc.STORAGE_TYPE_GPG:

src/documents/filters.py

@@ -4,7 +4,7 @@ from .models import Correspondent, Document, Tag, DocumentType, Log
 
 CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"]
 ID_KWARGS = ["in", "exact"]
-INT_KWARGS = ["exact", "gt", "gte", "lt", "lte"]
+INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"]
 DATE_KWARGS = ["year", "month", "day", "date__gt", "gt", "date__lt", "lt"]
 
 
@@ -37,6 +37,10 @@ class DocumentTypeFilterSet(FilterSet):
 
 class TagsFilter(Filter):
 
+    def __init__(self, exclude=False):
+        super(TagsFilter, self).__init__()
+        self.exclude = exclude
+
     def filter(self, qs, value):
         if not value:
             return qs
@@ -47,7 +51,10 @@ class TagsFilter(Filter):
             return qs
 
         for tag_id in tag_ids:
-            qs = qs.filter(tags__id=tag_id)
+            if self.exclude:
+                qs = qs.exclude(tags__id=tag_id)
+            else:
+                qs = qs.filter(tags__id=tag_id)
 
         return qs
 
@@ -74,6 +81,8 @@ class DocumentFilterSet(FilterSet):
 
     tags__id__all = TagsFilter()
 
+    tags__id__none = TagsFilter(exclude=True)
+
     is_in_inbox = InboxFilter()
 
     class Meta:
@@ -89,12 +98,14 @@ class DocumentFilterSet(FilterSet):
             "added": DATE_KWARGS,
             "modified": DATE_KWARGS,
 
+            "correspondent": ["isnull"],
             "correspondent__id": ID_KWARGS,
             "correspondent__name": CHAR_KWARGS,
 
             "tags__id": ID_KWARGS,
             "tags__name": CHAR_KWARGS,
 
+            "document_type": ["isnull"],
             "document_type__id": ID_KWARGS,
             "document_type__name": CHAR_KWARGS,
 

src/documents/index.py

@@ -3,7 +3,7 @@ import os
 from contextlib import contextmanager
 
 from django.conf import settings
-from whoosh import highlight
+from whoosh import highlight, classify, query
 from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME
 from whoosh.highlight import Formatter, get_text
 from whoosh.index import create_in, exists_in, open_dir
@@ -20,32 +20,37 @@ class JsonFormatter(Formatter):
         self.seen = {}
 
     def format_token(self, text, token, replace=False):
-        seen = self.seen
         ttext = self._text(get_text(text, token, replace))
-        if ttext in seen:
-            termnum = seen[ttext]
-        else:
-            termnum = len(seen)
-            seen[ttext] = termnum
-
-        return {'text': ttext, 'term': termnum}
+        return {'text': ttext, 'highlight': 'true'}
 
     def format_fragment(self, fragment, replace=False):
         output = []
         index = fragment.startchar
         text = fragment.text
-
+        amend_token = None
         for t in fragment.matches:
             if t.startchar is None:
                 continue
             if t.startchar < index:
                 continue
             if t.startchar > index:
-                output.append({'text': text[index:t.startchar]})
-            output.append(self.format_token(text, t, replace))
+                text_inbetween = text[index:t.startchar]
+                if amend_token and t.startchar - index < 10:
+                    amend_token['text'] += text_inbetween
+                else:
+                    output.append({'text': text_inbetween,
+                                   'highlight': False})
+                    amend_token = None
+            token = self.format_token(text, t, replace)
+            if amend_token:
+                amend_token['text'] += token['text']
+            else:
+                output.append(token)
+                amend_token = token
             index = t.endchar
         if index < fragment.endchar:
-            output.append({'text': text[index:fragment.endchar]})
+            output.append({'text': text[index:fragment.endchar],
+                           'highlight': False})
         return output
 
     def format(self, fragments, replace=False):
@@ -82,11 +87,6 @@ def open_index(recreate=False):
 
 
 def update_document(writer, doc):
-    # TODO: this line caused many issues all around, since:
-    #  We need to make sure that this method does not get called with
-    #  deserialized documents (i.e, document objects that don't come from
-    #  Django's ORM interfaces directly.
-    logger.debug("Indexing {}...".format(doc))
     tags = ",".join([t.name for t in doc.tags.all()])
     writer.update_document(
         id=doc.pk,
@@ -102,9 +102,11 @@ def update_document(writer, doc):
 
 
 def remove_document(writer, doc):
-    # TODO: see above.
-    logger.debug("Removing {} from index...".format(doc))
-    writer.delete_by_term('id', doc.pk)
+    remove_document_by_id(writer, doc.pk)
+
+
+def remove_document_by_id(writer, doc_id):
+    writer.delete_by_term('id', doc_id)
 
 
 def add_or_update_document(document):
@@ -120,22 +122,42 @@ def remove_document_from_index(document):
 
 
 @contextmanager
-def query_page(ix, querystring, page):
+def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content):
     searcher = ix.searcher()
     try:
-        qp = MultifieldParser(
-            ["content", "title", "correspondent", "tag", "type"],
-            ix.schema)
-        qp.add_plugin(DateParserPlugin())
+        if querystring:
+            qp = MultifieldParser(
+                ["content", "title", "correspondent", "tag", "type"],
+                ix.schema)
+            qp.add_plugin(DateParserPlugin())
+            str_q = qp.parse(querystring)
+            corrected = searcher.correct_query(str_q, querystring)
+        else:
+            str_q = None
+            corrected = None
+
+        if more_like_doc_id:
+            docnum = searcher.document_number(id=more_like_doc_id)
+            kts = searcher.key_terms_from_text(
+                'content', more_like_doc_content, numterms=20,
+                model=classify.Bo1Model, normalize=False)
+            more_like_q = query.Or(
+                [query.Term('content', word, boost=weight)
+                 for word, weight in kts])
+            result_page = searcher.search_page(
+                more_like_q, page, filter=str_q, mask={docnum})
+        elif str_q:
+            result_page = searcher.search_page(str_q, page)
+        else:
+            raise ValueError(
+                "Either querystring or more_like_doc_id is required."
+            )
 
-        q = qp.parse(querystring)
-        result_page = searcher.search_page(q, page)
         result_page.results.fragmenter = highlight.ContextFragmenter(
             surround=50)
         result_page.results.formatter = JsonFormatter()
 
-        corrected = searcher.correct_query(q, querystring)
-        if corrected.query != q:
+        if corrected and corrected.query != str_q:
             corrected_query = corrected.string
         else:
             corrected_query = None

src/documents/management/commands/decrypt_documents.py

@@ -2,7 +2,6 @@ import os
 
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
-from termcolor import colored as coloured
 
 from documents.models import Document
 from paperless.db import GnuPG
@@ -26,16 +25,14 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
 
         try:
-            print(coloured(
+            print(
                 "\n\nWARNING: This script is going to work directly on your "
                 "document originals, so\nWARNING: you probably shouldn't run "
                 "this unless you've got a recent backup\nWARNING: handy.  It "
                 "*should* work without a hitch, but be safe and backup your\n"
                 "WARNING: stuff first.\n\nHit Ctrl+C to exit now, or Enter to "
-                "continue.\n\n",
-                "yellow",
-                attrs=("bold",)
-            ))
+                "continue.\n\n"
+            )
             __ = input()
         except KeyboardInterrupt:
             return
@@ -57,8 +54,8 @@ class Command(BaseCommand):
 
         for document in encrypted_files:
 
-            print(coloured("Decrypting {}".format(
-                document).encode('utf-8'), "green"))
+            print("Decrypting {}".format(
+                document).encode('utf-8'))
 
             old_paths = [document.source_path, document.thumbnail_path]
 
@@ -82,7 +79,8 @@ class Command(BaseCommand):
             with open(document.thumbnail_path, "wb") as f:
                 f.write(raw_thumb)
 
-            document.save(update_fields=("storage_type", "filename"))
+            Document.objects.filter(id=document.id).update(
+                storage_type=document.storage_type, filename=document.filename)
 
             for path in old_paths:
                 os.unlink(path)

src/documents/management/commands/document_consumer.py

@@ -29,10 +29,9 @@ def _tags_from_path(filepath):
     path_parts = Path(filepath).relative_to(
                 settings.CONSUMPTION_DIR).parent.parts
     for part in path_parts:
-        tag_ids.add(Tag.objects.get_or_create(
-            slug=slugify(part),
-            defaults={"name": part},
-        )[0].pk)
+        tag_ids.add(Tag.objects.get_or_create(name__iexact=part, defaults={
+            "name": part
+        })[0].pk)
 
     return tag_ids
 

src/documents/management/commands/document_importer.py

@@ -1,16 +1,31 @@
 import json
+import logging
 import os
 import shutil
+from contextlib import contextmanager
 
+import tqdm
 from django.conf import settings
 from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
+from django.db.models.signals import post_save, m2m_changed
+from filelock import FileLock
 
 from documents.models import Document
 from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME, \
     EXPORTER_ARCHIVE_NAME
-from ...file_handling import generate_filename, create_source_path_directory
+from ...file_handling import create_source_path_directory
 from ...mixins import Renderable
+from ...signals.handlers import update_filename_and_move_files
+
+
+@contextmanager
+def disable_signal(sig, receiver, sender):
+    try:
+        sig.disconnect(receiver=receiver, sender=sender)
+        yield
+    finally:
+        sig.connect(receiver=receiver, sender=sender)
 
 
 class Command(Renderable, BaseCommand):
@@ -30,6 +45,8 @@ class Command(Renderable, BaseCommand):
 
     def handle(self, *args, **options):
 
+        logging.getLogger().handlers[0].level = logging.ERROR
+
         self.source = options["source"]
 
         if not os.path.exists(self.source):
@@ -45,11 +62,19 @@ class Command(Renderable, BaseCommand):
             self.manifest = json.load(f)
 
         self._check_manifest()
+        with disable_signal(post_save,
+                            receiver=update_filename_and_move_files,
+                            sender=Document):
+            with disable_signal(m2m_changed,
+                                receiver=update_filename_and_move_files,
+                                sender=Document.tags.through):
+                # Fill up the database with whatever is in the manifest
+                call_command("loaddata", manifest_path)
 
-        # Fill up the database with whatever is in the manifest
-        call_command("loaddata", manifest_path)
+                self._import_files_from_manifest()
 
-        self._import_files_from_manifest()
+        print("Updating search index...")
+        call_command('document_index', 'reindex')
 
     @staticmethod
     def _check_manifest_exists(path):
@@ -93,10 +118,13 @@ class Command(Renderable, BaseCommand):
         os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True)
         os.makedirs(settings.ARCHIVE_DIR, exist_ok=True)
 
-        for record in self.manifest:
+        print("Copy files into paperless...")
 
-            if not record["model"] == "documents.document":
-                continue
+        manifest_documents = list(filter(
+            lambda r: r["model"] == "documents.document",
+            self.manifest))
+
+        for record in tqdm.tqdm(manifest_documents):
 
             document = Document.objects.get(pk=record["pk"])
 
@@ -114,17 +142,16 @@ class Command(Renderable, BaseCommand):
 
             document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
 
-            document.filename = generate_filename(document)
+            with FileLock(settings.MEDIA_LOCK):
+                if os.path.isfile(document.source_path):
+                    raise FileExistsError(document.source_path)
 
-            if os.path.isfile(document.source_path):
-                raise FileExistsError(document.source_path)
+                create_source_path_directory(document.source_path)
 
-            create_source_path_directory(document.source_path)
-
-            print(f"Moving {document_path} to {document.source_path}")
-            shutil.copy(document_path, document.source_path)
-            shutil.copy(thumbnail_path, document.thumbnail_path)
-            if archive_path:
-                shutil.copy(archive_path, document.archive_path)
+                shutil.copy(document_path, document.source_path)
+                shutil.copy(thumbnail_path, document.thumbnail_path)
+                if archive_path:
+                    create_source_path_directory(document.archive_path)
+                    shutil.copy(archive_path, document.archive_path)
 
             document.save()

src/documents/management/commands/document_index.py

@@ -1,4 +1,5 @@
 from django.core.management import BaseCommand
+from django.db import transaction
 
 from documents.mixins import Renderable
 from documents.tasks import index_reindex, index_optimize
@@ -18,8 +19,8 @@ class Command(Renderable, BaseCommand):
     def handle(self, *args, **options):
 
         self.verbosity = options["verbosity"]
-
-        if options['command'] == 'reindex':
-            index_reindex()
-        elif options['command'] == 'optimize':
-            index_optimize()
+        with transaction.atomic():
+            if options['command'] == 'reindex':
+                index_reindex()
+            elif options['command'] == 'optimize':
+                index_optimize()

src/documents/management/commands/document_renamer.py

@@ -1,4 +1,8 @@
+import logging
+
+import tqdm
 from django.core.management.base import BaseCommand
+from django.db.models.signals import post_save
 
 from documents.models import Document
 from ...mixins import Renderable
@@ -18,6 +22,7 @@ class Command(Renderable, BaseCommand):
 
         self.verbosity = options["verbosity"]
 
-        for document in Document.objects.all():
-            # Saving the document again will generate a new filename and rename
-            document.save()
+        logging.getLogger().handlers[0].level = logging.ERROR
+
+        for document in tqdm.tqdm(Document.objects.all()):
+            post_save.send(Document, instance=document)

src/documents/management/commands/document_retagger.py

@@ -73,7 +73,7 @@ class Command(Renderable, BaseCommand):
         classifier = DocumentClassifier()
         try:
             classifier.reload()
-        except (FileNotFoundError, IncompatibleClassifierVersionError) as e:
+        except (OSError, EOFError, IncompatibleClassifierVersionError) as e:
             logging.getLogger(__name__).warning(
                 f"Cannot classify documents: {e}.")
             classifier = None

src/documents/management/commands/document_thumbnails.py

@@ -0,0 +1,68 @@
+import logging
+import multiprocessing
+import shutil
+
+import tqdm
+from django import db
+from django.core.management.base import BaseCommand
+
+from documents.models import Document
+from ...mixins import Renderable
+from ...parsers import get_parser_class_for_mime_type
+
+
+def _process_document(doc_in):
+    document = Document.objects.get(id=doc_in)
+    parser = get_parser_class_for_mime_type(document.mime_type)(
+        logging_group=None)
+    try:
+        thumb = parser.get_optimised_thumbnail(
+            document.source_path, document.mime_type)
+
+        shutil.move(thumb, document.thumbnail_path)
+    finally:
+        parser.cleanup()
+
+
+class Command(Renderable, BaseCommand):
+
+    help = """
+        This will regenerate the thumbnails for all documents.
+    """.replace("    ", "")
+
+    def __init__(self, *args, **kwargs):
+        self.verbosity = 0
+        BaseCommand.__init__(self, *args, **kwargs)
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-d", "--document",
+            default=None,
+            type=int,
+            required=False,
+            help="Specify the ID of a document, and this command will only "
+                 "run on this specific document."
+        )
+
+    def handle(self, *args, **options):
+
+        self.verbosity = options["verbosity"]
+
+        logging.getLogger().handlers[0].level = logging.ERROR
+
+        if options['document']:
+            documents = Document.objects.filter(pk=options['document'])
+        else:
+            documents = Document.objects.all()
+
+        ids = [doc.id for doc in documents]
+
+        # Note to future self: this prevents django from reusing database
+        # conncetions between processes, which is bad and does not work
+        # with postgres.
+        db.connections.close_all()
+
+        with multiprocessing.Pool() as pool:
+            list(tqdm.tqdm(
+                pool.imap_unordered(_process_document, ids), total=len(ids)
+            ))

src/documents/migrations/1003_mime_types.py

@@ -6,13 +6,18 @@ import magic
 from django.conf import settings
 from django.db import migrations, models
 
+from paperless.db import GnuPG
+
+STORAGE_TYPE_UNENCRYPTED = "unencrypted"
+STORAGE_TYPE_GPG = "gpg"
+
 
 def source_path(self):
     if self.filename:
         fname = str(self.filename)
     else:
         fname = "{:07}.{}".format(self.pk, self.file_type)
-        if self.storage_type == self.STORAGE_TYPE_GPG:
+        if self.storage_type == STORAGE_TYPE_GPG:
             fname += ".gpg"
 
     return os.path.join(
@@ -26,9 +31,18 @@ def add_mime_types(apps, schema_editor):
     documents = Document.objects.all()
 
     for d in documents:
-        d.mime_type = magic.from_file(source_path(d), mime=True)
+        f = open(source_path(d), "rb")
+        if d.storage_type == STORAGE_TYPE_GPG:
+
+            data = GnuPG.decrypted(f)
+        else:
+            data = f.read(1024)
+
+        d.mime_type = magic.from_buffer(data, mime=True)
         d.save()
 
+        f.close()
+
 
 def add_file_extensions(apps, schema_editor):
     Document = apps.get_model("documents", "Document")

src/documents/migrations/1006_auto_20201208_2209.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.1.4 on 2020-12-08 22:09
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '1005_checksums'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='correspondent',
+            name='slug',
+        ),
+        migrations.RemoveField(
+            model_name='documenttype',
+            name='slug',
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='slug',
+        ),
+    ]

src/documents/migrations/1007_savedview_savedviewfilterrule.py

@@ -0,0 +1,37 @@
+# Generated by Django 3.1.4 on 2020-12-12 14:41
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('documents', '1006_auto_20201208_2209'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SavedView',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=128)),
+                ('show_on_dashboard', models.BooleanField()),
+                ('show_in_sidebar', models.BooleanField()),
+                ('sort_field', models.CharField(max_length=128)),
+                ('sort_reverse', models.BooleanField(default=False)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='SavedViewFilterRule',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('rule_type', models.PositiveIntegerField(choices=[(0, 'Title contains'), (1, 'Content contains'), (2, 'ASN is'), (3, 'Correspondent is'), (4, 'Document type is'), (5, 'Is in inbox'), (6, 'Has tag'), (7, 'Has any tag'), (8, 'Created before'), (9, 'Created after'), (10, 'Created year is'), (11, 'Created month is'), (12, 'Created day is'), (13, 'Added before'), (14, 'Added after'), (15, 'Modified before'), (16, 'Modified after'), (17, 'Does not have tag')])),
+                ('value', models.CharField(max_length=128)),
+                ('saved_view', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='filter_rules', to='documents.savedview')),
+            ],
+        ),
+    ]

src/documents/migrations/1008_auto_20201216_1736.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.1.4 on 2020-12-16 17:36
+
+from django.db import migrations
+import django.db.models.functions.text
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '1007_savedview_savedviewfilterrule'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='correspondent',
+            options={'ordering': (django.db.models.functions.text.Lower('name'),)},
+        ),
+        migrations.AlterModelOptions(
+            name='document',
+            options={'ordering': ('-created',)},
+        ),
+        migrations.AlterModelOptions(
+            name='documenttype',
+            options={'ordering': (django.db.models.functions.text.Lower('name'),)},
+        ),
+        migrations.AlterModelOptions(
+            name='savedview',
+            options={'ordering': (django.db.models.functions.text.Lower('name'),)},
+        ),
+        migrations.AlterModelOptions(
+            name='tag',
+            options={'ordering': (django.db.models.functions.text.Lower('name'),)},
+        ),
+    ]

src/documents/migrations/1009_auto_20201216_2005.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.1.4 on 2020-12-16 20:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '1008_auto_20201216_1736'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='correspondent',
+            options={'ordering': ('name',)},
+        ),
+        migrations.AlterModelOptions(
+            name='documenttype',
+            options={'ordering': ('name',)},
+        ),
+        migrations.AlterModelOptions(
+            name='savedview',
+            options={'ordering': ('name',)},
+        ),
+        migrations.AlterModelOptions(
+            name='tag',
+            options={'ordering': ('name',)},
+        ),
+    ]

src/documents/migrations/1010_auto_20210101_2159.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.4 on 2021-01-01 21:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '1009_auto_20201216_2005'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='savedviewfilterrule',
+            name='value',
+            field=models.CharField(blank=True, max_length=128, null=True),
+        ),
+    ]

src/documents/migrations/1011_auto_20210101_2340.py

@@ -0,0 +1,250 @@
+# Generated by Django 3.1.4 on 2021-01-01 23:40
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('documents', '1010_auto_20210101_2159'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='correspondent',
+            options={'ordering': ('name',), 'verbose_name': 'correspondent', 'verbose_name_plural': 'correspondents'},
+        ),
+        migrations.AlterModelOptions(
+            name='document',
+            options={'ordering': ('-created',), 'verbose_name': 'document', 'verbose_name_plural': 'documents'},
+        ),
+        migrations.AlterModelOptions(
+            name='documenttype',
+            options={'verbose_name': 'document type', 'verbose_name_plural': 'document types'},
+        ),
+        migrations.AlterModelOptions(
+            name='log',
+            options={'ordering': ('-created',), 'verbose_name': 'log', 'verbose_name_plural': 'logs'},
+        ),
+        migrations.AlterModelOptions(
+            name='savedview',
+            options={'ordering': ('name',), 'verbose_name': 'saved view', 'verbose_name_plural': 'saved views'},
+        ),
+        migrations.AlterModelOptions(
+            name='savedviewfilterrule',
+            options={'verbose_name': 'filter rule', 'verbose_name_plural': 'filter rules'},
+        ),
+        migrations.AlterModelOptions(
+            name='tag',
+            options={'verbose_name': 'tag', 'verbose_name_plural': 'tags'},
+        ),
+        migrations.AlterField(
+            model_name='correspondent',
+            name='is_insensitive',
+            field=models.BooleanField(default=True, verbose_name='is insensitive'),
+        ),
+        migrations.AlterField(
+            model_name='correspondent',
+            name='match',
+            field=models.CharField(blank=True, max_length=256, verbose_name='match'),
+        ),
+        migrations.AlterField(
+            model_name='correspondent',
+            name='matching_algorithm',
+            field=models.PositiveIntegerField(choices=[(1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
+        ),
+        migrations.AlterField(
+            model_name='correspondent',
+            name='name',
+            field=models.CharField(max_length=128, unique=True, verbose_name='name'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='added',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, editable=False, verbose_name='added'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='archive_checksum',
+            field=models.CharField(blank=True, editable=False, help_text='The checksum of the archived document.', max_length=32, null=True, verbose_name='archive checksum'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='archive_serial_number',
+            field=models.IntegerField(blank=True, db_index=True, help_text='The position of this document in your physical document archive.', null=True, unique=True, verbose_name='archive serial number'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='checksum',
+            field=models.CharField(editable=False, help_text='The checksum of the original document.', max_length=32, unique=True, verbose_name='checksum'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='content',
+            field=models.TextField(blank=True, help_text='The raw, text-only data of the document. This field is primarily used for searching.', verbose_name='content'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='correspondent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='documents.correspondent', verbose_name='correspondent'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='created',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='created'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='document_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='documents.documenttype', verbose_name='document type'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='filename',
+            field=models.FilePathField(default=None, editable=False, help_text='Current filename in storage', max_length=1024, null=True, verbose_name='filename'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='mime_type',
+            field=models.CharField(editable=False, max_length=256, verbose_name='mime type'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='modified',
+            field=models.DateTimeField(auto_now=True, db_index=True, verbose_name='modified'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='storage_type',
+            field=models.CharField(choices=[('unencrypted', 'Unencrypted'), ('gpg', 'Encrypted with GNU Privacy Guard')], default='unencrypted', editable=False, max_length=11, verbose_name='storage type'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='tags',
+            field=models.ManyToManyField(blank=True, related_name='documents', to='documents.Tag', verbose_name='tags'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='title',
+            field=models.CharField(blank=True, db_index=True, max_length=128, verbose_name='title'),
+        ),
+        migrations.AlterField(
+            model_name='documenttype',
+            name='is_insensitive',
+            field=models.BooleanField(default=True, verbose_name='is insensitive'),
+        ),
+        migrations.AlterField(
+            model_name='documenttype',
+            name='match',
+            field=models.CharField(blank=True, max_length=256, verbose_name='match'),
+        ),
+        migrations.AlterField(
+            model_name='documenttype',
+            name='matching_algorithm',
+            field=models.PositiveIntegerField(choices=[(1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
+        ),
+        migrations.AlterField(
+            model_name='documenttype',
+            name='name',
+            field=models.CharField(max_length=128, unique=True, verbose_name='name'),
+        ),
+        migrations.AlterField(
+            model_name='log',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, verbose_name='created'),
+        ),
+        migrations.AlterField(
+            model_name='log',
+            name='group',
+            field=models.UUIDField(blank=True, null=True, verbose_name='group'),
+        ),
+        migrations.AlterField(
+            model_name='log',
+            name='level',
+            field=models.PositiveIntegerField(choices=[(10, 'debug'), (20, 'information'), (30, 'warning'), (40, 'error'), (50, 'critical')], default=20, verbose_name='level'),
+        ),
+        migrations.AlterField(
+            model_name='log',
+            name='message',
+            field=models.TextField(verbose_name='message'),
+        ),
+        migrations.AlterField(
+            model_name='savedview',
+            name='name',
+            field=models.CharField(max_length=128, verbose_name='name'),
+        ),
+        migrations.AlterField(
+            model_name='savedview',
+            name='show_in_sidebar',
+            field=models.BooleanField(verbose_name='show in sidebar'),
+        ),
+        migrations.AlterField(
+            model_name='savedview',
+            name='show_on_dashboard',
+            field=models.BooleanField(verbose_name='show on dashboard'),
+        ),
+        migrations.AlterField(
+            model_name='savedview',
+            name='sort_field',
+            field=models.CharField(max_length=128, verbose_name='sort field'),
+        ),
+        migrations.AlterField(
+            model_name='savedview',
+            name='sort_reverse',
+            field=models.BooleanField(default=False, verbose_name='sort reverse'),
+        ),
+        migrations.AlterField(
+            model_name='savedview',
+            name='user',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='user'),
+        ),
+        migrations.AlterField(
+            model_name='savedviewfilterrule',
+            name='rule_type',
+            field=models.PositiveIntegerField(choices=[(0, 'title contains'), (1, 'content contains'), (2, 'ASN is'), (3, 'correspondent is'), (4, 'document type is'), (5, 'is in inbox'), (6, 'has tag'), (7, 'has any tag'), (8, 'created before'), (9, 'created after'), (10, 'created year is'), (11, 'created month is'), (12, 'created day is'), (13, 'added before'), (14, 'added after'), (15, 'modified before'), (16, 'modified after'), (17, 'does not have tag')], verbose_name='rule type'),
+        ),
+        migrations.AlterField(
+            model_name='savedviewfilterrule',
+            name='saved_view',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='filter_rules', to='documents.savedview', verbose_name='saved view'),
+        ),
+        migrations.AlterField(
+            model_name='savedviewfilterrule',
+            name='value',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='value'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='colour',
+            field=models.PositiveIntegerField(choices=[(1, '#a6cee3'), (2, '#1f78b4'), (3, '#b2df8a'), (4, '#33a02c'), (5, '#fb9a99'), (6, '#e31a1c'), (7, '#fdbf6f'), (8, '#ff7f00'), (9, '#cab2d6'), (10, '#6a3d9a'), (11, '#b15928'), (12, '#000000'), (13, '#cccccc')], default=1, verbose_name='color'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='is_inbox_tag',
+            field=models.BooleanField(default=False, help_text='Marks this tag as an inbox tag: All newly consumed documents will be tagged with inbox tags.', verbose_name='is inbox tag'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='is_insensitive',
+            field=models.BooleanField(default=True, verbose_name='is insensitive'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='match',
+            field=models.CharField(blank=True, max_length=256, verbose_name='match'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='matching_algorithm',
+            field=models.PositiveIntegerField(choices=[(1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='name',
+            field=models.CharField(max_length=128, unique=True, verbose_name='name'),
+        ),
+    ]

src/documents/models.py

@@ -9,9 +9,11 @@ import pathvalidate
 
 import dateutil.parser
 from django.conf import settings
+from django.contrib.auth.models import User
 from django.db import models
 from django.utils import timezone
-from django.utils.text import slugify
+
+from django.utils.translation import gettext_lazy as _
 
 from documents.file_handling import archive_name_from_filename
 from documents.parsers import get_default_file_extension
@@ -27,37 +29,31 @@ class MatchingModel(models.Model):
     MATCH_AUTO = 6
 
     MATCHING_ALGORITHMS = (
-        (MATCH_ANY, "Any"),
-        (MATCH_ALL, "All"),
-        (MATCH_LITERAL, "Literal"),
-        (MATCH_REGEX, "Regular Expression"),
-        (MATCH_FUZZY, "Fuzzy Match"),
-        (MATCH_AUTO, "Automatic Classification"),
+        (MATCH_ANY, _("Any word")),
+        (MATCH_ALL, _("All words")),
+        (MATCH_LITERAL, _("Exact match")),
+        (MATCH_REGEX, _("Regular expression")),
+        (MATCH_FUZZY, _("Fuzzy word")),
+        (MATCH_AUTO, _("Automatic")),
     )
 
-    name = models.CharField(max_length=128, unique=True)
-    slug = models.SlugField(blank=True, editable=False)
+    name = models.CharField(
+        _("name"),
+        max_length=128, unique=True)
+
+    match = models.CharField(
+        _("match"),
+        max_length=256, blank=True)
 
-    match = models.CharField(max_length=256, blank=True)
     matching_algorithm = models.PositiveIntegerField(
+        _("matching algorithm"),
         choices=MATCHING_ALGORITHMS,
-        default=MATCH_ANY,
-        help_text=(
-            "Which algorithm you want to use when matching text to the OCR'd "
-            "PDF.  Here, \"any\" looks for any occurrence of any word "
-            "provided in the PDF, while \"all\" requires that every word "
-            "provided appear in the PDF, albeit not in the order provided.  A "
-            "\"literal\" match means that the text you enter must appear in "
-            "the PDF exactly as you've entered it, and \"regular expression\" "
-            "uses a regex to match the PDF.  (If you don't know what a regex "
-            "is, you probably don't want this option.)  Finally, a \"fuzzy "
-            "match\" looks for words or phrases that are mostly—but not "
-            "exactly—the same, which can be useful for matching against "
-            "documents containg imperfections that foil accurate OCR."
-        )
+        default=MATCH_ANY
     )
 
-    is_insensitive = models.BooleanField(default=True)
+    is_insensitive = models.BooleanField(
+        _("is insensitive"),
+        default=True)
 
     class Meta:
         abstract = True
@@ -69,7 +65,6 @@ class MatchingModel(models.Model):
     def save(self, *args, **kwargs):
 
         self.match = self.match.lower()
-        self.slug = slugify(self.name)
 
         models.Model.save(self, *args, **kwargs)
 
@@ -82,6 +77,8 @@ class Correspondent(MatchingModel):
 
     class Meta:
         ordering = ("name",)
+        verbose_name = _("correspondent")
+        verbose_name_plural = _("correspondents")
 
 
 class Tag(MatchingModel):
@@ -102,18 +99,27 @@ class Tag(MatchingModel):
         (13, "#cccccc")
     )
 
-    colour = models.PositiveIntegerField(choices=COLOURS, default=1)
+    colour = models.PositiveIntegerField(
+        _("color"),
+        choices=COLOURS, default=1)
 
     is_inbox_tag = models.BooleanField(
+        _("is inbox tag"),
         default=False,
-        help_text="Marks this tag as an inbox tag: All newly consumed "
-                  "documents will be tagged with inbox tags."
+        help_text=_("Marks this tag as an inbox tag: All newly consumed "
+                    "documents will be tagged with inbox tags.")
     )
 
+    class Meta:
+        verbose_name = _("tag")
+        verbose_name_plural = _("tags")
+
 
 class DocumentType(MatchingModel):
 
-    pass
+    class Meta:
+        verbose_name = _("document type")
+        verbose_name_plural = _("document types")
 
 
 class Document(models.Model):
@@ -121,8 +127,8 @@ class Document(models.Model):
     STORAGE_TYPE_UNENCRYPTED = "unencrypted"
     STORAGE_TYPE_GPG = "gpg"
     STORAGE_TYPES = (
-        (STORAGE_TYPE_UNENCRYPTED, "Unencrypted"),
-        (STORAGE_TYPE_GPG, "Encrypted with GNU Privacy Guard")
+        (STORAGE_TYPE_UNENCRYPTED, _("Unencrypted")),
+        (STORAGE_TYPE_GPG, _("Encrypted with GNU Privacy Guard"))
     )
 
     correspondent = models.ForeignKey(
@@ -130,54 +136,68 @@ class Document(models.Model):
         blank=True,
         null=True,
         related_name="documents",
-        on_delete=models.SET_NULL
+        on_delete=models.SET_NULL,
+        verbose_name=_("correspondent")
     )
 
-    title = models.CharField(max_length=128, blank=True, db_index=True)
+    title = models.CharField(
+        _("title"),
+        max_length=128, blank=True, db_index=True)
 
     document_type = models.ForeignKey(
         DocumentType,
         blank=True,
         null=True,
         related_name="documents",
-        on_delete=models.SET_NULL
+        on_delete=models.SET_NULL,
+        verbose_name=_("document type")
     )
 
     content = models.TextField(
+        _("content"),
         blank=True,
-        help_text="The raw, text-only data of the document. This field is "
-                  "primarily used for searching."
+        help_text=_("The raw, text-only data of the document. This field is "
+                    "primarily used for searching.")
     )
 
     mime_type = models.CharField(
+        _("mime type"),
         max_length=256,
         editable=False
     )
 
     tags = models.ManyToManyField(
-        Tag, related_name="documents", blank=True)
+        Tag, related_name="documents", blank=True,
+        verbose_name=_("tags")
+    )
 
     checksum = models.CharField(
+        _("checksum"),
         max_length=32,
         editable=False,
         unique=True,
-        help_text="The checksum of the original document."
+        help_text=_("The checksum of the original document.")
     )
 
     archive_checksum = models.CharField(
+        _("archive checksum"),
         max_length=32,
         editable=False,
         blank=True,
         null=True,
-        help_text="The checksum of the archived document."
+        help_text=_("The checksum of the archived document.")
     )
 
     created = models.DateTimeField(
+        _("created"),
         default=timezone.now, db_index=True)
+
     modified = models.DateTimeField(
+        _("modified"),
         auto_now=True, editable=False, db_index=True)
 
     storage_type = models.CharField(
+        _("storage type"),
         max_length=11,
         choices=STORAGE_TYPES,
         default=STORAGE_TYPE_UNENCRYPTED,
@@ -185,27 +205,32 @@ class Document(models.Model):
     )
 
     added = models.DateTimeField(
+        _("added"),
         default=timezone.now, editable=False, db_index=True)
 
     filename = models.FilePathField(
+        _("filename"),
         max_length=1024,
         editable=False,
         default=None,
         null=True,
-        help_text="Current filename in storage"
+        help_text=_("Current filename in storage")
     )
 
     archive_serial_number = models.IntegerField(
+        _("archive serial number"),
         blank=True,
         null=True,
         unique=True,
         db_index=True,
-        help_text="The position of this document in your physical document "
-                  "archive."
+        help_text=_("The position of this document in your physical document "
+                    "archive.")
     )
 
     class Meta:
-        ordering = ("correspondent", "title")
+        ordering = ("-created",)
+        verbose_name = _("document")
+        verbose_name_plural = _("documents")
 
     def __str__(self):
         created = datetime.date.isoformat(self.created)
@@ -221,7 +246,7 @@ class Document(models.Model):
         else:
             fname = "{:07}{}".format(self.pk, self.file_type)
             if self.storage_type == self.STORAGE_TYPE_GPG:
-                fname += ".gpg"
+                fname += ".gpg"  # pragma: no cover
 
         return os.path.join(
             settings.ORIGINALS_DIR,
@@ -287,76 +312,116 @@ class Document(models.Model):
 class Log(models.Model):
 
     LEVELS = (
-        (logging.DEBUG, "Debugging"),
-        (logging.INFO, "Informational"),
-        (logging.WARNING, "Warning"),
-        (logging.ERROR, "Error"),
-        (logging.CRITICAL, "Critical"),
+        (logging.DEBUG, _("debug")),
+        (logging.INFO, _("information")),
+        (logging.WARNING, _("warning")),
+        (logging.ERROR, _("error")),
+        (logging.CRITICAL, _("critical")),
     )
 
-    group = models.UUIDField(blank=True, null=True)
-    message = models.TextField()
-    level = models.PositiveIntegerField(choices=LEVELS, default=logging.INFO)
-    created = models.DateTimeField(auto_now_add=True)
+    group = models.UUIDField(
+        _("group"),
+        blank=True, null=True)
+
+    message = models.TextField(_("message"))
+
+    level = models.PositiveIntegerField(
+        _("level"),
+        choices=LEVELS, default=logging.INFO)
+
+    created = models.DateTimeField(_("created"), auto_now_add=True)
 
     class Meta:
         ordering = ("-created",)
+        verbose_name = _("log")
+        verbose_name_plural = _("logs")
 
     def __str__(self):
         return self.message
 
 
+class SavedView(models.Model):
+
+    class Meta:
+
+        ordering = ("name",)
+        verbose_name = _("saved view")
+        verbose_name_plural = _("saved views")
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE,
+                             verbose_name=_("user"))
+    name = models.CharField(
+        _("name"),
+        max_length=128)
+
+    show_on_dashboard = models.BooleanField(
+        _("show on dashboard"),
+    )
+    show_in_sidebar = models.BooleanField(
+        _("show in sidebar"),
+    )
+
+    sort_field = models.CharField(
+        _("sort field"),
+        max_length=128)
+    sort_reverse = models.BooleanField(
+        _("sort reverse"),
+        default=False)
+
+
+class SavedViewFilterRule(models.Model):
+    RULE_TYPES = [
+        (0, _("title contains")),
+        (1, _("content contains")),
+        (2, _("ASN is")),
+        (3, _("correspondent is")),
+        (4, _("document type is")),
+        (5, _("is in inbox")),
+        (6, _("has tag")),
+        (7, _("has any tag")),
+        (8, _("created before")),
+        (9, _("created after")),
+        (10, _("created year is")),
+        (11, _("created month is")),
+        (12, _("created day is")),
+        (13, _("added before")),
+        (14, _("added after")),
+        (15, _("modified before")),
+        (16, _("modified after")),
+        (17, _("does not have tag")),
+    ]
+
+    saved_view = models.ForeignKey(
+        SavedView,
+        on_delete=models.CASCADE,
+        related_name="filter_rules",
+        verbose_name=_("saved view")
+    )
+
+    rule_type = models.PositiveIntegerField(
+        _("rule type"),
+        choices=RULE_TYPES)
+
+    value = models.CharField(
+        _("value"),
+        max_length=128,
+        blank=True,
+        null=True)
+
+    class Meta:
+        verbose_name = _("filter rule")
+        verbose_name_plural = _("filter rules")
+
+
 # TODO: why is this in the models file?
 class FileInfo:
 
-    # This epic regex *almost* worked for our needs, so I'm keeping it here for
-    # posterity, in the hopes that we might find a way to make it work one day.
-    ALMOST_REGEX = re.compile(
-        r"^((?P<date>\d\d\d\d\d\d\d\d\d\d\d\d\d\dZ){separator})?"
-        r"((?P<correspondent>{non_separated_word}+){separator})??"
-        r"(?P<title>{non_separated_word}+)"
-        r"({separator}(?P<tags>[a-z,0-9-]+))?"
-        r"\.(?P<extension>[a-zA-Z.-]+)$".format(
-            separator=r"\s+-\s+",
-            non_separated_word=r"([\w,. ]|([^\s]-))"
-        )
-    )
     REGEXES = OrderedDict([
-        ("created-correspondent-title-tags", re.compile(
-            r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*) - "
-            r"(?P<tags>[a-z0-9\-,]*)$",
-            flags=re.IGNORECASE
-        )),
-        ("created-title-tags", re.compile(
-            r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
-            r"(?P<title>.*) - "
-            r"(?P<tags>[a-z0-9\-,]*)$",
-            flags=re.IGNORECASE
-        )),
-        ("created-correspondent-title", re.compile(
-            r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*)$",
-            flags=re.IGNORECASE
-        )),
         ("created-title", re.compile(
             r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
             r"(?P<title>.*)$",
             flags=re.IGNORECASE
         )),
-        ("correspondent-title-tags", re.compile(
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*) - "
-            r"(?P<tags>[a-z0-9\-,]*)$",
-            flags=re.IGNORECASE
-        )),
-        ("correspondent-title", re.compile(
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*)?$",
-            flags=re.IGNORECASE
-        )),
         ("title", re.compile(
             r"(?P<title>.*)$",
             flags=re.IGNORECASE
@@ -379,28 +444,10 @@ class FileInfo:
         except ValueError:
             return None
 
-    @classmethod
-    def _get_correspondent(cls, name):
-        if not name:
-            return None
-        return Correspondent.objects.get_or_create(name=name, defaults={
-            "slug": slugify(name)
-        })[0]
-
     @classmethod
     def _get_title(cls, title):
         return title
 
-    @classmethod
-    def _get_tags(cls, tags):
-        r = []
-        for t in tags.split(","):
-            r.append(Tag.objects.get_or_create(
-                slug=slugify(t),
-                defaults={"name": t}
-            )[0])
-        return tuple(r)
-
     @classmethod
     def _mangle_property(cls, properties, name):
         if name in properties:
@@ -410,15 +457,6 @@ class FileInfo:
 
     @classmethod
     def from_filename(cls, filename):
-        """
-        We use a crude naming convention to make handling the correspondent,
-        title, and tags easier:
-          "<date> - <correspondent> - <title> - <tags>"
-          "<correspondent> - <title> - <tags>"
-          "<correspondent> - <title>"
-          "<title>"
-        """
-
         # Mutate filename in-place before parsing its components
         # by applying at most one of the configured transformations.
         for (pattern, repl) in settings.FILENAME_PARSE_TRANSFORMS:
@@ -449,7 +487,5 @@ class FileInfo:
             if m:
                 properties = m.groupdict()
                 cls._mangle_property(properties, "created")
-                cls._mangle_property(properties, "correspondent")
                 cls._mangle_property(properties, "title")
-                cls._mangle_property(properties, "tags")
                 return cls(**properties)

src/documents/parsers.py

@@ -117,6 +117,7 @@ def run_convert(input_file,
                 trim=False,
                 type=None,
                 depth=None,
+                auto_orient=False,
                 extra=None,
                 logging_group=None):
 
@@ -134,6 +135,7 @@ def run_convert(input_file,
     args += ['-trim'] if trim else []
     args += ['-type', str(type)] if type else []
     args += ['-depth', str(depth)] if depth else []
+    args += ['-auto-orient'] if auto_orient else []
     args += [input_file, output_file]
 
     logger.debug("Execute: " + " ".join(args), extra={'group': logging_group})
@@ -142,6 +144,53 @@ def run_convert(input_file,
         raise ParseError("Convert failed at {}".format(args))
 
 
+def make_thumbnail_from_pdf(in_path, temp_dir, logging_group=None):
+    """
+    The thumbnail of a PDF is just a 500px wide image of the first page.
+    """
+    out_path = os.path.join(temp_dir, "convert.png")
+
+    # Run convert to get a decent thumbnail
+    try:
+        run_convert(density=300,
+                    scale="500x5000>",
+                    alpha="remove",
+                    strip=True,
+                    trim=False,
+                    auto_orient=True,
+                    input_file="{}[0]".format(in_path),
+                    output_file=out_path,
+                    logging_group=logging_group)
+    except ParseError:
+        # if convert fails, fall back to extracting
+        # the first PDF page as a PNG using Ghostscript
+        logger.warning(
+            "Thumbnail generation with ImageMagick failed, falling back "
+            "to ghostscript. Check your /etc/ImageMagick-x/policy.xml!",
+            extra={'group': logging_group}
+        )
+        gs_out_path = os.path.join(temp_dir, "gs_out.png")
+        cmd = [settings.GS_BINARY,
+               "-q",
+               "-sDEVICE=pngalpha",
+               "-o", gs_out_path,
+               in_path]
+        if not subprocess.Popen(cmd).wait() == 0:
+            raise ParseError("Thumbnail (gs) failed at {}".format(cmd))
+        # then run convert on the output from gs
+        run_convert(density=300,
+                    scale="500x5000>",
+                    alpha="remove",
+                    strip=True,
+                    trim=False,
+                    auto_orient=True,
+                    input_file=gs_out_path,
+                    output_file=out_path,
+                    logging_group=logging_group)
+
+    return out_path
+
+
 def parse_date(filename, text):
     """
     Returns the date of the document.
@@ -163,8 +212,6 @@ def parse_date(filename, text):
 
     date = None
 
-    next_year = timezone.now().year + 5  # Arbitrary 5 year future limit
-
     # if filename date parsing is enabled, search there first:
     if settings.FILENAME_DATE_ORDER:
         for m in re.finditer(DATE_REGEX, filename):
@@ -176,7 +223,7 @@ def parse_date(filename, text):
                 # Skip all matches that do not parse to a proper date
                 continue
 
-            if date is not None and next_year > date.year > 1900:
+            if date and date.year > 1900 and date <= timezone.now():
                 return date
 
     # Iterate through all regex matches in text and try to parse the date
@@ -189,7 +236,7 @@ def parse_date(filename, text):
             # Skip all matches that do not parse to a proper date
             continue
 
-        if date is not None and next_year > date.year > 1900:
+        if date and date.year > 1900 and date <= timezone.now():
             break
         else:
             date = None
@@ -210,6 +257,7 @@ class DocumentParser(LoggingMixin):
     def __init__(self, logging_group, progress_callback):
         super().__init__()
         self.logging_group = logging_group
+        os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
         self.tempdir = tempfile.mkdtemp(
             prefix="paperless-", dir=settings.SCRATCH_DIR)
 
@@ -218,7 +266,10 @@ class DocumentParser(LoggingMixin):
         self.date = None
         self.progress_callback = progress_callback
 
-    def parse(self, document_path, mime_type):
+    def extract_metadata(self, document_path, mime_type):
+        return []
+
+    def parse(self, document_path, mime_type, file_name=None):
         raise NotImplementedError()
 
     def get_archive_path(self):

src/documents/sanity_checker.py

@@ -46,6 +46,10 @@ def check_sanity():
         for f in files:
             present_files.append(os.path.normpath(os.path.join(root, f)))
 
+    lockfile = os.path.normpath(settings.MEDIA_LOCK)
+    if lockfile in present_files:
+        present_files.remove(lockfile)
+
     for doc in Document.objects.all():
         # Check sanity of the thumbnail
         if not os.path.isfile(doc.thumbnail_path):

src/documents/serialisers.py

@@ -1,17 +1,46 @@
 import magic
-from pathvalidate import validate_filename, ValidationError
+from django.utils.text import slugify
 from rest_framework import serializers
+from rest_framework.fields import SerializerMethodField
 
-from .models import Correspondent, Tag, Document, Log, DocumentType
+from . import bulk_edit
+from .models import Correspondent, Tag, Document, Log, DocumentType, \
+    SavedView, SavedViewFilterRule
 from .parsers import is_mime_type_supported
 
 
-class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
+# https://www.django-rest-framework.org/api-guide/serializers/#example
+class DynamicFieldsModelSerializer(serializers.ModelSerializer):
+    """
+    A ModelSerializer that takes an additional `fields` argument that
+    controls which fields should be displayed.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # Don't pass the 'fields' arg up to the superclass
+        fields = kwargs.pop('fields', None)
+
+        # Instantiate the superclass normally
+        super(DynamicFieldsModelSerializer, self).__init__(*args, **kwargs)
+
+        if fields is not None:
+            # Drop any fields that are not specified in the `fields` argument.
+            allowed = set(fields)
+            existing = set(self.fields)
+            for field_name in existing - allowed:
+                self.fields.pop(field_name)
+
+
+class CorrespondentSerializer(serializers.ModelSerializer):
 
     document_count = serializers.IntegerField(read_only=True)
 
     last_correspondence = serializers.DateTimeField(read_only=True)
 
+    def get_slug(self, obj):
+        return slugify(obj.name)
+    slug = SerializerMethodField()
+
     class Meta:
         model = Correspondent
         fields = (
@@ -26,10 +55,14 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
         )
 
 
-class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
+class DocumentTypeSerializer(serializers.ModelSerializer):
 
     document_count = serializers.IntegerField(read_only=True)
 
+    def get_slug(self, obj):
+        return slugify(obj.name)
+    slug = SerializerMethodField()
+
     class Meta:
         model = DocumentType
         fields = (
@@ -43,10 +76,14 @@ class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
         )
 
 
-class TagSerializer(serializers.HyperlinkedModelSerializer):
+class TagSerializer(serializers.ModelSerializer):
 
     document_count = serializers.IntegerField(read_only=True)
 
+    def get_slug(self, obj):
+        return slugify(obj.name)
+    slug = SerializerMethodField()
+
     class Meta:
         model = Tag
         fields = (
@@ -77,12 +114,24 @@ class DocumentTypeField(serializers.PrimaryKeyRelatedField):
         return DocumentType.objects.all()
 
 
-class DocumentSerializer(serializers.ModelSerializer):
+class DocumentSerializer(DynamicFieldsModelSerializer):
 
     correspondent = CorrespondentField(allow_null=True)
     tags = TagsField(many=True)
     document_type = DocumentTypeField(allow_null=True)
 
+    original_file_name = SerializerMethodField()
+    archived_file_name = SerializerMethodField()
+
+    def get_original_file_name(self, obj):
+        return obj.get_public_filename()
+
+    def get_archived_file_name(self, obj):
+        if obj.archive_checksum:
+            return obj.get_public_filename(archive=True)
+        else:
+            return None
+
     class Meta:
         model = Document
         depth = 1
@@ -96,7 +145,9 @@ class DocumentSerializer(serializers.ModelSerializer):
             "created",
             "modified",
             "added",
-            "archive_serial_number"
+            "archive_serial_number",
+            "original_file_name",
+            "archived_file_name",
         )
 
 
@@ -113,6 +164,177 @@ class LogSerializer(serializers.ModelSerializer):
         )
 
 
+class SavedViewFilterRuleSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = SavedViewFilterRule
+        fields = ["rule_type", "value"]
+
+
+class SavedViewSerializer(serializers.ModelSerializer):
+
+    filter_rules = SavedViewFilterRuleSerializer(many=True)
+
+    class Meta:
+        model = SavedView
+        depth = 1
+        fields = ["id", "name", "show_on_dashboard", "show_in_sidebar",
+                  "sort_field", "sort_reverse", "filter_rules"]
+
+    def update(self, instance, validated_data):
+        if 'filter_rules' in validated_data:
+            rules_data = validated_data.pop('filter_rules')
+        else:
+            rules_data = None
+        super(SavedViewSerializer, self).update(instance, validated_data)
+        if rules_data is not None:
+            SavedViewFilterRule.objects.filter(saved_view=instance).delete()
+            for rule_data in rules_data:
+                SavedViewFilterRule.objects.create(
+                    saved_view=instance, **rule_data)
+        return instance
+
+    def create(self, validated_data):
+        rules_data = validated_data.pop('filter_rules')
+        saved_view = SavedView.objects.create(**validated_data)
+        for rule_data in rules_data:
+            SavedViewFilterRule.objects.create(
+                saved_view=saved_view, **rule_data)
+        return saved_view
+
+
+class BulkEditSerializer(serializers.Serializer):
+
+    documents = serializers.ListField(
+        child=serializers.IntegerField(),
+        label="Documents",
+        write_only=True
+    )
+
+    method = serializers.ChoiceField(
+        choices=[
+            "set_correspondent",
+            "set_document_type",
+            "add_tag",
+            "remove_tag",
+            "modify_tags",
+            "delete"
+        ],
+        label="Method",
+        write_only=True,
+    )
+
+    parameters = serializers.DictField(allow_empty=True)
+
+    def _validate_document_id_list(self, documents, name="documents"):
+        if not type(documents) == list:
+            raise serializers.ValidationError(f"{name} must be a list")
+        if not all([type(i) == int for i in documents]):
+            raise serializers.ValidationError(
+                f"{name} must be a list of integers")
+        count = Document.objects.filter(id__in=documents).count()
+        if not count == len(documents):
+            raise serializers.ValidationError(
+                f"Some documents in {name} don't exist or were "
+                f"specified twice.")
+
+    def _validate_tag_id_list(self, tags, name="tags"):
+        if not type(tags) == list:
+            raise serializers.ValidationError(f"{name} must be a list")
+        if not all([type(i) == int for i in tags]):
+            raise serializers.ValidationError(
+                f"{name} must be a list of integers")
+        count = Tag.objects.filter(id__in=tags).count()
+        if not count == len(tags):
+            raise serializers.ValidationError(
+                f"Some tags in {name} don't exist or were specified twice.")
+
+    def validate_documents(self, documents):
+        self._validate_document_id_list(documents)
+        return documents
+
+    def validate_method(self, method):
+        if method == "set_correspondent":
+            return bulk_edit.set_correspondent
+        elif method == "set_document_type":
+            return bulk_edit.set_document_type
+        elif method == "add_tag":
+            return bulk_edit.add_tag
+        elif method == "remove_tag":
+            return bulk_edit.remove_tag
+        elif method == "modify_tags":
+            return bulk_edit.modify_tags
+        elif method == "delete":
+            return bulk_edit.delete
+        else:
+            raise serializers.ValidationError("Unsupported method.")
+
+    def _validate_parameters_tags(self, parameters):
+        if 'tag' in parameters:
+            tag_id = parameters['tag']
+            try:
+                Tag.objects.get(id=tag_id)
+            except Tag.DoesNotExist:
+                raise serializers.ValidationError("Tag does not exist")
+        else:
+            raise serializers.ValidationError("tag not specified")
+
+    def _validate_parameters_document_type(self, parameters):
+        if 'document_type' in parameters:
+            document_type_id = parameters['document_type']
+            if document_type_id is None:
+                # None is ok
+                return
+            try:
+                DocumentType.objects.get(id=document_type_id)
+            except DocumentType.DoesNotExist:
+                raise serializers.ValidationError(
+                    "Document type does not exist")
+        else:
+            raise serializers.ValidationError("document_type not specified")
+
+    def _validate_parameters_correspondent(self, parameters):
+        if 'correspondent' in parameters:
+            correspondent_id = parameters['correspondent']
+            if correspondent_id is None:
+                return
+            try:
+                Correspondent.objects.get(id=correspondent_id)
+            except Correspondent.DoesNotExist:
+                raise serializers.ValidationError(
+                    "Correspondent does not exist")
+        else:
+            raise serializers.ValidationError("correspondent not specified")
+
+    def _validate_parameters_modify_tags(self, parameters):
+        if "add_tags" in parameters:
+            self._validate_tag_id_list(parameters['add_tags'], "add_tags")
+        else:
+            raise serializers.ValidationError("add_tags not specified")
+
+        if "remove_tags" in parameters:
+            self._validate_tag_id_list(parameters['remove_tags'],
+                                       "remove_tags")
+        else:
+            raise serializers.ValidationError("remove_tags not specified")
+
+    def validate(self, attrs):
+
+        method = attrs['method']
+        parameters = attrs['parameters']
+
+        if method == bulk_edit.set_correspondent:
+            self._validate_parameters_correspondent(parameters)
+        elif method == bulk_edit.set_document_type:
+            self._validate_parameters_document_type(parameters)
+        elif method == bulk_edit.add_tag or method == bulk_edit.remove_tag:
+            self._validate_parameters_tags(parameters)
+        elif method == bulk_edit.modify_tags:
+            self._validate_parameters_modify_tags(parameters)
+
+        return attrs
+
+
 class PostDocumentSerializer(serializers.Serializer):
 
     document = serializers.FileField(
@@ -151,12 +373,6 @@ class PostDocumentSerializer(serializers.Serializer):
     )
 
     def validate_document(self, document):
-
-        try:
-            validate_filename(document.name)
-        except ValidationError:
-            raise serializers.ValidationError("Invalid filename.")
-
         document_data = document.file.read()
         mime_type = magic.from_buffer(document_data, mime=True)
 
@@ -190,3 +406,11 @@ class PostDocumentSerializer(serializers.Serializer):
             return [tag.id for tag in tags]
         else:
             return None
+
+
+class SelectionDataSerializer(serializers.Serializer):
+
+    documents = serializers.ListField(
+        required=True,
+        child=serializers.IntegerField()
+    )

src/documents/signals/handlers.py

@@ -7,13 +7,15 @@ from django.contrib.admin.models import ADDITION, LogEntry
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.db import models, DatabaseError
+from django.db.models import Q
 from django.dispatch import receiver
 from django.utils import timezone
-from rest_framework.reverse import reverse
+from filelock import FileLock
 
 from .. import index, matching
-from ..file_handling import delete_empty_directories, generate_filename, \
-    create_source_path_directory, archive_name_from_filename
+from ..file_handling import delete_empty_directories, \
+    create_source_path_directory, archive_name_from_filename, \
+    generate_unique_filename
 from ..models import Document, Tag
 
 
@@ -119,11 +121,14 @@ def set_tags(sender,
              classifier=None,
              replace=False,
              **kwargs):
+
     if replace:
-        document.tags.clear()
-        current_tags = set([])
-    else:
-        current_tags = set(document.tags.all())
+        Document.tags.through.objects.filter(document=document).exclude(
+            Q(tag__is_inbox_tag=True)).exclude(
+            Q(tag__match="") & ~Q(tag__matching_algorithm=Tag.MATCH_AUTO)
+        ).delete()
+
+    current_tags = set(document.tags.all())
 
     matched_tags = matching.match_tags(document.content, classifier)
 
@@ -134,64 +139,39 @@ def set_tags(sender,
 
     message = 'Tagging "{}" with "{}"'
     logger(
-        message.format(document, ", ".join([t.slug for t in relevant_tags])),
+        message.format(document, ", ".join([t.name for t in relevant_tags])),
         logging_group
     )
 
     document.tags.add(*relevant_tags)
 
 
-def run_pre_consume_script(sender, filename, **kwargs):
-
-    if not settings.PRE_CONSUME_SCRIPT:
-        return
-
-    Popen((settings.PRE_CONSUME_SCRIPT, filename)).wait()
-
-
-def run_post_consume_script(sender, document, **kwargs):
-
-    if not settings.POST_CONSUME_SCRIPT:
-        return
-
-    Popen((
-        settings.POST_CONSUME_SCRIPT,
-        str(document.pk),
-        document.get_public_filename(),
-        os.path.normpath(document.source_path),
-        os.path.normpath(document.thumbnail_path),
-        reverse("document-download", kwargs={"pk": document.pk}),
-        reverse("document-thumb", kwargs={"pk": document.pk}),
-        str(document.correspondent),
-        str(",".join(document.tags.all().values_list("slug", flat=True)))
-    )).wait()
-
-
 @receiver(models.signals.post_delete, sender=Document)
 def cleanup_document_deletion(sender, instance, using, **kwargs):
-    for f in (instance.source_path,
-              instance.archive_path,
-              instance.thumbnail_path):
-        if os.path.isfile(f):
-            try:
-                os.unlink(f)
-                logging.getLogger(__name__).debug(
-                    f"Deleted file {f}.")
-            except OSError as e:
-                logging.getLogger(__name__).warning(
-                    f"While deleting document {str(instance)}, the file "
-                    f"{f} could not be deleted: {e}"
-                )
+    with FileLock(settings.MEDIA_LOCK):
+        for f in (instance.source_path,
+                  instance.archive_path,
+                  instance.thumbnail_path):
+            if os.path.isfile(f):
+                try:
+                    os.unlink(f)
+                    logging.getLogger(__name__).debug(
+                        f"Deleted file {f}.")
+                except OSError as e:
+                    logging.getLogger(__name__).warning(
+                        f"While deleting document {str(instance)}, the file "
+                        f"{f} could not be deleted: {e}"
+                    )
 
-    delete_empty_directories(
-        os.path.dirname(instance.source_path),
-        root=settings.ORIGINALS_DIR
-    )
+        delete_empty_directories(
+            os.path.dirname(instance.source_path),
+            root=settings.ORIGINALS_DIR
+        )
 
-    delete_empty_directories(
-        os.path.dirname(instance.archive_path),
-        root=settings.ARCHIVE_DIR
-    )
+        delete_empty_directories(
+            os.path.dirname(instance.archive_path),
+            root=settings.ARCHIVE_DIR
+        )
 
 
 def validate_move(instance, old_path, new_path):
@@ -226,81 +206,87 @@ def update_filename_and_move_files(sender, instance, **kwargs):
         # This will in turn cause this logic to move the file where it belongs.
         return
 
-    old_filename = instance.filename
-    new_filename = generate_filename(instance)
+    with FileLock(settings.MEDIA_LOCK):
+        old_filename = instance.filename
+        new_filename = generate_unique_filename(
+            instance, settings.ORIGINALS_DIR)
 
-    if new_filename == instance.filename:
-        # Don't do anything if its the same.
-        return
-
-    old_source_path = instance.source_path
-    new_source_path = os.path.join(settings.ORIGINALS_DIR, new_filename)
-
-    if not validate_move(instance, old_source_path, new_source_path):
-        return
-
-    # archive files are optional, archive checksum tells us if we have one,
-    # since this is None for documents without archived files.
-    if instance.archive_checksum:
-        new_archive_filename = archive_name_from_filename(new_filename)
-        old_archive_path = instance.archive_path
-        new_archive_path = os.path.join(settings.ARCHIVE_DIR,
-                                        new_archive_filename)
-
-        if not validate_move(instance, old_archive_path, new_archive_path):
+        if new_filename == instance.filename:
+            # Don't do anything if its the same.
             return
 
-        create_source_path_directory(new_archive_path)
-    else:
-        old_archive_path = None
-        new_archive_path = None
+        old_source_path = instance.source_path
+        new_source_path = os.path.join(settings.ORIGINALS_DIR, new_filename)
 
-    create_source_path_directory(new_source_path)
+        if not validate_move(instance, old_source_path, new_source_path):
+            return
 
-    try:
-        os.rename(old_source_path, new_source_path)
+        # archive files are optional, archive checksum tells us if we have one,
+        # since this is None for documents without archived files.
         if instance.archive_checksum:
-            os.rename(old_archive_path, new_archive_path)
-        instance.filename = new_filename
-        # Don't save here to prevent infinite recursion.
-        Document.objects.filter(pk=instance.pk).update(filename=new_filename)
+            new_archive_filename = archive_name_from_filename(new_filename)
+            old_archive_path = instance.archive_path
+            new_archive_path = os.path.join(settings.ARCHIVE_DIR,
+                                            new_archive_filename)
 
-        logging.getLogger(__name__).debug(
-            f"Moved file {old_source_path} to {new_source_path}.")
+            if not validate_move(instance, old_archive_path, new_archive_path):
+                return
 
-        if instance.archive_checksum:
-            logging.getLogger(__name__).debug(
-                f"Moved file {old_archive_path} to {new_archive_path}.")
+            create_source_path_directory(new_archive_path)
+        else:
+            old_archive_path = None
+            new_archive_path = None
+
+        create_source_path_directory(new_source_path)
 
-    except OSError as e:
-        instance.filename = old_filename
-        # this happens when we can't move a file. If that's the case for the
-        # archive file, we try our best to revert the changes.
         try:
+            os.rename(old_source_path, new_source_path)
+            if instance.archive_checksum:
+                os.rename(old_archive_path, new_archive_path)
+            instance.filename = new_filename
+
+            # Don't save() here to prevent infinite recursion.
+            Document.objects.filter(pk=instance.pk).update(
+                filename=new_filename)
+
+        except OSError as e:
+            instance.filename = old_filename
+            # this happens when we can't move a file. If that's the case for
+            # the archive file, we try our best to revert the changes.
+            # no need to save the instance, the update() has not happened yet.
+            try:
+                os.rename(new_source_path, old_source_path)
+                os.rename(new_archive_path, old_archive_path)
+            except Exception as e:
+                # This is fine, since:
+                # A: if we managed to move source from A to B, we will also
+                #  manage to move it from B to A. If not, we have a serious
+                #  issue that's going to get caught by the santiy checker.
+                #  All files remain in place and will never be overwritten,
+                #  so this is not the end of the world.
+                # B: if moving the orignal file failed, nothing has changed
+                #  anyway.
+                pass
+        except DatabaseError as e:
+            # this happens after moving files, so move them back into place.
+            # since moving them once succeeded, it's very likely going to
+            # succeed again.
             os.rename(new_source_path, old_source_path)
-            os.rename(new_archive_path, old_archive_path)
-        except Exception as e:
-            # This is fine, since:
-            # A: if we managed to move source from A to B, we will also manage
-            #  to move it from B to A. If not, we have a serious issue
-            #  that's going to get caught by the santiy checker.
-            #  all files remain in place and will never be overwritten,
-            #  so this is not the end of the world.
-            # B: if moving the orignal file failed, nothing has changed anyway.
-            pass
-    except DatabaseError as e:
-        os.rename(new_source_path, old_source_path)
-        if instance.archive_checksum:
-            os.rename(new_archive_path, old_archive_path)
-        instance.filename = old_filename
+            if instance.archive_checksum:
+                os.rename(new_archive_path, old_archive_path)
+            instance.filename = old_filename
+            # again, no need to save the instance, since the actual update()
+            # operation failed.
 
-    if not os.path.isfile(old_source_path):
-        delete_empty_directories(os.path.dirname(old_source_path),
-                                 root=settings.ORIGINALS_DIR)
+        # finally, remove any empty sub folders. This will do nothing if
+        # something has failed above.
+        if not os.path.isfile(old_source_path):
+            delete_empty_directories(os.path.dirname(old_source_path),
+                                     root=settings.ORIGINALS_DIR)
 
-    if old_archive_path and not os.path.isfile(old_archive_path):
-        delete_empty_directories(os.path.dirname(old_archive_path),
-                                 root=settings.ARCHIVE_DIR)
+        if old_archive_path and not os.path.isfile(old_archive_path):
+            delete_empty_directories(os.path.dirname(old_archive_path),
+                                     root=settings.ARCHIVE_DIR)
 
 
 def set_log_entry(sender, document=None, logging_group=None, **kwargs):

src/documents/tasks.py

@@ -1,6 +1,8 @@
 import logging
 
+import tqdm
 from django.conf import settings
+from django.db.models.signals import post_save
 from whoosh.writing import AsyncWriter
 
 from documents import index, sanity_checker
@@ -23,7 +25,7 @@ def index_reindex():
     ix = index.open_index(recreate=True)
 
     with AsyncWriter(ix) as writer:
-        for document in documents:
+        for document in tqdm.tqdm(documents):
             index.update_document(writer, document)
 
 
@@ -33,9 +35,9 @@ def train_classifier():
     try:
         # load the classifier, since we might not have to train it again.
         classifier.reload()
-    except (FileNotFoundError, IncompatibleClassifierVersionError):
+    except (OSError, EOFError, IncompatibleClassifierVersionError):
         # This is what we're going to fix here.
-        pass
+        classifier = DocumentClassifier()
 
     try:
         if classifier.train():
@@ -86,3 +88,16 @@ def sanity_check():
         raise SanityFailedError(messages)
     else:
         return "No issues detected."
+
+
+def bulk_update_documents(document_ids):
+    documents = Document.objects.filter(id__in=document_ids)
+
+    ix = index.open_index()
+
+    for doc in documents:
+        post_save.send(Document, instance=doc, created=False)
+
+    with AsyncWriter(ix) as writer:
+        for doc in documents:
+            index.update_document(writer, doc)

src/documents/templates/index.html

@@ -5,15 +5,20 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <title>PaperlessUi</title>
+  <title>Paperless-ng</title>
   <base href="/">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+	<meta name="username" content="{{username}}">
+	<meta name="full_name" content="{{full_name}}">
+	<meta name="cookie_prefix" content="{{cookie_prefix}}">
   <link rel="icon" type="image/x-icon" href="favicon.ico">
-<link rel="stylesheet" href="{% static 'frontend/styles.css' %}"></head>
+  <link rel="manifest" href="{% static webmanifest %}">
+	<link rel="stylesheet" href="{% static styles_css %}">
+</head>
 <body>
   <app-root>Loading...</app-root>
-	<script src="{% static 'frontend/runtime.js' %}" defer></script>
-	<script src="{% static 'frontend/polyfills.js' %}" defer></script>
-	<script src="{% static 'frontend/main.js' %}" defer></script>
+	<script src="{% static runtime_js %}" defer></script>
+	<script src="{% static polyfills_js %}" defer></script>
+	<script src="{% static main_js %}" defer></script>
 </body>
 </html>

src/documents/tests/test_admin.py

@@ -0,0 +1,57 @@
+from unittest import mock
+
+from django.contrib.admin.sites import AdminSite
+from django.test import TestCase
+from django.utils import timezone
+
+from documents.admin import DocumentAdmin
+from documents.models import Document, Tag
+
+
+class TestDocumentAdmin(TestCase):
+
+    def setUp(self) -> None:
+        self.doc_admin = DocumentAdmin(model=Document, admin_site=AdminSite())
+
+    @mock.patch("documents.admin.index.add_or_update_document")
+    def test_save_model(self, m):
+        doc = Document.objects.create(title="test")
+        doc.title = "new title"
+        self.doc_admin.save_model(None, doc, None, None)
+        self.assertEqual(Document.objects.get(id=doc.id).title, "new title")
+        m.assert_called_once()
+
+    def test_tags(self):
+        doc = Document.objects.create(title="test")
+        doc.tags.create(name="t1")
+        doc.tags.create(name="t2")
+
+        self.assertEqual(self.doc_admin.tags_(doc), "<span >t1, </span><span >t2, </span>")
+
+    def test_tags_empty(self):
+        doc = Document.objects.create(title="test")
+
+        self.assertEqual(self.doc_admin.tags_(doc), "")
+
+    @mock.patch("documents.admin.index.remove_document")
+    def test_delete_model(self, m):
+        doc = Document.objects.create(title="test")
+        self.doc_admin.delete_model(None, doc)
+        self.assertRaises(Document.DoesNotExist, Document.objects.get, id=doc.id)
+        m.assert_called_once()
+
+    @mock.patch("documents.admin.index.remove_document")
+    def test_delete_queryset(self, m):
+        for i in range(42):
+            Document.objects.create(title="Many documents with the same title", checksum=f"{i:02}")
+
+        self.assertEqual(Document.objects.count(), 42)
+
+        self.doc_admin.delete_queryset(None, Document.objects.all())
+
+        self.assertEqual(m.call_count, 42)
+        self.assertEqual(Document.objects.count(), 0)
+
+    def test_created(self):
+        doc = Document.objects.create(title="test", created=timezone.datetime(2020, 4, 12))
+        self.assertEqual(self.doc_admin.created_(doc), "2020-04-12")

src/documents/tests/test_api.py

@@ -1,14 +1,15 @@
+import json
 import os
+import shutil
 import tempfile
 from unittest import mock
 
 from django.contrib.auth.models import User
-from pathvalidate import ValidationError
 from rest_framework.test import APITestCase
 from whoosh.writing import AsyncWriter
 
-from documents import index
-from documents.models import Document, Correspondent, DocumentType, Tag
+from documents import index, bulk_edit
+from documents.models import Document, Correspondent, DocumentType, Tag, SavedView
 from documents.tests.utils import DirectoriesMixin
 
 
@@ -17,8 +18,8 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
     def setUp(self):
         super(TestDocumentApi, self).setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
-        self.client.force_login(user=user)
+        self.user = User.objects.create_superuser(username="temp_admin")
+        self.client.force_login(user=self.user)
 
     def testDocuments(self):
 
@@ -63,6 +64,58 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
 
         self.assertEqual(len(Document.objects.all()), 0)
 
+    def test_document_fields(self):
+        c = Correspondent.objects.create(name="c", pk=41)
+        dt = DocumentType.objects.create(name="dt", pk=63)
+        tag = Tag.objects.create(name="t", pk=85)
+        doc = Document.objects.create(title="WOW", content="the content", correspondent=c, document_type=dt, checksum="123", mime_type="application/pdf")
+
+        response = self.client.get("/api/documents/", format='json')
+        self.assertEqual(response.status_code, 200)
+        results_full = response.data['results']
+        self.assertTrue("content" in results_full[0])
+        self.assertTrue("id" in results_full[0])
+
+        response = self.client.get("/api/documents/?fields=id", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertFalse("content" in results[0])
+        self.assertTrue("id" in results[0])
+        self.assertEqual(len(results[0]), 1)
+
+        response = self.client.get("/api/documents/?fields=content", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertTrue("content" in results[0])
+        self.assertFalse("id" in results[0])
+        self.assertEqual(len(results[0]), 1)
+
+        response = self.client.get("/api/documents/?fields=id,content", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertTrue("content" in results[0])
+        self.assertTrue("id" in results[0])
+        self.assertEqual(len(results[0]), 2)
+
+        response = self.client.get("/api/documents/?fields=id,conteasdnt", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertFalse("content" in results[0])
+        self.assertTrue("id" in results[0])
+        self.assertEqual(len(results[0]), 1)
+
+        response = self.client.get("/api/documents/?fields=", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertEqual(results_full, results)
+
+        response = self.client.get("/api/documents/?fields=dgfhs", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertEqual(len(results[0]), 0)
+
+
+
     def test_document_actions(self):
 
         _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir)
@@ -169,15 +222,13 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, 200)
         results = response.data['results']
         self.assertEqual(len(results), 2)
-        self.assertEqual(results[0]['id'], doc2.id)
-        self.assertEqual(results[1]['id'], doc3.id)
+        self.assertCountEqual([results[0]['id'], results[1]['id']], [doc2.id, doc3.id])
 
         response = self.client.get("/api/documents/?tags__id__in={},{}".format(tag_inbox.id, tag_3.id))
         self.assertEqual(response.status_code, 200)
         results = response.data['results']
         self.assertEqual(len(results), 2)
-        self.assertEqual(results[0]['id'], doc1.id)
-        self.assertEqual(results[1]['id'], doc3.id)
+        self.assertCountEqual([results[0]['id'], results[1]['id']], [doc1.id, doc3.id])
 
         response = self.client.get("/api/documents/?tags__id__all={},{}".format(tag_2.id, tag_3.id))
         self.assertEqual(response.status_code, 200)
@@ -195,6 +246,23 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         results = response.data['results']
         self.assertEqual(len(results), 3)
 
+        response = self.client.get("/api/documents/?tags__id__none={}".format(tag_3.id))
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertEqual(len(results), 2)
+        self.assertCountEqual([results[0]['id'], results[1]['id']], [doc1.id, doc2.id])
+
+        response = self.client.get("/api/documents/?tags__id__none={},{}".format(tag_3.id, tag_2.id))
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]['id'], doc1.id)
+
+        response = self.client.get("/api/documents/?tags__id__none={},{}".format(tag_2.id, tag_inbox.id))
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertEqual(len(results), 0)
+
     def test_search_no_query(self):
         response = self.client.get("/api/search/")
         results = response.data['results']
@@ -336,6 +404,25 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
 
         self.assertEqual(correction, None)
 
+    def test_search_more_like(self):
+        d1=Document.objects.create(title="invoice", content="the thing i bought at a shop and paid with bank account", checksum="A", pk=1)
+        d2=Document.objects.create(title="bank statement 1", content="things i paid for in august", pk=2, checksum="B")
+        d3=Document.objects.create(title="bank statement 3", content="things i paid for in september", pk=3, checksum="C")
+        with AsyncWriter(index.open_index()) as writer:
+            index.update_document(writer, d1)
+            index.update_document(writer, d2)
+            index.update_document(writer, d3)
+
+        response = self.client.get(f"/api/search/?more_like={d2.id}")
+
+        self.assertEqual(response.status_code, 200)
+
+        results = response.data['results']
+
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results[0]['id'], d3.id)
+        self.assertEqual(results[1]['id'], d1.id)
+
     def test_statistics(self):
 
         doc1 = Document.objects.create(title="none1", checksum="A")
@@ -384,16 +471,6 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, 400)
         m.assert_not_called()
 
-    @mock.patch("documents.views.async_task")
-    @mock.patch("documents.serialisers.validate_filename")
-    def test_upload_invalid_filename(self, validate_filename, async_task):
-        validate_filename.side_effect = ValidationError()
-        with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f:
-            response = self.client.post("/api/documents/post_document/", {"document": f})
-        self.assertEqual(response.status_code, 400)
-
-        async_task.assert_not_called()
-
     @mock.patch("documents.views.async_task")
     def test_upload_with_title(self, async_task):
         with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f:
@@ -475,3 +552,450 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, 400)
 
         async_task.assert_not_called()
+
+    def test_get_metadata(self):
+        doc = Document.objects.create(title="test", filename="file.pdf", mime_type="image/png", archive_checksum="A")
+
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000001.png"), doc.source_path)
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), doc.archive_path)
+
+        response = self.client.get(f"/api/documents/{doc.pk}/metadata/")
+        self.assertEqual(response.status_code, 200)
+
+        meta = response.data
+
+        self.assertEqual(meta['original_mime_type'], "image/png")
+        self.assertTrue(meta['has_archive_version'])
+        self.assertEqual(len(meta['original_metadata']), 0)
+        self.assertGreater(len(meta['archive_metadata']), 0)
+
+    def test_get_metadata_no_archive(self):
+        doc = Document.objects.create(title="test", filename="file.pdf", mime_type="application/pdf")
+
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), doc.source_path)
+
+        response = self.client.get(f"/api/documents/{doc.pk}/metadata/")
+        self.assertEqual(response.status_code, 200)
+
+        meta = response.data
+
+        self.assertEqual(meta['original_mime_type'], "application/pdf")
+        self.assertFalse(meta['has_archive_version'])
+        self.assertGreater(len(meta['original_metadata']), 0)
+        self.assertIsNone(meta['archive_metadata'])
+
+    def test_saved_views(self):
+        u1 = User.objects.create_user("user1")
+        u2 = User.objects.create_user("user2")
+
+        v1 = SavedView.objects.create(user=u1, name="test1", sort_field="", show_on_dashboard=False, show_in_sidebar=False)
+        v2 = SavedView.objects.create(user=u2, name="test2", sort_field="", show_on_dashboard=False, show_in_sidebar=False)
+        v3 = SavedView.objects.create(user=u2, name="test3", sort_field="", show_on_dashboard=False, show_in_sidebar=False)
+
+        response = self.client.get("/api/saved_views/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 0)
+
+        self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 404)
+
+        self.client.force_login(user=u1)
+
+        response = self.client.get("/api/saved_views/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 1)
+
+        self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 200)
+
+        self.client.force_login(user=u2)
+
+        response = self.client.get("/api/saved_views/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 2)
+
+        self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 404)
+
+    def test_create_update_patch(self):
+
+        u1 = User.objects.create_user("user1")
+
+        view = {
+            "name": "test",
+            "show_on_dashboard": True,
+            "show_in_sidebar": True,
+            "sort_field": "created2",
+            "filter_rules": [
+                {
+                    "rule_type": 4,
+                    "value": "test"
+                }
+            ]
+        }
+
+        response = self.client.post("/api/saved_views/", view, format='json')
+        self.assertEqual(response.status_code, 201)
+
+        v1 = SavedView.objects.get(name="test")
+        self.assertEqual(v1.sort_field, "created2")
+        self.assertEqual(v1.filter_rules.count(), 1)
+        self.assertEqual(v1.user, self.user)
+
+        response = self.client.patch(f"/api/saved_views/{v1.id}/", {
+            "show_in_sidebar": False
+        }, format='json')
+
+        v1 = SavedView.objects.get(id=v1.id)
+        self.assertEqual(response.status_code, 200)
+        self.assertFalse(v1.show_in_sidebar)
+        self.assertEqual(v1.filter_rules.count(), 1)
+
+        view['filter_rules'] = [{
+            "rule_type": 12,
+            "value": "secret"
+        }]
+
+        response = self.client.put(f"/api/saved_views/{v1.id}/", view, format='json')
+        self.assertEqual(response.status_code, 200)
+
+        v1 = SavedView.objects.get(id=v1.id)
+        self.assertEqual(v1.filter_rules.count(), 1)
+        self.assertEqual(v1.filter_rules.first().value, "secret")
+
+        view['filter_rules'] = []
+
+        response = self.client.put(f"/api/saved_views/{v1.id}/", view, format='json')
+        self.assertEqual(response.status_code, 200)
+
+        v1 = SavedView.objects.get(id=v1.id)
+        self.assertEqual(v1.filter_rules.count(), 0)
+
+
+class TestBulkEdit(DirectoriesMixin, APITestCase):
+
+    def setUp(self):
+        super(TestBulkEdit, self).setUp()
+
+        user = User.objects.create_superuser(username="temp_admin")
+        self.client.force_login(user=user)
+
+        patcher = mock.patch('documents.bulk_edit.async_task')
+        self.async_task = patcher.start()
+        self.addCleanup(patcher.stop)
+        self.c1 = Correspondent.objects.create(name="c1")
+        self.c2 = Correspondent.objects.create(name="c2")
+        self.dt1 = DocumentType.objects.create(name="dt1")
+        self.dt2 = DocumentType.objects.create(name="dt2")
+        self.t1 = Tag.objects.create(name="t1")
+        self.t2 = Tag.objects.create(name="t2")
+        self.doc1 = Document.objects.create(checksum="A", title="A")
+        self.doc2 = Document.objects.create(checksum="B", title="B", correspondent=self.c1, document_type=self.dt1)
+        self.doc3 = Document.objects.create(checksum="C", title="C", correspondent=self.c2, document_type=self.dt2)
+        self.doc4 = Document.objects.create(checksum="D", title="D")
+        self.doc5 = Document.objects.create(checksum="E", title="E")
+        self.doc2.tags.add(self.t1)
+        self.doc3.tags.add(self.t2)
+        self.doc4.tags.add(self.t1, self.t2)
+
+    def test_set_correspondent(self):
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 1)
+        bulk_edit.set_correspondent([self.doc1.id, self.doc2.id, self.doc3.id], self.c2.id)
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 3)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc2.id])
+
+    def test_unset_correspondent(self):
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 1)
+        bulk_edit.set_correspondent([self.doc1.id, self.doc2.id, self.doc3.id], None)
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 0)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc2.id, self.doc3.id])
+
+    def test_set_document_type(self):
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 1)
+        bulk_edit.set_document_type([self.doc1.id, self.doc2.id, self.doc3.id], self.dt2.id)
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 3)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc2.id])
+
+    def test_unset_document_type(self):
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 1)
+        bulk_edit.set_document_type([self.doc1.id, self.doc2.id, self.doc3.id], None)
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 0)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc2.id, self.doc3.id])
+
+    def test_add_tag(self):
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2)
+        bulk_edit.add_tag([self.doc1.id, self.doc2.id, self.doc3.id, self.doc4.id], self.t1.id)
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 4)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc3.id])
+
+    def test_remove_tag(self):
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2)
+        bulk_edit.remove_tag([self.doc1.id, self.doc3.id, self.doc4.id], self.t1.id)
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 1)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc4.id])
+
+    def test_modify_tags(self):
+        tag_unrelated = Tag.objects.create(name="unrelated")
+        self.doc2.tags.add(tag_unrelated)
+        self.doc3.tags.add(tag_unrelated)
+        bulk_edit.modify_tags([self.doc2.id, self.doc3.id], add_tags=[self.t2.id], remove_tags=[self.t1.id])
+
+        self.assertCountEqual(list(self.doc2.tags.all()), [self.t2, tag_unrelated])
+        self.assertCountEqual(list(self.doc3.tags.all()), [self.t2, tag_unrelated])
+
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        # TODO: doc3 should not be affected, but the query for that is rather complicated
+        self.assertCountEqual(kwargs['document_ids'], [self.doc2.id, self.doc3.id])
+
+    def test_delete(self):
+        self.assertEqual(Document.objects.count(), 5)
+        bulk_edit.delete([self.doc1.id, self.doc2.id])
+        self.assertEqual(Document.objects.count(), 3)
+        self.assertCountEqual([doc.id for doc in Document.objects.all()], [self.doc3.id, self.doc4.id, self.doc5.id])
+
+    @mock.patch("documents.serialisers.bulk_edit.set_correspondent")
+    def test_api_set_correspondent(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_correspondent",
+            "parameters": {"correspondent": self.c1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['correspondent'], self.c1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.set_correspondent")
+    def test_api_unset_correspondent(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_correspondent",
+            "parameters": {"correspondent": None}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertIsNone(kwargs['correspondent'])
+
+    @mock.patch("documents.serialisers.bulk_edit.set_document_type")
+    def test_api_set_type(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_document_type",
+            "parameters": {"document_type": self.dt1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['document_type'], self.dt1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.set_document_type")
+    def test_api_unset_type(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_document_type",
+            "parameters": {"document_type": None}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertIsNone(kwargs['document_type'])
+
+    @mock.patch("documents.serialisers.bulk_edit.add_tag")
+    def test_api_add_tag(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "add_tag",
+            "parameters": {"tag": self.t1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['tag'], self.t1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.remove_tag")
+    def test_api_remove_tag(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "remove_tag",
+            "parameters": {"tag": self.t1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['tag'], self.t1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.modify_tags")
+    def test_api_modify_tags(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id, self.doc3.id],
+            "method": "modify_tags",
+            "parameters": {"add_tags": [self.t1.id], "remove_tags": [self.t2.id]}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertListEqual(args[0], [self.doc1.id, self.doc3.id])
+        self.assertEqual(kwargs['add_tags'], [self.t1.id])
+        self.assertEqual(kwargs['remove_tags'], [self.t2.id])
+
+    @mock.patch("documents.serialisers.bulk_edit.delete")
+    def test_api_delete(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "delete",
+            "parameters": {}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(len(kwargs), 0)
+
+    def test_api_invalid_doc(self):
+        self.assertEqual(Document.objects.count(), 5)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [-235],
+            "method": "delete",
+            "parameters": {}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(Document.objects.count(), 5)
+
+    def test_api_invalid_method(self):
+        self.assertEqual(Document.objects.count(), 5)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "exterminate",
+            "parameters": {}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(Document.objects.count(), 5)
+
+    def test_api_invalid_correspondent(self):
+        self.assertEqual(self.doc2.correspondent, self.c1)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "set_correspondent",
+            "parameters": {'correspondent': 345657}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+        doc2 = Document.objects.get(id=self.doc2.id)
+        self.assertEqual(doc2.correspondent, self.c1)
+
+    def test_api_invalid_document_type(self):
+        self.assertEqual(self.doc2.document_type, self.dt1)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "set_document_type",
+            "parameters": {'document_type': 345657}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+        doc2 = Document.objects.get(id=self.doc2.id)
+        self.assertEqual(doc2.document_type, self.dt1)
+
+    def test_api_add_invalid_tag(self):
+        self.assertEqual(list(self.doc2.tags.all()), [self.t1])
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "add_tag",
+            "parameters": {'tag': 345657}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+        self.assertEqual(list(self.doc2.tags.all()), [self.t1])
+
+    def test_api_delete_invalid_tag(self):
+        self.assertEqual(list(self.doc2.tags.all()), [self.t1])
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "remove_tag",
+            "parameters": {'tag': 345657}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+        self.assertEqual(list(self.doc2.tags.all()), [self.t1])
+
+    def test_api_modify_invalid_tags(self):
+        self.assertEqual(list(self.doc2.tags.all()), [self.t1])
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "modify_tags",
+            "parameters": {'add_tags': [self.t2.id, 1657], "remove_tags": [1123123]}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+    def test_api_selection_data_empty(self):
+        response = self.client.post("/api/documents/selection_data/", json.dumps({
+            "documents": []
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        for field, Entity in [('selected_correspondents', Correspondent), ('selected_tags', Tag), ('selected_document_types', DocumentType)]:
+            self.assertEqual(len(response.data[field]), Entity.objects.count())
+            for correspondent in response.data[field]:
+                self.assertEqual(correspondent['document_count'], 0)
+            self.assertCountEqual(
+                map(lambda c: c['id'], response.data[field]),
+                map(lambda c: c['id'], Entity.objects.values('id')))
+
+    def test_api_selection_data(self):
+        response = self.client.post("/api/documents/selection_data/", json.dumps({
+            "documents": [self.doc1.id, self.doc2.id, self.doc4.id, self.doc5.id]
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+
+        self.assertCountEqual(response.data['selected_correspondents'], [{"id": self.c1.id, "document_count": 1}, {"id": self.c2.id, "document_count": 0}])
+        self.assertCountEqual(response.data['selected_tags'], [{"id": self.t1.id, "document_count": 2}, {"id": self.t2.id, "document_count": 1}])
+        self.assertCountEqual(response.data['selected_document_types'], [{"id": self.c1.id, "document_count": 1}, {"id": self.c2.id, "document_count": 0}])
+
+
+class TestApiAuth(APITestCase):
+
+    def test_auth_required(self):
+
+        d = Document.objects.create(title="Test")
+
+        self.assertEqual(self.client.get("/api/documents/").status_code, 401)
+
+        self.assertEqual(self.client.get(f"/api/documents/{d.id}/").status_code, 401)
+        self.assertEqual(self.client.get(f"/api/documents/{d.id}/download/").status_code, 401)
+        self.assertEqual(self.client.get(f"/api/documents/{d.id}/preview/").status_code, 401)
+        self.assertEqual(self.client.get(f"/api/documents/{d.id}/thumb/").status_code, 401)
+
+        self.assertEqual(self.client.get("/api/tags/").status_code, 401)
+        self.assertEqual(self.client.get("/api/correspondents/").status_code, 401)
+        self.assertEqual(self.client.get("/api/document_types/").status_code, 401)
+
+        self.assertEqual(self.client.get("/api/logs/").status_code, 401)
+        self.assertEqual(self.client.get("/api/saved_views/").status_code, 401)
+
+        self.assertEqual(self.client.get("/api/search/").status_code, 401)
+        self.assertEqual(self.client.get("/api/search/auto_complete/").status_code, 401)
+        self.assertEqual(self.client.get("/api/documents/bulk_edit/").status_code, 401)
+        self.assertEqual(self.client.get("/api/documents/selection_data/").status_code, 401)

src/documents/tests/test_checks.py

@@ -1,9 +1,12 @@
 import unittest
+from unittest import mock
 
+from django.core.checks import Error
 from django.test import TestCase
 
 from .factories import DocumentFactory
-from ..checks import changed_password_check
+from .. import document_consumer_declaration
+from ..checks import changed_password_check, parser_check
 from ..models import Document
 
 
@@ -15,3 +18,13 @@ class ChecksTestCase(TestCase):
     def test_changed_password_check_no_encryption(self):
         DocumentFactory.create(storage_type=Document.STORAGE_TYPE_UNENCRYPTED)
         self.assertEqual(changed_password_check(None), [])
+
+    def test_parser_check(self):
+
+        self.assertEqual(parser_check(None), [])
+
+        with mock.patch('documents.checks.document_consumer_declaration.send') as m:
+            m.return_value = []
+
+            self.assertEqual(parser_check(None), [Error("No parsers found. This is a bug. The consumer won't be "
+                                                        "able to consume any documents without parsers.")])

src/documents/tests/test_consumer.py

@@ -27,83 +27,8 @@ class TestAttributes(TestCase):
 
         self.assertEqual(file_info.title, title, filename)
 
-        self.assertEqual(tuple([t.slug for t in file_info.tags]), tags, filename)
+        self.assertEqual(tuple([t.name for t in file_info.tags]), tags, filename)
 
-    def test_guess_attributes_from_name0(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Title.pdf", "Sender", "Title", ())
-
-    def test_guess_attributes_from_name1(self):
-        self._test_guess_attributes_from_name(
-            "Spaced Sender - Title.pdf", "Spaced Sender", "Title", ())
-
-    def test_guess_attributes_from_name2(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Spaced Title.pdf", "Sender", "Spaced Title", ())
-
-    def test_guess_attributes_from_name3(self):
-        self._test_guess_attributes_from_name(
-            "Dashed-Sender - Title.pdf", "Dashed-Sender", "Title", ())
-
-    def test_guess_attributes_from_name4(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Dashed-Title.pdf", "Sender", "Dashed-Title", ())
-
-    def test_guess_attributes_from_name5(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Title - tag1,tag2,tag3.pdf",
-            "Sender",
-            "Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name6(self):
-        self._test_guess_attributes_from_name(
-            "Spaced Sender - Title - tag1,tag2,tag3.pdf",
-            "Spaced Sender",
-            "Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name7(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Spaced Title - tag1,tag2,tag3.pdf",
-            "Sender",
-            "Spaced Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name8(self):
-        self._test_guess_attributes_from_name(
-            "Dashed-Sender - Title - tag1,tag2,tag3.pdf",
-            "Dashed-Sender",
-            "Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name9(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Dashed-Title - tag1,tag2,tag3.pdf",
-            "Sender",
-            "Dashed-Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name10(self):
-        self._test_guess_attributes_from_name(
-            "Σενδερ - Τιτλε - tag1,tag2,tag3.pdf",
-            "Σενδερ",
-            "Τιτλε",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name_when_correspondent_empty(self):
-        self._test_guess_attributes_from_name(
-            ' - weird empty correspondent but should not break.pdf',
-            None,
-            'weird empty correspondent but should not break',
-            ()
-        )
 
     def test_guess_attributes_from_name_when_title_starts_with_dash(self):
         self._test_guess_attributes_from_name(
@@ -121,28 +46,6 @@ class TestAttributes(TestCase):
             ()
         )
 
-    def test_guess_attributes_from_name_when_title_is_empty(self):
-        self._test_guess_attributes_from_name(
-            'weird correspondent but should not break - .pdf',
-            'weird correspondent but should not break',
-            '',
-            ()
-        )
-
-    def test_case_insensitive_tag_creation(self):
-        """
-        Tags should be detected and created as lower case.
-        :return:
-        """
-
-        filename = "Title - Correspondent - tAg1,TAG2.pdf"
-        self.assertEqual(len(FileInfo.from_filename(filename).tags), 2)
-
-        path = "Title - Correspondent - tag1,tag2.pdf"
-        self.assertEqual(len(FileInfo.from_filename(filename).tags), 2)
-
-        self.assertEqual(Tag.objects.all().count(), 2)
-
 
 class TestFieldPermutations(TestCase):
 
@@ -188,7 +91,7 @@ class TestFieldPermutations(TestCase):
             self.assertEqual(info.tags, (), filename)
         else:
             self.assertEqual(
-                [t.slug for t in info.tags], tags.split(','),
+                [t.name for t in info.tags], tags.split(','),
                 filename
             )
 
@@ -199,69 +102,7 @@ class TestFieldPermutations(TestCase):
             filename = template.format(**spec)
             self._test_guessed_attributes(filename, **spec)
 
-    def test_title_and_correspondent(self):
-        template = '{correspondent} - {title}.pdf'
-        for correspondent in self.valid_correspondents:
-            for title in self.valid_titles:
-                spec = dict(correspondent=correspondent, title=title)
-                filename = template.format(**spec)
-                self._test_guessed_attributes(filename, **spec)
-
-    def test_title_and_correspondent_and_tags(self):
-        template = '{correspondent} - {title} - {tags}.pdf'
-        for correspondent in self.valid_correspondents:
-            for title in self.valid_titles:
-                for tags in self.valid_tags:
-                    spec = dict(correspondent=correspondent, title=title,
-                                tags=tags)
-                    filename = template.format(**spec)
-                    self._test_guessed_attributes(filename, **spec)
-
-    def test_created_and_correspondent_and_title_and_tags(self):
-
-        template = (
-            "{created} - "
-            "{correspondent} - "
-            "{title} - "
-            "{tags}.pdf"
-        )
-
-        for created in self.valid_dates:
-            for correspondent in self.valid_correspondents:
-                for title in self.valid_titles:
-                    for tags in self.valid_tags:
-                        spec = {
-                            "created": created,
-                            "correspondent": correspondent,
-                            "title": title,
-                            "tags": tags,
-                        }
-                        self._test_guessed_attributes(
-                            template.format(**spec), **spec)
-
-    def test_created_and_correspondent_and_title(self):
-
-        template = "{created} - {correspondent} - {title}.pdf"
-
-        for created in self.valid_dates:
-            for correspondent in self.valid_correspondents:
-                for title in self.valid_titles:
-
-                    # Skip cases where title looks like a tag as we can't
-                    # accommodate such cases.
-                    if title.lower() == title:
-                        continue
-
-                    spec = {
-                        "created": created,
-                        "correspondent": correspondent,
-                        "title": title
-                    }
-                    self._test_guessed_attributes(
-                        template.format(**spec), **spec)
-
     def test_created_and_title(self):
-
         template = "{created} - {title}.pdf"
 
         for created in self.valid_dates:
@@ -273,21 +114,6 @@ class TestFieldPermutations(TestCase):
                 self._test_guessed_attributes(
                     template.format(**spec), **spec)
 
-    def test_created_and_title_and_tags(self):
-
-        template = "{created} - {title} - {tags}.pdf"
-
-        for created in self.valid_dates:
-            for title in self.valid_titles:
-                for tags in self.valid_tags:
-                    spec = {
-                        "created": created,
-                        "title": title,
-                        "tags": tags
-                    }
-                    self._test_guessed_attributes(
-                        template.format(**spec), **spec)
-
     def test_invalid_date_format(self):
         info = FileInfo.from_filename("06112017Z - title.pdf")
         self.assertEqual(info.title, "title")
@@ -336,32 +162,6 @@ class TestFieldPermutations(TestCase):
             info = FileInfo.from_filename(filename)
             self.assertEqual(info.title, "anotherall")
 
-        # Complex transformation without date in replacement string
-        with self.settings(
-                FILENAME_PARSE_TRANSFORMS=[(exact_patt, repl1)]):
-            info = FileInfo.from_filename(filename)
-            self.assertEqual(info.title, "0001")
-            self.assertEqual(len(info.tags), 2)
-            self.assertEqual(info.tags[0].slug, "tag1")
-            self.assertEqual(info.tags[1].slug, "tag2")
-            self.assertIsNone(info.created)
-
-        # Complex transformation with date in replacement string
-        with self.settings(
-            FILENAME_PARSE_TRANSFORMS=[
-                (none_patt, "none.gif"),
-                (exact_patt, repl2),    # <-- matches
-                (exact_patt, repl1),
-                (all_patt, "all.gif")]):
-            info = FileInfo.from_filename(filename)
-            self.assertEqual(info.title, "0001")
-            self.assertEqual(len(info.tags), 2)
-            self.assertEqual(info.tags[0].slug, "tag1")
-            self.assertEqual(info.tags[1].slug, "tag2")
-            self.assertEqual(info.created.year, 2019)
-            self.assertEqual(info.created.month, 9)
-            self.assertEqual(info.created.day, 8)
-
 
 class DummyParser(DocumentParser):
 
@@ -377,7 +177,7 @@ class DummyParser(DocumentParser):
     def get_optimised_thumbnail(self, document_path, mime_type):
         return self.fake_thumb
 
-    def parse(self, document_path, mime_type):
+    def parse(self, document_path, mime_type, file_name=None):
         self.text = "The Text"
 
 
@@ -394,7 +194,7 @@ class FaultyParser(DocumentParser):
     def get_optimised_thumbnail(self, document_path, mime_type):
         return self.fake_thumb
 
-    def parse(self, document_path, mime_type):
+    def parse(self, document_path, mime_type, file_name=None):
         raise ParseError("Does not compute.")
 
 
@@ -476,15 +276,13 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
     def testOverrideFilename(self):
         filename = self.get_test_file()
-        override_filename = "My Bank - Statement for November.pdf"
+        override_filename = "Statement for November.pdf"
 
         document = self.consumer.try_consume_file(filename, override_filename=override_filename)
 
-        self.assertEqual(document.correspondent.name, "My Bank")
         self.assertEqual(document.title, "Statement for November")
 
     def testOverrideTitle(self):
-
         document = self.consumer.try_consume_file(self.get_test_file(), override_title="Override Title")
         self.assertEqual(document.title, "Override Title")
 
@@ -552,7 +350,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
         try:
             self.consumer.try_consume_file(self.get_test_file())
         except ConsumerError as e:
-            self.assertTrue("No parsers abvailable for" in str(e))
+            self.assertEqual("Unsupported mime type application/pdf of file sample.pdf", str(e))
             return
 
         self.fail("Should throw exception")
@@ -594,14 +392,13 @@ class TestConsumer(DirectoriesMixin, TestCase):
     def testFilenameHandling(self):
         filename = self.get_test_file()
 
-        document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs")
+        document = self.consumer.try_consume_file(filename, override_title="new docs")
 
         self.assertEqual(document.title, "new docs")
-        self.assertEqual(document.correspondent.name, "Bank")
-        self.assertEqual(document.filename, "Bank/new docs-0000001.pdf")
+        self.assertEqual(document.filename, "none/new docs.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
-    @mock.patch("documents.signals.handlers.generate_filename")
+    @mock.patch("documents.signals.handlers.generate_unique_filename")
     def testFilenameHandlingUnstableFormat(self, m):
 
         filenames = ["this", "that", "now this", "i cant decide"]
@@ -611,16 +408,15 @@ class TestConsumer(DirectoriesMixin, TestCase):
             filenames.insert(0, f)
             return f
 
-        m.side_effect = lambda f: get_filename()
+        m.side_effect = lambda f, root: get_filename()
 
         filename = self.get_test_file()
 
         Tag.objects.create(name="test", is_inbox_tag=True)
 
-        document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs")
+        document = self.consumer.try_consume_file(filename, override_title="new docs")
 
         self.assertEqual(document.title, "new docs")
-        self.assertEqual(document.correspondent.name, "Bank")
         self.assertIsNotNone(os.path.isfile(document.title))
         self.assertTrue(os.path.isfile(document.source_path))
 
@@ -642,3 +438,81 @@ class TestConsumer(DirectoriesMixin, TestCase):
         self.assertEqual(document.document_type, dtype)
         self.assertIn(t1, document.tags.all())
         self.assertNotIn(t2, document.tags.all())
+
+    @override_settings(CONSUMER_DELETE_DUPLICATES=True)
+    def test_delete_duplicate(self):
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        doc = self.consumer.try_consume_file(dst)
+
+        self.assertFalse(os.path.isfile(dst))
+        self.assertIsNotNone(doc)
+
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst)
+        self.assertFalse(os.path.isfile(dst))
+
+    @override_settings(CONSUMER_DELETE_DUPLICATES=False)
+    def test_no_delete_duplicate(self):
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        doc = self.consumer.try_consume_file(dst)
+
+        self.assertFalse(os.path.isfile(dst))
+        self.assertIsNotNone(doc)
+
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst)
+        self.assertTrue(os.path.isfile(dst))
+
+
+class PostConsumeTestCase(TestCase):
+
+    @mock.patch("documents.consumer.Popen")
+    @override_settings(POST_CONSUME_SCRIPT=None)
+    def test_no_post_consume_script(self, m):
+        doc = Document.objects.create(title="Test", mime_type="application/pdf")
+        tag1 = Tag.objects.create(name="a")
+        tag2 = Tag.objects.create(name="b")
+        doc.tags.add(tag1)
+        doc.tags.add(tag2)
+
+        Consumer().run_post_consume_script(doc)
+
+        m.assert_not_called()
+
+    @mock.patch("documents.consumer.Popen")
+    @override_settings(POST_CONSUME_SCRIPT="script")
+    def test_post_consume_script_simple(self, m):
+        doc = Document.objects.create(title="Test", mime_type="application/pdf")
+
+        Consumer().run_post_consume_script(doc)
+
+        m.assert_called_once()
+
+    @mock.patch("documents.consumer.Popen")
+    @override_settings(POST_CONSUME_SCRIPT="script")
+    def test_post_consume_script_with_correspondent(self, m):
+        c = Correspondent.objects.create(name="my_bank")
+        doc = Document.objects.create(title="Test", mime_type="application/pdf", correspondent=c)
+        tag1 = Tag.objects.create(name="a")
+        tag2 = Tag.objects.create(name="b")
+        doc.tags.add(tag1)
+        doc.tags.add(tag2)
+
+        Consumer().run_post_consume_script(doc)
+
+        m.assert_called_once()
+
+        args, kwargs = m.call_args
+
+        command = args[0]
+
+        self.assertEqual(command[0], "script")
+        self.assertEqual(command[1], str(doc.pk))
+        self.assertEqual(command[5], f"/api/documents/{doc.pk}/download/")
+        self.assertEqual(command[6], f"/api/documents/{doc.pk}/thumb/")
+        self.assertEqual(command[7], "my_bank")
+        self.assertCountEqual(command[8].split(","), ["a", "b"])

src/documents/tests/test_file_handling.py

@@ -1,15 +1,20 @@
 import datetime
+import hashlib
 import os
+import random
+import uuid
 from pathlib import Path
 from unittest import mock
 
 from django.conf import settings
 from django.db import DatabaseError
 from django.test import TestCase, override_settings
+from django.utils import timezone
 
 from .utils import DirectoriesMixin
-from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories
-from ..models import Document, Correspondent
+from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \
+    generate_unique_filename
+from ..models import Document, Correspondent, Tag, DocumentType
 
 
 class TestFileHandling(DirectoriesMixin, TestCase):
@@ -40,13 +45,13 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         document.filename = generate_filename(document)
 
         # Ensure that filename is properly generated
-        self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk))
+        self.assertEqual(document.filename, "none/none.pdf")
 
         # Enable encryption and check again
         document.storage_type = Document.STORAGE_TYPE_GPG
         document.filename = generate_filename(document)
         self.assertEqual(document.filename,
-                         "none/none-{:07d}.pdf.gpg".format(document.pk))
+                         "none/none.pdf.gpg")
 
         document.save()
 
@@ -62,7 +67,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         # Check proper handling of files
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/test"), True)
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), False)
-        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/test/test-{:07d}.pdf.gpg".format(document.pk)), True)
+        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/test/test.pdf.gpg"), True)
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
     def test_file_renaming_missing_permissions(self):
@@ -74,12 +79,12 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         # Ensure that filename is properly generated
         document.filename = generate_filename(document)
         self.assertEqual(document.filename,
-                         "none/none-{:07d}.pdf".format(document.pk))
+                         "none/none.pdf")
         create_source_path_directory(document.source_path)
         Path(document.source_path).touch()
 
         # Test source_path
-        self.assertEqual(document.source_path, settings.ORIGINALS_DIR + "/none/none-{:07d}.pdf".format(document.pk))
+        self.assertEqual(document.source_path, settings.ORIGINALS_DIR + "/none/none.pdf")
 
         # Make the folder read- and execute-only (no writing and no renaming)
         os.chmod(settings.ORIGINALS_DIR + "/none", 0o555)
@@ -89,8 +94,8 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         document.save()
 
         # Check proper handling of files
-        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none-{:07d}.pdf".format(document.pk)), True)
-        self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk))
+        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none.pdf"), True)
+        self.assertEqual(document.filename, "none/none.pdf")
 
         os.chmod(settings.ORIGINALS_DIR + "/none", 0o777)
 
@@ -108,7 +113,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         # Ensure that filename is properly generated
         document.filename = generate_filename(document)
         self.assertEqual(document.filename,
-                         "none/none-{:07d}.pdf".format(document.pk))
+                         "none/none.pdf")
         create_source_path_directory(document.source_path)
         Path(document.source_path).touch()
 
@@ -125,8 +130,8 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
             # Check proper handling of files
             self.assertTrue(os.path.isfile(document.source_path))
-            self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none-{:07d}.pdf".format(document.pk)), True)
-            self.assertEqual(document.filename, "none/none-{:07d}.pdf".format(document.pk))
+            self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none.pdf"), True)
+            self.assertEqual(document.filename, "none/none.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
     def test_document_delete(self):
@@ -138,7 +143,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         # Ensure that filename is properly generated
         document.filename = generate_filename(document)
         self.assertEqual(document.filename,
-                         "none/none-{:07d}.pdf".format(document.pk))
+                         "none/none.pdf")
 
         create_source_path_directory(document.source_path)
         Path(document.source_path).touch()
@@ -146,7 +151,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         # Ensure file deletion after delete
         pk = document.pk
         document.delete()
-        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none-{:07d}.pdf".format(pk)), False)
+        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none.pdf"), False)
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), False)
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}")
@@ -168,7 +173,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         # Ensure that filename is properly generated
         document.filename = generate_filename(document)
         self.assertEqual(document.filename,
-                         "none/none-{:07d}.pdf".format(document.pk))
+                         "none/none.pdf")
 
         create_source_path_directory(document.source_path)
 
@@ -185,6 +190,17 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), True)
         self.assertTrue(os.path.isfile(important_file))
 
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{document_type} - {title}")
+    def test_document_type(self):
+        dt = DocumentType.objects.create(name="my_doc_type")
+        d = Document.objects.create(title="the_doc", mime_type="application/pdf")
+
+        self.assertEqual(generate_filename(d), "none - the_doc.pdf")
+
+        d.document_type = dt
+
+        self.assertEqual(generate_filename(d), "my_doc_type - the_doc.pdf")
+
     @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
     def test_tags_with_underscore(self):
         document = Document()
@@ -199,7 +215,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
         # Ensure that filename is properly generated
         self.assertEqual(generate_filename(document),
-                         "demo-{:07d}.pdf".format(document.pk))
+                         "demo.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
     def test_tags_with_dash(self):
@@ -215,7 +231,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
         # Ensure that filename is properly generated
         self.assertEqual(generate_filename(document),
-                         "demo-{:07d}.pdf".format(document.pk))
+                         "demo.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
     def test_tags_malformed(self):
@@ -231,7 +247,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
         # Ensure that filename is properly generated
         self.assertEqual(generate_filename(document),
-                         "none-{:07d}.pdf".format(document.pk))
+                         "none.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[0]}")
     def test_tags_all(self):
@@ -246,7 +262,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
         # Ensure that filename is properly generated
         self.assertEqual(generate_filename(document),
-                         "demo-{:07d}.pdf".format(document.pk))
+                         "demo.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[1]}")
     def test_tags_out_of_bounds(self):
@@ -261,7 +277,58 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
         # Ensure that filename is properly generated
         self.assertEqual(generate_filename(document),
-                         "none-{:07d}.pdf".format(document.pk))
+                         "none.pdf")
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{tags}")
+    def test_tags_without_args(self):
+        document = Document()
+        document.mime_type = "application/pdf"
+        document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
+        document.save()
+
+        self.assertEqual(generate_filename(document), f"{document.pk:07}.pdf")
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{title} {tag_list}")
+    def test_tag_list(self):
+        doc = Document.objects.create(title="doc1", mime_type="application/pdf")
+        doc.tags.create(name="tag2")
+        doc.tags.create(name="tag1")
+
+        self.assertEqual(generate_filename(doc), "doc1 tag1,tag2.pdf")
+
+        doc = Document.objects.create(title="doc2", checksum="B", mime_type="application/pdf")
+
+        self.assertEqual(generate_filename(doc), "doc2.pdf")
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="//etc/something/{title}")
+    def test_filename_relative(self):
+        doc = Document.objects.create(title="doc1", mime_type="application/pdf")
+        doc.filename = generate_filename(doc)
+        doc.save()
+
+        self.assertEqual(doc.source_path, os.path.join(settings.ORIGINALS_DIR, "etc", "something", "doc1.pdf"))
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{created_year}-{created_month}-{created_day}")
+    def test_created_year_month_day(self):
+        d1 = timezone.make_aware(datetime.datetime(2020, 3, 6, 1, 1, 1))
+        doc1 = Document.objects.create(title="doc1", mime_type="application/pdf", created=d1)
+
+        self.assertEqual(generate_filename(doc1), "2020-03-06.pdf")
+
+        doc1.created = timezone.make_aware(datetime.datetime(2020, 11, 16, 1, 1, 1))
+
+        self.assertEqual(generate_filename(doc1), "2020-11-16.pdf")
+
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{added_year}-{added_month}-{added_day}")
+    def test_added_year_month_day(self):
+        d1 = timezone.make_aware(datetime.datetime(232, 1, 9, 1, 1, 1))
+        doc1 = Document.objects.create(title="doc1", mime_type="application/pdf", added=d1)
+
+        self.assertEqual(generate_filename(doc1), "232-01-09.pdf")
+
+        doc1.added = timezone.make_aware(datetime.datetime(2020, 11, 16, 1, 1, 1))
+
+        self.assertEqual(generate_filename(doc1), "2020-11-16.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}/{correspondent}")
     def test_nested_directory_cleanup(self):
@@ -272,7 +339,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
         # Ensure that filename is properly generated
         document.filename = generate_filename(document)
-        self.assertEqual(document.filename, "none/none/none-{:07d}.pdf".format(document.pk))
+        self.assertEqual(document.filename, "none/none/none.pdf")
         create_source_path_directory(document.source_path)
         Path(document.source_path).touch()
 
@@ -282,7 +349,7 @@ class TestFileHandling(DirectoriesMixin, TestCase):
         pk = document.pk
         document.delete()
 
-        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none/none-{:07d}.pdf".format(pk)), False)
+        self.assertEqual(os.path.isfile(settings.ORIGINALS_DIR + "/none/none/none.pdf"), False)
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none/none"), False)
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), False)
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR), True)
@@ -330,6 +397,48 @@ class TestFileHandling(DirectoriesMixin, TestCase):
 
         self.assertEqual(generate_filename(document), "0000001.pdf")
 
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
+    def test_duplicates(self):
+        document = Document.objects.create(mime_type="application/pdf", title="qwe", checksum="A", pk=1)
+        document2 = Document.objects.create(mime_type="application/pdf", title="qwe", checksum="B", pk=2)
+        Path(document.source_path).touch()
+        Path(document2.source_path).touch()
+        document.filename = "0000001.pdf"
+        document.save()
+
+        self.assertTrue(os.path.isfile(document.source_path))
+        self.assertEqual(document.filename, "qwe.pdf")
+
+        document2.filename = "0000002.pdf"
+        document2.save()
+
+        self.assertTrue(os.path.isfile(document.source_path))
+        self.assertEqual(document2.filename, "qwe_01.pdf")
+
+        # saving should not change the file names.
+
+        document.save()
+
+        self.assertTrue(os.path.isfile(document.source_path))
+        self.assertEqual(document.filename, "qwe.pdf")
+
+        document2.save()
+
+        self.assertTrue(os.path.isfile(document.source_path))
+        self.assertEqual(document2.filename, "qwe_01.pdf")
+
+        document.delete()
+
+        self.assertFalse(os.path.isfile(document.source_path))
+
+        # filename free, should remove _01 suffix
+
+        document2.save()
+
+        self.assertTrue(os.path.isfile(document.source_path))
+        self.assertEqual(document2.filename, "qwe.pdf")
+
+
 
 class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
 
@@ -358,15 +467,14 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
         self.assertFalse(os.path.isfile(archive))
         self.assertTrue(os.path.isfile(doc.source_path))
         self.assertTrue(os.path.isfile(doc.archive_path))
-        self.assertEqual(doc.source_path, os.path.join(settings.ORIGINALS_DIR, "none", "my_doc-0000001.pdf"))
-        self.assertEqual(doc.archive_path, os.path.join(settings.ARCHIVE_DIR, "none", "my_doc-0000001.pdf"))
+        self.assertEqual(doc.source_path, os.path.join(settings.ORIGINALS_DIR, "none", "my_doc.pdf"))
+        self.assertEqual(doc.archive_path, os.path.join(settings.ARCHIVE_DIR, "none", "my_doc.pdf"))
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
     def test_move_archive_gone(self):
         original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")
         archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")
         Path(original).touch()
-        #Path(archive).touch()
         doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
 
         self.assertTrue(os.path.isfile(original))
@@ -381,7 +489,7 @@ class TestFileHandlingWithArchive(DirectoriesMixin, TestCase):
         Path(original).touch()
         Path(archive).touch()
         os.makedirs(os.path.join(settings.ARCHIVE_DIR, "none"))
-        Path(os.path.join(settings.ARCHIVE_DIR, "none", "my_doc-0000001.pdf")).touch()
+        Path(os.path.join(settings.ARCHIVE_DIR, "none", "my_doc.pdf")).touch()
         doc = Document.objects.create(mime_type="application/pdf", title="my_doc", filename="0000001.pdf", checksum="A", archive_checksum="B")
 
         self.assertTrue(os.path.isfile(original))
@@ -494,14 +602,35 @@ class TestFilenameGeneration(TestCase):
     def test_invalid_characters(self):
 
         doc = Document.objects.create(title="This. is the title.", mime_type="application/pdf", pk=1, checksum="1")
-        self.assertEqual(generate_filename(doc), "This. is the title-0000001.pdf")
+        self.assertEqual(generate_filename(doc), "This. is the title.pdf")
 
         doc = Document.objects.create(title="my\\invalid/../title:yay", mime_type="application/pdf", pk=2, checksum="2")
-        self.assertEqual(generate_filename(doc), "my-invalid-..-title-yay-0000002.pdf")
+        self.assertEqual(generate_filename(doc), "my-invalid-..-title-yay.pdf")
 
     @override_settings(
         PAPERLESS_FILENAME_FORMAT="{created}"
     )
     def test_date(self):
-        doc = Document.objects.create(title="does not matter", created=datetime.datetime(2020,5,21, 7,36,51, 153), mime_type="application/pdf", pk=2, checksum="2")
-        self.assertEqual(generate_filename(doc), "2020-05-21-0000002.pdf")
+        doc = Document.objects.create(title="does not matter", created=timezone.make_aware(datetime.datetime(2020,5,21, 7,36,51, 153)), mime_type="application/pdf", pk=2, checksum="2")
+        self.assertEqual(generate_filename(doc), "2020-05-21.pdf")
+
+
+def run():
+    doc = Document.objects.create(checksum=str(uuid.uuid4()), title=str(uuid.uuid4()), content="wow")
+    doc.filename = generate_unique_filename(doc, settings.ORIGINALS_DIR)
+    Path(doc.thumbnail_path).touch()
+    with open(doc.source_path, "w") as f:
+        f.write(str(uuid.uuid4()))
+    with open(doc.source_path, "rb") as f:
+        doc.checksum = hashlib.md5(f.read()).hexdigest()
+
+    with open(doc.archive_path, "w") as f:
+        f.write(str(uuid.uuid4()))
+    with open(doc.archive_path, "rb") as f:
+        doc.archive_checksum = hashlib.md5(f.read()).hexdigest()
+
+    doc.save()
+
+    for i in range(30):
+        doc.title = str(random.randrange(1, 5))
+        doc.save()

src/documents/tests/test_index.py

@@ -1,6 +1,9 @@
 from django.test import TestCase
 
+from documents import index
 from documents.index import JsonFormatter
+from documents.models import Document
+from documents.tests.utils import DirectoriesMixin
 
 
 class JsonFormatterTest(TestCase):
@@ -12,3 +15,21 @@ class JsonFormatterTest(TestCase):
         self.assertListEqual(self.formatter.format([]), [])
 
 
+class TestAutoComplete(DirectoriesMixin, TestCase):
+
+    def test_auto_complete(self):
+
+        doc1 = Document.objects.create(title="doc1", checksum="A", content="test test2 test3")
+        doc2 = Document.objects.create(title="doc2", checksum="B", content="test test2")
+        doc3 = Document.objects.create(title="doc3", checksum="C", content="test2")
+
+        index.add_or_update_document(doc1)
+        index.add_or_update_document(doc2)
+        index.add_or_update_document(doc3)
+
+        ix = index.open_index()
+
+        self.assertListEqual(index.autocomplete(ix, "tes"), [b"test3", b"test", b"test2"])
+        self.assertListEqual(index.autocomplete(ix, "tes", limit=3), [b"test3", b"test", b"test2"])
+        self.assertListEqual(index.autocomplete(ix, "tes", limit=1), [b"test3"])
+        self.assertListEqual(index.autocomplete(ix, "tes", limit=0), [])

src/documents/tests/test_management.py

@@ -0,0 +1,135 @@
+import hashlib
+import tempfile
+import filecmp
+import os
+import shutil
+from pathlib import Path
+from unittest import mock
+
+from django.test import TestCase, override_settings
+
+
+from django.core.management import call_command
+
+from documents.file_handling import generate_filename
+from documents.management.commands.document_archiver import handle_document
+from documents.models import Document
+from documents.tests.utils import DirectoriesMixin
+
+
+sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
+
+
+class TestArchiver(DirectoriesMixin, TestCase):
+
+    def make_models(self):
+        return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf")
+
+    def test_archiver(self):
+
+        doc = self.make_models()
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
+
+        call_command('document_archiver')
+
+    def test_handle_document(self):
+
+        doc = self.make_models()
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
+
+        handle_document(doc.pk)
+
+        doc = Document.objects.get(id=doc.id)
+
+        self.assertIsNotNone(doc.checksum)
+        self.assertTrue(os.path.isfile(doc.archive_path))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(filecmp.cmp(sample_file, doc.source_path))
+
+
+class TestDecryptDocuments(TestCase):
+
+    @override_settings(
+        ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"),
+        THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"),
+        PASSPHRASE="test",
+        PAPERLESS_FILENAME_FORMAT=None
+    )
+    @mock.patch("documents.management.commands.decrypt_documents.input")
+    def test_decrypt(self, m):
+
+        media_dir = tempfile.mkdtemp()
+        originals_dir = os.path.join(media_dir, "documents", "originals")
+        thumb_dir = os.path.join(media_dir, "documents", "thumbnails")
+        os.makedirs(originals_dir, exist_ok=True)
+        os.makedirs(thumb_dir, exist_ok=True)
+
+        override_settings(
+            ORIGINALS_DIR=originals_dir,
+            THUMBNAIL_DIR=thumb_dir,
+            PASSPHRASE="test"
+        ).enable()
+
+        doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg",  mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
+
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg"))
+
+        call_command('decrypt_documents')
+
+        doc.refresh_from_db()
+
+        self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
+        self.assertEqual(doc.filename, "0000002.pdf")
+        self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf")))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png")))
+        self.assertTrue(os.path.isfile(doc.thumbnail_path))
+
+        with doc.source_file as f:
+            checksum = hashlib.md5(f.read()).hexdigest()
+            self.assertEqual(checksum, doc.checksum)
+
+
+class TestMakeIndex(TestCase):
+
+    @mock.patch("documents.management.commands.document_index.index_reindex")
+    def test_reindex(self, m):
+        call_command("document_index", "reindex")
+        m.assert_called_once()
+
+    @mock.patch("documents.management.commands.document_index.index_optimize")
+    def test_optimize(self, m):
+        call_command("document_index", "optimize")
+        m.assert_called_once()
+
+
+class TestRenamer(DirectoriesMixin, TestCase):
+
+    def test_rename(self):
+        doc = Document.objects.create(title="test", mime_type="application/pdf")
+        doc.filename = generate_filename(doc)
+        doc.save()
+
+        Path(doc.source_path).touch()
+
+        old_source_path = doc.source_path
+
+        with override_settings(PAPERLESS_FILENAME_FORMAT="{title}"):
+            call_command("document_renamer")
+
+        doc2 = Document.objects.get(id=doc.id)
+
+        self.assertEqual(doc2.filename, "test.pdf")
+        self.assertFalse(os.path.isfile(old_source_path))
+        self.assertFalse(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc2.source_path))
+
+
+class TestCreateClassifier(TestCase):
+
+    @mock.patch("documents.management.commands.document_create_classifier.train_classifier")
+    def test_create_classifier(self, m):
+        call_command("document_create_classifier")
+
+        m.assert_called_once()

src/documents/tests/test_management_archiver.py

@@ -1,42 +0,0 @@
-import filecmp
-import os
-import shutil
-
-from django.core.management import call_command
-from django.test import TestCase
-
-from documents.management.commands.document_archiver import handle_document
-from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
-
-
-sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
-
-
-class TestArchiver(DirectoriesMixin, TestCase):
-
-    def make_models(self):
-        self.d1 = Document.objects.create(checksum="A", title="A", content="first document", pk=1, mime_type="application/pdf")
-        #self.d2 = Document.objects.create(checksum="B", title="B", content="second document")
-        #self.d3 = Document.objects.create(checksum="C", title="C", content="unrelated document")
-
-    def test_archiver(self):
-
-        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
-        self.make_models()
-
-        call_command('document_archiver')
-
-    def test_handle_document(self):
-
-        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
-        self.make_models()
-
-        handle_document(self.d1.pk)
-
-        doc = Document.objects.get(id=self.d1.id)
-
-        self.assertIsNotNone(doc.checksum)
-        self.assertTrue(os.path.isfile(doc.archive_path))
-        self.assertTrue(os.path.isfile(doc.source_path))
-        self.assertTrue(filecmp.cmp(sample_file, doc.source_path))

src/documents/tests/test_management_consumer.py

@@ -230,7 +230,7 @@ class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
 
         tag_names = ("existingTag", "Space Tag")
         # Create a Tag prior to consuming a file using it in path
-        tag_ids = [Tag.objects.create(name=tag_names[0]).pk,]
+        tag_ids = [Tag.objects.create(name="existingtag").pk,]
 
         self.t_start()
 

src/documents/tests/test_management_decrypt.py

@@ -1,57 +0,0 @@
-import hashlib
-import json
-import os
-import shutil
-import tempfile
-from unittest import mock
-
-from django.core.management import call_command
-from django.test import TestCase, override_settings
-
-from documents.management.commands import document_exporter
-from documents.models import Document, Tag, DocumentType, Correspondent
-
-
-class TestDecryptDocuments(TestCase):
-
-    @override_settings(
-        ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"),
-        THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"),
-        PASSPHRASE="test",
-        PAPERLESS_FILENAME_FORMAT=None
-    )
-    @mock.patch("documents.management.commands.decrypt_documents.input")
-    def test_decrypt(self, m):
-
-        media_dir = tempfile.mkdtemp()
-        originals_dir = os.path.join(media_dir, "documents", "originals")
-        thumb_dir = os.path.join(media_dir, "documents", "thumbnails")
-        os.makedirs(originals_dir, exist_ok=True)
-        os.makedirs(thumb_dir, exist_ok=True)
-
-        override_settings(
-            ORIGINALS_DIR=originals_dir,
-            THUMBNAIL_DIR=thumb_dir,
-            PASSPHRASE="test"
-        ).enable()
-
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg"))
-
-        Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
-
-        call_command('decrypt_documents')
-
-        doc = Document.objects.get(id=2)
-
-        self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
-        self.assertEqual(doc.filename, "0000002.pdf")
-        self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf")))
-        self.assertTrue(os.path.isfile(doc.source_path))
-        self.assertTrue(os.path.isfile(os.path.join(thumb_dir, "0000002.png")))
-        self.assertTrue(os.path.isfile(doc.thumbnail_path))
-
-        with doc.source_file as f:
-            checksum = hashlib.md5(f.read()).hexdigest()
-            self.assertEqual(checksum, doc.checksum)
-

src/documents/tests/test_management_exporter.py

@@ -24,13 +24,20 @@ class TestExportImport(DirectoriesMixin, TestCase):
 
         file = os.path.join(self.dirs.originals_dir, "0000001.pdf")
 
-        Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", id=1, mime_type="application/pdf")
-        Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
-        Tag.objects.create(name="t")
-        DocumentType.objects.create(name="dt")
-        Correspondent.objects.create(name="c")
+        d1 = Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf")
+        d2 = Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
+        t1 = Tag.objects.create(name="t")
+        dt1 = DocumentType.objects.create(name="dt")
+        c1 = Correspondent.objects.create(name="c")
+
+        d1.tags.add(t1)
+        d1.correspondents = c1
+        d1.document_type = dt1
+        d1.save()
+        d2.save()
 
         target = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, target)
 
         call_command('document_exporter', target)
 
@@ -58,17 +65,28 @@ class TestExportImport(DirectoriesMixin, TestCase):
                     self.assertEqual(checksum, element['fields']['archive_checksum'])
 
         with paperless_environment() as dirs:
+            self.assertEqual(Document.objects.count(), 2)
+            Document.objects.all().delete()
+            Correspondent.objects.all().delete()
+            DocumentType.objects.all().delete()
+            Tag.objects.all().delete()
+            self.assertEqual(Document.objects.count(), 0)
+
             call_command('document_importer', target)
+            self.assertEqual(Document.objects.count(), 2)
             messages = check_sanity()
             # everything is alright after the test
             self.assertEqual(len(messages), 0, str([str(m) for m in messages]))
 
+    @override_settings(
+        PAPERLESS_FILENAME_FORMAT="{title}"
+    )
+    def test_exporter_with_filename_format(self):
+        self.test_exporter()
+
     def test_export_missing_files(self):
 
         target = tempfile.mkdtemp()
-        Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", id=3, mime_type="application/pdf")
+        self.addCleanup(shutil.rmtree, target)
+        Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", mime_type="application/pdf")
         self.assertRaises(FileNotFoundError, call_command, 'document_exporter', target)
-
-    def test_duplicate_titles(self):
-        # TODO
-        pass

src/documents/tests/test_management_retagger.py

@@ -14,6 +14,12 @@ class TestRetagger(DirectoriesMixin, TestCase):
 
         self.tag_first = Tag.objects.create(name="tag1", match="first", matching_algorithm=Tag.MATCH_ANY)
         self.tag_second = Tag.objects.create(name="tag2", match="second", matching_algorithm=Tag.MATCH_ANY)
+        self.tag_inbox = Tag.objects.create(name="test", is_inbox_tag=True)
+        self.tag_no_match = Tag.objects.create(name="test2")
+
+        self.d3.tags.add(self.tag_inbox)
+        self.d3.tags.add(self.tag_no_match)
+
 
         self.correspondent_first = Correspondent.objects.create(
             name="c1", match="first", matching_algorithm=Correspondent.MATCH_ANY)
@@ -38,7 +44,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
 
         self.assertEqual(d_first.tags.count(), 1)
         self.assertEqual(d_second.tags.count(), 1)
-        self.assertEqual(d_unrelated.tags.count(), 0)
+        self.assertEqual(d_unrelated.tags.count(), 2)
 
         self.assertEqual(d_first.tags.first(), self.tag_first)
         self.assertEqual(d_second.tags.first(), self.tag_second)
@@ -56,3 +62,17 @@ class TestRetagger(DirectoriesMixin, TestCase):
 
         self.assertEqual(d_first.correspondent, self.correspondent_first)
         self.assertEqual(d_second.correspondent, self.correspondent_second)
+
+    def test_overwrite_preserve_inbox(self):
+        self.d1.tags.add(self.tag_second)
+
+        call_command('document_retagger', '--tags', '--overwrite')
+
+        d_first, d_second, d_unrelated = self.get_updated_docs()
+
+        self.assertIsNotNone(Tag.objects.get(id=self.tag_second.id))
+
+        self.assertCountEqual([tag.id for tag in d_first.tags.all()], [self.tag_first.id])
+        self.assertCountEqual([tag.id for tag in d_second.tags.all()], [self.tag_second.id])
+        self.assertCountEqual([tag.id for tag in d_unrelated.tags.all()], [self.tag_inbox.id, self.tag_no_match.id])
+

src/documents/tests/test_migrations.py

@@ -0,0 +1,129 @@
+import os
+import shutil
+from pathlib import Path
+
+from django.apps import apps
+from django.conf import settings
+from django.db import connection
+from django.db.migrations.executor import MigrationExecutor
+from django.test import TestCase, TransactionTestCase, override_settings
+
+from documents.models import Document
+from documents.parsers import get_default_file_extension
+from documents.tests.utils import DirectoriesMixin
+
+
+class TestMigrations(TransactionTestCase):
+
+    @property
+    def app(self):
+        return apps.get_containing_app_config(type(self).__module__).name
+
+    migrate_from = None
+    migrate_to = None
+
+    def setUp(self):
+        super(TestMigrations, self).setUp()
+
+        assert self.migrate_from and self.migrate_to, \
+            "TestCase '{}' must define migrate_from and migrate_to     properties".format(type(self).__name__)
+        self.migrate_from = [(self.app, self.migrate_from)]
+        self.migrate_to = [(self.app, self.migrate_to)]
+        executor = MigrationExecutor(connection)
+        old_apps = executor.loader.project_state(self.migrate_from).apps
+
+        # Reverse to the original migration
+        executor.migrate(self.migrate_from)
+
+        self.setUpBeforeMigration(old_apps)
+
+        # Run the migration to test
+        executor = MigrationExecutor(connection)
+        executor.loader.build_graph()  # reload.
+        executor.migrate(self.migrate_to)
+
+        self.apps = executor.loader.project_state(self.migrate_to).apps
+
+    def setUpBeforeMigration(self, apps):
+        pass
+
+
+STORAGE_TYPE_UNENCRYPTED = "unencrypted"
+STORAGE_TYPE_GPG = "gpg"
+
+
+def source_path_before(self):
+    if self.filename:
+        fname = str(self.filename)
+    else:
+        fname = "{:07}.{}".format(self.pk, self.file_type)
+        if self.storage_type == STORAGE_TYPE_GPG:
+            fname += ".gpg"
+
+    return os.path.join(
+        settings.ORIGINALS_DIR,
+        fname
+    )
+
+
+def file_type_after(self):
+    return get_default_file_extension(self.mime_type)
+
+
+def source_path_after(doc):
+    if doc.filename:
+        fname = str(doc.filename)
+    else:
+        fname = "{:07}{}".format(doc.pk, file_type_after(doc))
+        if doc.storage_type == STORAGE_TYPE_GPG:
+            fname += ".gpg"  # pragma: no cover
+
+    return os.path.join(
+        settings.ORIGINALS_DIR,
+        fname
+    )
+
+
+@override_settings(PASSPHRASE="test")
+class TestMigrateMimeType(DirectoriesMixin, TestMigrations):
+
+    migrate_from = '1002_auto_20201111_1105'
+    migrate_to = '1003_mime_types'
+
+    def setUpBeforeMigration(self, apps):
+        Document = apps.get_model("documents", "Document")
+        doc = Document.objects.create(title="test", file_type="pdf", filename="file1.pdf")
+        self.doc_id = doc.id
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_before(doc))
+
+        doc2 = Document.objects.create(checksum="B", file_type="pdf", storage_type=STORAGE_TYPE_GPG)
+        self.doc2_id = doc2.id
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), source_path_before(doc2))
+
+    def testMimeTypesMigrated(self):
+        Document = self.apps.get_model('documents', 'Document')
+
+        doc = Document.objects.get(id=self.doc_id)
+        self.assertEqual(doc.mime_type, "application/pdf")
+
+        doc2 = Document.objects.get(id=self.doc2_id)
+        self.assertEqual(doc2.mime_type, "application/pdf")
+
+
+@override_settings(PASSPHRASE="test")
+class TestMigrateMimeTypeBackwards(DirectoriesMixin, TestMigrations):
+
+    migrate_from = '1003_mime_types'
+    migrate_to = '1002_auto_20201111_1105'
+
+    def setUpBeforeMigration(self, apps):
+        Document = apps.get_model("documents", "Document")
+        doc = Document.objects.create(title="test", mime_type="application/pdf", filename="file1.pdf")
+        self.doc_id = doc.id
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_after(doc))
+
+    def testMimeTypesReverted(self):
+        Document = self.apps.get_model('documents', 'Document')
+
+        doc = Document.objects.get(id=self.doc_id)
+        self.assertEqual(doc.file_type, "pdf")

src/documents/tests/test_post_consume_handlers.py

@@ -1,56 +0,0 @@
-from unittest import mock
-
-from django.test import TestCase, override_settings
-
-from documents.models import Document, Tag, Correspondent
-from documents.signals.handlers import run_post_consume_script
-
-
-class PostConsumeTestCase(TestCase):
-
-    @mock.patch("documents.signals.handlers.Popen")
-    @override_settings(POST_CONSUME_SCRIPT=None)
-    def test_no_post_consume_script(self, m):
-        doc = Document.objects.create(title="Test", mime_type="application/pdf")
-        tag1 = Tag.objects.create(name="a")
-        tag2 = Tag.objects.create(name="b")
-        doc.tags.add(tag1)
-        doc.tags.add(tag2)
-
-        run_post_consume_script(None, doc)
-
-        m.assert_not_called()
-
-    @mock.patch("documents.signals.handlers.Popen")
-    @override_settings(POST_CONSUME_SCRIPT="script")
-    def test_post_consume_script_simple(self, m):
-        doc = Document.objects.create(title="Test", mime_type="application/pdf")
-
-        run_post_consume_script(None, doc)
-
-        m.assert_called_once()
-
-    @mock.patch("documents.signals.handlers.Popen")
-    @override_settings(POST_CONSUME_SCRIPT="script")
-    def test_post_consume_script_with_correspondent(self, m):
-        c = Correspondent.objects.create(name="my_bank")
-        doc = Document.objects.create(title="Test", mime_type="application/pdf", correspondent=c)
-        tag1 = Tag.objects.create(name="a")
-        tag2 = Tag.objects.create(name="b")
-        doc.tags.add(tag1)
-        doc.tags.add(tag2)
-
-        run_post_consume_script(None, doc)
-
-        m.assert_called_once()
-
-        args, kwargs = m.call_args
-
-        command = args[0]
-
-        self.assertEqual(command[0], "script")
-        self.assertEqual(command[1], str(doc.pk))
-        self.assertEqual(command[5], f"/api/documents/{doc.pk}/download/")
-        self.assertEqual(command[6], f"/api/documents/{doc.pk}/thumb/")
-        self.assertEqual(command[7], "my_bank")
-        self.assertCountEqual(command[8].split(","), ["a", "b"])

src/documents/tests/test_sanity_check.py

@@ -2,6 +2,8 @@ import os
 import shutil
 from pathlib import Path
 
+import filelock
+from django.conf import settings
 from django.test import TestCase
 
 from documents.models import Document
@@ -13,9 +15,11 @@ class TestSanityCheck(DirectoriesMixin, TestCase):
 
     def make_test_data(self):
 
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000001.pdf"), os.path.join(self.dirs.originals_dir, "0000001.pdf"))
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "archive", "0000001.pdf"), os.path.join(self.dirs.archive_dir, "0000001.pdf"))
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000001.png"), os.path.join(self.dirs.thumbnail_dir, "0000001.png"))
+        with filelock.FileLock(settings.MEDIA_LOCK):
+            # just make sure that the lockfile is present.
+            shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000001.pdf"), os.path.join(self.dirs.originals_dir, "0000001.pdf"))
+            shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "archive", "0000001.pdf"), os.path.join(self.dirs.archive_dir, "0000001.pdf"))
+            shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000001.png"), os.path.join(self.dirs.thumbnail_dir, "0000001.png"))
 
         return Document.objects.create(title="test", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", content="test", pk=1, filename="0000001.pdf", mime_type="application/pdf")
 

src/documents/tests/test_tasks.py

@@ -1,10 +1,12 @@
 from datetime import datetime
+from unittest import mock
 
 from django.test import TestCase
 from django.utils import timezone
 
 from documents import tasks
 from documents.models import Document
+from documents.sanity_checker import SanityError, SanityFailedError
 from documents.tests.utils import DirectoriesMixin
 
 
@@ -22,3 +24,19 @@ class TestTasks(DirectoriesMixin, TestCase):
 
     def test_train_classifier(self):
         tasks.train_classifier()
+
+    @mock.patch("documents.tasks.sanity_checker.check_sanity")
+    def test_sanity_check(self, m):
+        m.return_value = []
+        tasks.sanity_check()
+        m.assert_called_once()
+        m.reset_mock()
+        m.return_value = [SanityError("")]
+        self.assertRaises(SanityFailedError, tasks.sanity_check)
+        m.assert_called_once()
+
+    def test_culk_update_documents(self):
+        doc1 = Document.objects.create(title="test", content="my document", checksum="wow", added=timezone.now(),
+                                created=timezone.now(), modified=timezone.now())
+
+        tasks.bulk_update_documents([doc1.pk])

src/documents/tests/utils.py

@@ -34,7 +34,8 @@ def setup_directories():
         ARCHIVE_DIR=dirs.archive_dir,
         CONSUMPTION_DIR=dirs.consumption_dir,
         INDEX_DIR=dirs.index_dir,
-        MODEL_FILE=os.path.join(dirs.data_dir, "classification_model.pickle")
+        MODEL_FILE=os.path.join(dirs.data_dir, "classification_model.pickle"),
+        MEDIA_LOCK=os.path.join(dirs.media_dir, "media.lock")
 
     )
     dirs.settings_override.enable()

src/documents/views.py

@@ -4,8 +4,10 @@ from datetime import datetime
 from time import mktime
 
 from django.conf import settings
-from django.db.models import Count, Max
+from django.db.models import Count, Max, Case, When, IntegerField
+from django.db.models.functions import Lower
 from django.http import HttpResponse, HttpResponseBadRequest, Http404
+from django.utils.translation import get_language
 from django.views.decorators.cache import cache_control
 from django.views.generic import TemplateView
 from django_filters.rest_framework import DjangoFilterBackend
@@ -38,27 +40,55 @@ from .filters import (
     DocumentTypeFilterSet,
     LogFilterSet
 )
-from .models import Correspondent, Document, Log, Tag, DocumentType
+from .models import Correspondent, Document, Log, Tag, DocumentType, SavedView
+from .parsers import get_parser_class_for_mime_type
 from .serialisers import (
     CorrespondentSerializer,
     DocumentSerializer,
     LogSerializer,
     TagSerializer,
     DocumentTypeSerializer,
-    PostDocumentSerializer
+    PostDocumentSerializer,
+    SavedViewSerializer,
+    BulkEditSerializer, SelectionDataSerializer
 )
 
 
 class IndexView(TemplateView):
     template_name = "index.html"
 
+    def get_language(self):
+        # This is here for the following reason:
+        # Django identifies languages in the form "en-us"
+        # However, angular generates locales as "en-US".
+        # this translates between these two forms.
+        lang = get_language()
+        if "-" in lang:
+            first = lang[:lang.index("-")]
+            second = lang[lang.index("-")+1:]
+            return f"{first}-{second.upper()}"
+        else:
+            return lang
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context['cookie_prefix'] = settings.COOKIE_PREFIX
+        context['username'] = self.request.user.username
+        context['full_name'] = self.request.user.get_full_name()
+        context['styles_css'] = f"frontend/{self.get_language()}/styles.css"
+        context['runtime_js'] = f"frontend/{self.get_language()}/runtime.js"
+        context['polyfills_js'] = f"frontend/{self.get_language()}/polyfills.js"  # NOQA: E501
+        context['main_js'] = f"frontend/{self.get_language()}/main.js"
+        context['manifest'] = f"frontend/{self.get_language()}/manifest.webmanifest"  # NOQA: E501
+        return context
+
 
 class CorrespondentViewSet(ModelViewSet):
     model = Correspondent
 
     queryset = Correspondent.objects.annotate(
         document_count=Count('documents'),
-        last_correspondence=Max('documents__created')).order_by('name')
+        last_correspondence=Max('documents__created')).order_by(Lower('name'))
 
     serializer_class = CorrespondentSerializer
     pagination_class = StandardPagination
@@ -77,7 +107,7 @@ class TagViewSet(ModelViewSet):
     model = Tag
 
     queryset = Tag.objects.annotate(
-        document_count=Count('documents')).order_by('name')
+        document_count=Count('documents')).order_by(Lower('name'))
 
     serializer_class = TagSerializer
     pagination_class = StandardPagination
@@ -91,7 +121,7 @@ class DocumentTypeViewSet(ModelViewSet):
     model = DocumentType
 
     queryset = DocumentType.objects.annotate(
-        document_count=Count('documents')).order_by('name')
+        document_count=Count('documents')).order_by(Lower('name'))
 
     serializer_class = DocumentTypeSerializer
     pagination_class = StandardPagination
@@ -101,6 +131,10 @@ class DocumentTypeViewSet(ModelViewSet):
     ordering_fields = ("name", "matching_algorithm", "match", "document_count")
 
 
+class BulkEditForm(object):
+    pass
+
+
 class DocumentViewSet(RetrieveModelMixin,
                       UpdateModelMixin,
                       DestroyModelMixin,
@@ -124,6 +158,17 @@ class DocumentViewSet(RetrieveModelMixin,
         "added",
         "archive_serial_number")
 
+    def get_serializer(self, *args, **kwargs):
+        fields_param = self.request.query_params.get('fields', None)
+        if fields_param:
+            fields = fields_param.split(",")
+        else:
+            fields = None
+        serializer_class = self.get_serializer_class()
+        kwargs.setdefault('context', self.get_serializer_context())
+        kwargs.setdefault('fields', fields)
+        return serializer_class(*args, **kwargs)
+
     def update(self, request, *args, **kwargs):
         response = super(DocumentViewSet, self).update(
             request, *args, **kwargs)
@@ -160,17 +205,48 @@ class DocumentViewSet(RetrieveModelMixin,
             disposition, filename)
         return response
 
+    def get_metadata(self, file, mime_type):
+        if not os.path.isfile(file):
+            return None
+
+        parser_class = get_parser_class_for_mime_type(mime_type)
+        if parser_class:
+            parser = parser_class(logging_group=None)
+
+            try:
+                return parser.extract_metadata(file, mime_type)
+            except Exception as e:
+                # TODO: cover GPG errors, remove later.
+                return []
+        else:
+            return []
+
     @action(methods=['get'], detail=True)
     def metadata(self, request, pk=None):
         try:
             doc = Document.objects.get(pk=pk)
-            return Response({
-                "paperless__checksum": doc.checksum,
-                "paperless__mime_type": doc.mime_type,
-                "paperless__filename": doc.filename,
-                "paperless__has_archive_version":
-                    os.path.isfile(doc.archive_path)
-            })
+
+            meta = {
+                "original_checksum": doc.checksum,
+                "original_size": os.stat(doc.source_path).st_size,
+                "original_mime_type": doc.mime_type,
+                "media_filename": doc.filename,
+                "has_archive_version": os.path.isfile(doc.archive_path),
+                "original_metadata": self.get_metadata(
+                    doc.source_path, doc.mime_type)
+            }
+
+            if doc.archive_checksum and os.path.isfile(doc.archive_path):
+                meta['archive_checksum'] = doc.archive_checksum
+                meta['archive_size'] = os.stat(doc.archive_path).st_size,
+                meta['archive_metadata'] = self.get_metadata(
+                    doc.archive_path, "application/pdf")
+            else:
+                meta['archive_checksum'] = None
+                meta['archive_size'] = None
+                meta['archive_metadata'] = None
+
+            return Response(meta)
         except Document.DoesNotExist:
             raise Http404()
 
@@ -187,7 +263,12 @@ class DocumentViewSet(RetrieveModelMixin,
     @cache_control(public=False, max_age=315360000)
     def thumb(self, request, pk=None):
         try:
-            return HttpResponse(Document.objects.get(id=pk).thumbnail_file,
+            doc = Document.objects.get(id=pk)
+            if doc.storage_type == Document.STORAGE_TYPE_GPG:
+                handle = GnuPG.decrypted(doc.thumbnail_file)
+            else:
+                handle = doc.thumbnail_file
+            return HttpResponse(handle,
                                 content_type='image/png')
         except (FileNotFoundError, Document.DoesNotExist):
             raise Http404()
@@ -213,6 +294,55 @@ class LogViewSet(ReadOnlyModelViewSet):
     ordering_fields = ("created",)
 
 
+class SavedViewViewSet(ModelViewSet):
+    model = SavedView
+
+    queryset = SavedView.objects.all()
+    serializer_class = SavedViewSerializer
+    pagination_class = StandardPagination
+    permission_classes = (IsAuthenticated,)
+
+    def get_queryset(self):
+        user = self.request.user
+        return SavedView.objects.filter(user=user)
+
+    def perform_create(self, serializer):
+        serializer.save(user=self.request.user)
+
+
+class BulkEditView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    serializer_class = BulkEditSerializer
+    parser_classes = (parsers.JSONParser,)
+
+    def get_serializer_context(self):
+        return {
+            'request': self.request,
+            'format': self.format_kwarg,
+            'view': self
+        }
+
+    def get_serializer(self, *args, **kwargs):
+        kwargs['context'] = self.get_serializer_context()
+        return self.serializer_class(*args, **kwargs)
+
+    def post(self, request, *args, **kwargs):
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+
+        method = serializer.validated_data.get("method")
+        parameters = serializer.validated_data.get("parameters")
+        documents = serializer.validated_data.get("documents")
+
+        try:
+            # TODO: parameter validation
+            result = method(documents, **parameters)
+            return Response({"result": result})
+        except Exception as e:
+            return HttpResponseBadRequest(str(e))
+
+
 class PostDocumentView(APIView):
 
     permission_classes = (IsAuthenticated,)
@@ -262,6 +392,63 @@ class PostDocumentView(APIView):
         return Response("OK")
 
 
+class SelectionDataView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    serializer_class = SelectionDataSerializer
+    parser_classes = (parsers.MultiPartParser, parsers.JSONParser)
+
+    def get_serializer_context(self):
+        return {
+            'request': self.request,
+            'format': self.format_kwarg,
+            'view': self
+        }
+
+    def get_serializer(self, *args, **kwargs):
+        kwargs['context'] = self.get_serializer_context()
+        return self.serializer_class(*args, **kwargs)
+
+    def post(self, request, format=None):
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+
+        ids = serializer.validated_data.get('documents')
+
+        correspondents = Correspondent.objects.annotate(
+            document_count=Count(Case(
+                When(documents__id__in=ids, then=1),
+                output_field=IntegerField()
+            )))
+
+        tags = Tag.objects.annotate(document_count=Count(Case(
+            When(documents__id__in=ids, then=1),
+            output_field=IntegerField()
+        )))
+
+        types = DocumentType.objects.annotate(document_count=Count(Case(
+            When(documents__id__in=ids, then=1),
+            output_field=IntegerField()
+        )))
+
+        r = Response({
+            "selected_correspondents": [{
+                "id": t.id,
+                "document_count": t.document_count
+            } for t in correspondents],
+            "selected_tags": [{
+                "id": t.id,
+                "document_count": t.document_count
+            } for t in tags],
+            "selected_document_types": [{
+                "id": t.id,
+                "document_count": t.document_count
+            } for t in types]
+        })
+
+        return r
+
+
 class SearchView(APIView):
 
     permission_classes = (IsAuthenticated,)
@@ -281,14 +468,27 @@ class SearchView(APIView):
                 }
 
     def get(self, request, format=None):
-        if 'query' not in request.query_params:
+
+        if 'query' in request.query_params:
+            query = request.query_params['query']
+        else:
+            query = None
+
+        if 'more_like' in request.query_params:
+            more_like_id = request.query_params['more_like']
+            more_like_content = Document.objects.get(id=more_like_id).content
+        else:
+            more_like_id = None
+            more_like_content = None
+
+        if not query and not more_like_id:
             return Response({
                 'count': 0,
                 'page': 0,
                 'page_count': 0,
+                'corrected_query': None,
                 'results': []})
 
-        query = request.query_params['query']
         try:
             page = int(request.query_params.get('page', 1))
         except (ValueError, TypeError):
@@ -298,8 +498,7 @@ class SearchView(APIView):
             page = 1
 
         try:
-            with index.query_page(self.ix, query, page) as (result_page,
-                                                            corrected_query):
+            with index.query_page(self.ix, page, query, more_like_id, more_like_content) as (result_page, corrected_query):  # NOQA: E501
                 return Response(
                     {'count': len(result_page),
                      'page': result_page.pagenum,

src/locale/de/LC_MESSAGES/django.po

@@ -0,0 +1,567 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+# 
+# Translators:
+# Jonas Winkler <dev@jpwinkler.de>, 2021
+# 
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2021-01-02 00:26+0000\n"
+"PO-Revision-Date: 2020-12-30 19:27+0000\n"
+"Last-Translator: Jonas Winkler <dev@jpwinkler.de>, 2021\n"
+"Language-Team: German (https://www.transifex.com/paperless/teams/115905/de/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: de\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: documents/apps.py:10
+msgid "Documents"
+msgstr "Dokumente"
+
+#: documents/models.py:32
+msgid "Any word"
+msgstr "Irgendein Wort"
+
+#: documents/models.py:33
+msgid "All words"
+msgstr "Alle Wörter"
+
+#: documents/models.py:34
+msgid "Exact match"
+msgstr "Exakte Übereinstimmung"
+
+#: documents/models.py:35
+msgid "Regular expression"
+msgstr "Regulärer Ausdruck"
+
+#: documents/models.py:36
+msgid "Fuzzy word"
+msgstr "Ungenaues Wort"
+
+#: documents/models.py:37
+msgid "Automatic"
+msgstr "Automatisch"
+
+#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
+#: paperless_mail/models.py:100
+msgid "name"
+msgstr "Name"
+
+#: documents/models.py:45
+msgid "match"
+msgstr "Zuweisungsmuster"
+
+#: documents/models.py:49
+msgid "matching algorithm"
+msgstr "Zuweisungsalgorithmus"
+
+#: documents/models.py:55
+msgid "is insensitive"
+msgstr "Groß-/Kleinschreibung irrelevant"
+
+#: documents/models.py:80 documents/models.py:140
+msgid "correspondent"
+msgstr "Korrespondent"
+
+#: documents/models.py:81
+msgid "correspondents"
+msgstr "Korrespondenten"
+
+#: documents/models.py:103
+msgid "color"
+msgstr "Farbe"
+
+#: documents/models.py:107
+msgid "is inbox tag"
+msgstr "Posteingangs-Tag"
+
+#: documents/models.py:109
+msgid ""
+"Marks this tag as an inbox tag: All newly consumed documents will be tagged "
+"with inbox tags."
+msgstr ""
+"Markiert das Tag als Posteingangs-Tag. Neue Dokumente werden immer mit "
+"diesem Tag versehen."
+
+#: documents/models.py:114
+msgid "tag"
+msgstr "Tag"
+
+#: documents/models.py:115 documents/models.py:171
+msgid "tags"
+msgstr "Tags"
+
+#: documents/models.py:121 documents/models.py:153
+msgid "document type"
+msgstr "Dokumenttyp"
+
+#: documents/models.py:122
+msgid "document types"
+msgstr "Dokumenttypen"
+
+#: documents/models.py:130
+msgid "Unencrypted"
+msgstr "Nicht verschlüsselt"
+
+#: documents/models.py:131
+msgid "Encrypted with GNU Privacy Guard"
+msgstr "Verschlüsselt mit GNU Privacy Guard"
+
+#: documents/models.py:144
+msgid "title"
+msgstr "Titel"
+
+#: documents/models.py:157
+msgid "content"
+msgstr "Inhalt"
+
+#: documents/models.py:159
+msgid ""
+"The raw, text-only data of the document. This field is primarily used for "
+"searching."
+msgstr ""
+"Der Inhalt des Dokuments in Textform. Dieses Feld wird primär für die Suche "
+"verwendet."
+
+#: documents/models.py:164
+msgid "mime type"
+msgstr "MIME-Typ"
+
+#: documents/models.py:175
+msgid "checksum"
+msgstr "Prüfsumme"
+
+#: documents/models.py:179
+msgid "The checksum of the original document."
+msgstr "Die Prüfsumme des originalen Dokuments."
+
+#: documents/models.py:183
+msgid "archive checksum"
+msgstr "Archiv-Prüfsumme"
+
+#: documents/models.py:188
+msgid "The checksum of the archived document."
+msgstr "Die Prüfsumme des archivierten Dokuments."
+
+#: documents/models.py:192 documents/models.py:332
+msgid "created"
+msgstr "Erstellt"
+
+#: documents/models.py:196
+msgid "modified"
+msgstr "Geändert"
+
+#: documents/models.py:200
+msgid "storage type"
+msgstr "Speichertyp"
+
+#: documents/models.py:208
+msgid "added"
+msgstr "Hinzugefügt"
+
+#: documents/models.py:212
+msgid "filename"
+msgstr "Dateiname"
+
+#: documents/models.py:217
+msgid "Current filename in storage"
+msgstr "Aktueller Dateiname im Datenspeicher"
+
+#: documents/models.py:221
+msgid "archive serial number"
+msgstr "Archiv-Seriennummer"
+
+#: documents/models.py:226
+msgid "The position of this document in your physical document archive."
+msgstr "Die Position dieses Dokuments in Ihrem physischen Dokumentenarchiv."
+
+#: documents/models.py:232
+msgid "document"
+msgstr "Dokument"
+
+#: documents/models.py:233
+msgid "documents"
+msgstr "Dokumente"
+
+#: documents/models.py:315
+msgid "debug"
+msgstr "Debug"
+
+#: documents/models.py:316
+msgid "information"
+msgstr "Information"
+
+#: documents/models.py:317
+msgid "warning"
+msgstr "Warnung"
+
+#: documents/models.py:318
+msgid "error"
+msgstr "Fehler"
+
+#: documents/models.py:319
+msgid "critical"
+msgstr "Kritisch"
+
+#: documents/models.py:323
+msgid "group"
+msgstr "Gruppe"
+
+#: documents/models.py:326
+msgid "message"
+msgstr "Nachricht"
+
+#: documents/models.py:329
+msgid "level"
+msgstr "Level"
+
+#: documents/models.py:336
+msgid "log"
+msgstr "Protokoll"
+
+#: documents/models.py:337
+msgid "logs"
+msgstr "Protokoll"
+
+#: documents/models.py:348 documents/models.py:398
+msgid "saved view"
+msgstr "Gespeicherte Ansicht"
+
+#: documents/models.py:349
+msgid "saved views"
+msgstr "Gespeicherte Ansichten"
+
+#: documents/models.py:352
+msgid "user"
+msgstr "Benutzer"
+
+#: documents/models.py:358
+msgid "show on dashboard"
+msgstr "Auf Startseite zeigen"
+
+#: documents/models.py:361
+msgid "show in sidebar"
+msgstr "In Seitenleiste zeigen"
+
+#: documents/models.py:365
+msgid "sort field"
+msgstr "Sortierfeld"
+
+#: documents/models.py:368
+msgid "sort reverse"
+msgstr "Umgekehrte Sortierung"
+
+#: documents/models.py:374
+msgid "title contains"
+msgstr "Titel enthält"
+
+#: documents/models.py:375
+msgid "content contains"
+msgstr "Inhalt enthält"
+
+#: documents/models.py:376
+msgid "ASN is"
+msgstr "ASN ist"
+
+#: documents/models.py:377
+msgid "correspondent is"
+msgstr "Korrespondent ist"
+
+#: documents/models.py:378
+msgid "document type is"
+msgstr "Dokumenttyp ist"
+
+#: documents/models.py:379
+msgid "is in inbox"
+msgstr "Ist im Posteingang"
+
+#: documents/models.py:380
+msgid "has tag"
+msgstr "Hat Tag"
+
+#: documents/models.py:381
+msgid "has any tag"
+msgstr "Hat irgendein Tag"
+
+#: documents/models.py:382
+msgid "created before"
+msgstr "Erstellt vor"
+
+#: documents/models.py:383
+msgid "created after"
+msgstr "Erstellt nach"
+
+#: documents/models.py:384
+msgid "created year is"
+msgstr "Erstellt im Jahr"
+
+#: documents/models.py:385
+msgid "created month is"
+msgstr "Erstellt im Monat"
+
+#: documents/models.py:386
+msgid "created day is"
+msgstr "Erstellt am Tag"
+
+#: documents/models.py:387
+msgid "added before"
+msgstr "Hinzugefügt vor"
+
+#: documents/models.py:388
+msgid "added after"
+msgstr "Hinzugefügt nach"
+
+#: documents/models.py:389
+msgid "modified before"
+msgstr "Geändert vor"
+
+#: documents/models.py:390
+msgid "modified after"
+msgstr "Geändert nach"
+
+#: documents/models.py:391
+msgid "does not have tag"
+msgstr "Hat nicht folgendes Tag"
+
+#: documents/models.py:402
+msgid "rule type"
+msgstr "Regeltyp"
+
+#: documents/models.py:406
+msgid "value"
+msgstr "Wert"
+
+#: documents/models.py:412
+msgid "filter rule"
+msgstr "Filterregel"
+
+#: documents/models.py:413
+msgid "filter rules"
+msgstr "Filterregeln"
+
+#: paperless/settings.py:254
+msgid "English"
+msgstr "Englisch"
+
+#: paperless/settings.py:255
+msgid "German"
+msgstr "Deutsch"
+
+#: paperless/urls.py:108
+msgid "Paperless-ng administration"
+msgstr "Paperless-ng Administration"
+
+#: paperless_mail/admin.py:24
+msgid "Filter"
+msgstr "Filter"
+
+#: paperless_mail/admin.py:26
+msgid ""
+"Paperless will only process mails that match ALL of the filters given below."
+msgstr ""
+"Paperless wird nur E-Mails verarbeiten, für die alle der hier angegebenen "
+"Filter zutreffen."
+
+#: paperless_mail/admin.py:34
+msgid "Actions"
+msgstr "Aktionen"
+
+#: paperless_mail/admin.py:36
+msgid ""
+"The action applied to the mail. This action is only performed when documents"
+" were consumed from the mail. Mails without attachments will remain entirely"
+" untouched."
+msgstr ""
+"Die Aktion, die auf E-Mails angewendet werden soll. Diese Aktion wird nur "
+"auf E-Mails angewendet, aus denen Anhänge verarbeitet wurden. E-Mails ohne "
+"Anhänge werden vollständig ignoriert."
+
+#: paperless_mail/admin.py:43
+msgid "Metadata"
+msgstr "Metadaten"
+
+#: paperless_mail/admin.py:45
+msgid ""
+"Assign metadata to documents consumed from this rule automatically. If you "
+"do not assign tags, types or correspondents here, paperless will still "
+"process all matching rules that you have defined."
+msgstr ""
+"Folgende Metadaten werden Dokumenten dieser Regel automatisch zugewiesen. "
+"Wenn Sie hier nichts auswählen wird Paperless weiterhin alle "
+"Zuweisungsalgorithmen ausführen und Metadaten auf Basis des Dokumentinhalts "
+"zuweisen."
+
+#: paperless_mail/apps.py:9
+msgid "Paperless mail"
+msgstr "Paperless E-Mail"
+
+#: paperless_mail/models.py:11
+msgid "mail account"
+msgstr "E-Mail-Konto"
+
+#: paperless_mail/models.py:12
+msgid "mail accounts"
+msgstr "E-Mail-Konten"
+
+#: paperless_mail/models.py:19
+msgid "No encryption"
+msgstr "Keine Verschlüsselung"
+
+#: paperless_mail/models.py:20
+msgid "Use SSL"
+msgstr "SSL benutzen"
+
+#: paperless_mail/models.py:21
+msgid "Use STARTTLS"
+msgstr "STARTTLS benutzen"
+
+#: paperless_mail/models.py:29
+msgid "IMAP server"
+msgstr "IMAP-Server"
+
+#: paperless_mail/models.py:33
+msgid "IMAP port"
+msgstr "IMAP-Port"
+
+#: paperless_mail/models.py:36
+msgid ""
+"This is usually 143 for unencrypted and STARTTLS connections, and 993 for "
+"SSL connections."
+msgstr ""
+"Dies ist in der Regel 143 für unverschlüsselte und STARTTLS-Verbindungen und"
+" 993 für SSL-Verbindungen."
+
+#: paperless_mail/models.py:40
+msgid "IMAP security"
+msgstr "IMAP-Sicherheit"
+
+#: paperless_mail/models.py:46
+msgid "username"
+msgstr "Benutzername"
+
+#: paperless_mail/models.py:50
+msgid "password"
+msgstr "Password"
+
+#: paperless_mail/models.py:60
+msgid "mail rule"
+msgstr "E-Mail-Regel"
+
+#: paperless_mail/models.py:61
+msgid "mail rules"
+msgstr "E-Mail-Regeln"
+
+#: paperless_mail/models.py:69
+msgid "Mark as read, don't process read mails"
+msgstr "Als gelesen markieren, gelesene E-Mails nicht verarbeiten"
+
+#: paperless_mail/models.py:70
+msgid "Flag the mail, don't process flagged mails"
+msgstr "Als wichtig markieren, markierte E-Mails nicht verarbeiten"
+
+#: paperless_mail/models.py:71
+msgid "Move to specified folder"
+msgstr "In angegebenen Ordner verschieben"
+
+#: paperless_mail/models.py:72
+msgid "Delete"
+msgstr "Löschen"
+
+#: paperless_mail/models.py:79
+msgid "Use subject as title"
+msgstr "Betreff als Titel verwenden"
+
+#: paperless_mail/models.py:80
+msgid "Use attachment filename as title"
+msgstr "Dateiname des Anhangs als Titel verwenden"
+
+#: paperless_mail/models.py:90
+msgid "Do not assign a correspondent"
+msgstr "Keinen Korrespondenten zuweisen"
+
+#: paperless_mail/models.py:92
+msgid "Use mail address"
+msgstr "E-Mail-Adresse benutzen"
+
+#: paperless_mail/models.py:94
+msgid "Use name (or mail address if not available)"
+msgstr "Absendername benutzen (oder E-Mail-Adressen, wenn nicht verfügbar)"
+
+#: paperless_mail/models.py:96
+msgid "Use correspondent selected below"
+msgstr "Nachfolgend ausgewählten Korrespondent verwenden"
+
+#: paperless_mail/models.py:104
+msgid "order"
+msgstr "Reihenfolge"
+
+#: paperless_mail/models.py:111
+msgid "account"
+msgstr "Konto"
+
+#: paperless_mail/models.py:115
+msgid "folder"
+msgstr "Ordner"
+
+#: paperless_mail/models.py:119
+msgid "filter from"
+msgstr "Absender filtern"
+
+#: paperless_mail/models.py:122
+msgid "filter subject"
+msgstr "Betreff filtern"
+
+#: paperless_mail/models.py:125
+msgid "filter body"
+msgstr "Nachrichteninhalt filtern"
+
+#: paperless_mail/models.py:129
+msgid "maximum age"
+msgstr "Maximales Alter"
+
+#: paperless_mail/models.py:131
+msgid "Specified in days."
+msgstr "Angegeben in Tagen."
+
+#: paperless_mail/models.py:134
+msgid "action"
+msgstr "Aktion"
+
+#: paperless_mail/models.py:140
+msgid "action parameter"
+msgstr "Parameter für Aktion"
+
+#: paperless_mail/models.py:142
+msgid ""
+"Additional parameter for the action selected above, i.e., the target folder "
+"of the move to folder action."
+msgstr ""
+"Zusätzlicher Parameter für die oben ausgewählte Aktion, zum Beispiel der "
+"Zielordner für die Aktion \"In angegebenen Ordner verschieben\""
+
+#: paperless_mail/models.py:148
+msgid "assign title from"
+msgstr "Titel zuweisen von"
+
+#: paperless_mail/models.py:158
+msgid "assign this tag"
+msgstr "Dieses Tag zuweisen"
+
+#: paperless_mail/models.py:166
+msgid "assign this document type"
+msgstr "Diesen Dokumenttyp zuweisen"
+
+#: paperless_mail/models.py:170
+msgid "assign correspondent from"
+msgstr "Korrespondent zuweisen von"
+
+#: paperless_mail/models.py:180
+msgid "assign this correspondent"
+msgstr "Diesen Korrespondent zuweisen"

src/locale/en-us/LC_MESSAGES/django.po

@@ -0,0 +1,546 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2021-01-02 00:26+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: documents/apps.py:10
+msgid "Documents"
+msgstr ""
+
+#: documents/models.py:32
+msgid "Any word"
+msgstr ""
+
+#: documents/models.py:33
+msgid "All words"
+msgstr ""
+
+#: documents/models.py:34
+msgid "Exact match"
+msgstr ""
+
+#: documents/models.py:35
+msgid "Regular expression"
+msgstr ""
+
+#: documents/models.py:36
+msgid "Fuzzy word"
+msgstr ""
+
+#: documents/models.py:37
+msgid "Automatic"
+msgstr ""
+
+#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
+#: paperless_mail/models.py:100
+msgid "name"
+msgstr ""
+
+#: documents/models.py:45
+msgid "match"
+msgstr ""
+
+#: documents/models.py:49
+msgid "matching algorithm"
+msgstr ""
+
+#: documents/models.py:55
+msgid "is insensitive"
+msgstr ""
+
+#: documents/models.py:80 documents/models.py:140
+msgid "correspondent"
+msgstr ""
+
+#: documents/models.py:81
+msgid "correspondents"
+msgstr ""
+
+#: documents/models.py:103
+msgid "color"
+msgstr ""
+
+#: documents/models.py:107
+msgid "is inbox tag"
+msgstr ""
+
+#: documents/models.py:109
+msgid ""
+"Marks this tag as an inbox tag: All newly consumed documents will be tagged "
+"with inbox tags."
+msgstr ""
+
+#: documents/models.py:114
+msgid "tag"
+msgstr ""
+
+#: documents/models.py:115 documents/models.py:171
+msgid "tags"
+msgstr ""
+
+#: documents/models.py:121 documents/models.py:153
+msgid "document type"
+msgstr ""
+
+#: documents/models.py:122
+msgid "document types"
+msgstr ""
+
+#: documents/models.py:130
+msgid "Unencrypted"
+msgstr ""
+
+#: documents/models.py:131
+msgid "Encrypted with GNU Privacy Guard"
+msgstr ""
+
+#: documents/models.py:144
+msgid "title"
+msgstr ""
+
+#: documents/models.py:157
+msgid "content"
+msgstr ""
+
+#: documents/models.py:159
+msgid ""
+"The raw, text-only data of the document. This field is primarily used for "
+"searching."
+msgstr ""
+
+#: documents/models.py:164
+msgid "mime type"
+msgstr ""
+
+#: documents/models.py:175
+msgid "checksum"
+msgstr ""
+
+#: documents/models.py:179
+msgid "The checksum of the original document."
+msgstr ""
+
+#: documents/models.py:183
+msgid "archive checksum"
+msgstr ""
+
+#: documents/models.py:188
+msgid "The checksum of the archived document."
+msgstr ""
+
+#: documents/models.py:192 documents/models.py:332
+msgid "created"
+msgstr ""
+
+#: documents/models.py:196
+msgid "modified"
+msgstr ""
+
+#: documents/models.py:200
+msgid "storage type"
+msgstr ""
+
+#: documents/models.py:208
+msgid "added"
+msgstr ""
+
+#: documents/models.py:212
+msgid "filename"
+msgstr ""
+
+#: documents/models.py:217
+msgid "Current filename in storage"
+msgstr ""
+
+#: documents/models.py:221
+msgid "archive serial number"
+msgstr ""
+
+#: documents/models.py:226
+msgid "The position of this document in your physical document archive."
+msgstr ""
+
+#: documents/models.py:232
+msgid "document"
+msgstr ""
+
+#: documents/models.py:233
+msgid "documents"
+msgstr ""
+
+#: documents/models.py:315
+msgid "debug"
+msgstr ""
+
+#: documents/models.py:316
+msgid "information"
+msgstr ""
+
+#: documents/models.py:317
+msgid "warning"
+msgstr ""
+
+#: documents/models.py:318
+msgid "error"
+msgstr ""
+
+#: documents/models.py:319
+msgid "critical"
+msgstr ""
+
+#: documents/models.py:323
+msgid "group"
+msgstr ""
+
+#: documents/models.py:326
+msgid "message"
+msgstr ""
+
+#: documents/models.py:329
+msgid "level"
+msgstr ""
+
+#: documents/models.py:336
+msgid "log"
+msgstr ""
+
+#: documents/models.py:337
+msgid "logs"
+msgstr ""
+
+#: documents/models.py:348 documents/models.py:398
+msgid "saved view"
+msgstr ""
+
+#: documents/models.py:349
+msgid "saved views"
+msgstr ""
+
+#: documents/models.py:352
+msgid "user"
+msgstr ""
+
+#: documents/models.py:358
+msgid "show on dashboard"
+msgstr ""
+
+#: documents/models.py:361
+msgid "show in sidebar"
+msgstr ""
+
+#: documents/models.py:365
+msgid "sort field"
+msgstr ""
+
+#: documents/models.py:368
+msgid "sort reverse"
+msgstr ""
+
+#: documents/models.py:374
+msgid "title contains"
+msgstr ""
+
+#: documents/models.py:375
+msgid "content contains"
+msgstr ""
+
+#: documents/models.py:376
+msgid "ASN is"
+msgstr ""
+
+#: documents/models.py:377
+msgid "correspondent is"
+msgstr ""
+
+#: documents/models.py:378
+msgid "document type is"
+msgstr ""
+
+#: documents/models.py:379
+msgid "is in inbox"
+msgstr ""
+
+#: documents/models.py:380
+msgid "has tag"
+msgstr ""
+
+#: documents/models.py:381
+msgid "has any tag"
+msgstr ""
+
+#: documents/models.py:382
+msgid "created before"
+msgstr ""
+
+#: documents/models.py:383
+msgid "created after"
+msgstr ""
+
+#: documents/models.py:384
+msgid "created year is"
+msgstr ""
+
+#: documents/models.py:385
+msgid "created month is"
+msgstr ""
+
+#: documents/models.py:386
+msgid "created day is"
+msgstr ""
+
+#: documents/models.py:387
+msgid "added before"
+msgstr ""
+
+#: documents/models.py:388
+msgid "added after"
+msgstr ""
+
+#: documents/models.py:389
+msgid "modified before"
+msgstr ""
+
+#: documents/models.py:390
+msgid "modified after"
+msgstr ""
+
+#: documents/models.py:391
+msgid "does not have tag"
+msgstr ""
+
+#: documents/models.py:402
+msgid "rule type"
+msgstr ""
+
+#: documents/models.py:406
+msgid "value"
+msgstr ""
+
+#: documents/models.py:412
+msgid "filter rule"
+msgstr ""
+
+#: documents/models.py:413
+msgid "filter rules"
+msgstr ""
+
+#: paperless/settings.py:254
+msgid "English"
+msgstr ""
+
+#: paperless/settings.py:255
+msgid "German"
+msgstr ""
+
+#: paperless/urls.py:108
+msgid "Paperless-ng administration"
+msgstr ""
+
+#: paperless_mail/admin.py:24
+msgid "Filter"
+msgstr ""
+
+#: paperless_mail/admin.py:26
+msgid ""
+"Paperless will only process mails that match ALL of the filters given below."
+msgstr ""
+
+#: paperless_mail/admin.py:34
+msgid "Actions"
+msgstr ""
+
+#: paperless_mail/admin.py:36
+msgid ""
+"The action applied to the mail. This action is only performed when documents "
+"were consumed from the mail. Mails without attachments will remain entirely "
+"untouched."
+msgstr ""
+
+#: paperless_mail/admin.py:43
+msgid "Metadata"
+msgstr ""
+
+#: paperless_mail/admin.py:45
+msgid ""
+"Assign metadata to documents consumed from this rule automatically. If you "
+"do not assign tags, types or correspondents here, paperless will still "
+"process all matching rules that you have defined."
+msgstr ""
+
+#: paperless_mail/apps.py:9
+msgid "Paperless mail"
+msgstr ""
+
+#: paperless_mail/models.py:11
+msgid "mail account"
+msgstr ""
+
+#: paperless_mail/models.py:12
+msgid "mail accounts"
+msgstr ""
+
+#: paperless_mail/models.py:19
+msgid "No encryption"
+msgstr ""
+
+#: paperless_mail/models.py:20
+msgid "Use SSL"
+msgstr ""
+
+#: paperless_mail/models.py:21
+msgid "Use STARTTLS"
+msgstr ""
+
+#: paperless_mail/models.py:29
+msgid "IMAP server"
+msgstr ""
+
+#: paperless_mail/models.py:33
+msgid "IMAP port"
+msgstr ""
+
+#: paperless_mail/models.py:36
+msgid ""
+"This is usually 143 for unencrypted and STARTTLS connections, and 993 for "
+"SSL connections."
+msgstr ""
+
+#: paperless_mail/models.py:40
+msgid "IMAP security"
+msgstr ""
+
+#: paperless_mail/models.py:46
+msgid "username"
+msgstr ""
+
+#: paperless_mail/models.py:50
+msgid "password"
+msgstr ""
+
+#: paperless_mail/models.py:60
+msgid "mail rule"
+msgstr ""
+
+#: paperless_mail/models.py:61
+msgid "mail rules"
+msgstr ""
+
+#: paperless_mail/models.py:69
+msgid "Mark as read, don't process read mails"
+msgstr ""
+
+#: paperless_mail/models.py:70
+msgid "Flag the mail, don't process flagged mails"
+msgstr ""
+
+#: paperless_mail/models.py:71
+msgid "Move to specified folder"
+msgstr ""
+
+#: paperless_mail/models.py:72
+msgid "Delete"
+msgstr ""
+
+#: paperless_mail/models.py:79
+msgid "Use subject as title"
+msgstr ""
+
+#: paperless_mail/models.py:80
+msgid "Use attachment filename as title"
+msgstr ""
+
+#: paperless_mail/models.py:90
+msgid "Do not assign a correspondent"
+msgstr ""
+
+#: paperless_mail/models.py:92
+msgid "Use mail address"
+msgstr ""
+
+#: paperless_mail/models.py:94
+msgid "Use name (or mail address if not available)"
+msgstr ""
+
+#: paperless_mail/models.py:96
+msgid "Use correspondent selected below"
+msgstr ""
+
+#: paperless_mail/models.py:104
+msgid "order"
+msgstr ""
+
+#: paperless_mail/models.py:111
+msgid "account"
+msgstr ""
+
+#: paperless_mail/models.py:115
+msgid "folder"
+msgstr ""
+
+#: paperless_mail/models.py:119
+msgid "filter from"
+msgstr ""
+
+#: paperless_mail/models.py:122
+msgid "filter subject"
+msgstr ""
+
+#: paperless_mail/models.py:125
+msgid "filter body"
+msgstr ""
+
+#: paperless_mail/models.py:129
+msgid "maximum age"
+msgstr ""
+
+#: paperless_mail/models.py:131
+msgid "Specified in days."
+msgstr ""
+
+#: paperless_mail/models.py:134
+msgid "action"
+msgstr ""
+
+#: paperless_mail/models.py:140
+msgid "action parameter"
+msgstr ""
+
+#: paperless_mail/models.py:142
+msgid ""
+"Additional parameter for the action selected above, i.e., the target folder "
+"of the move to folder action."
+msgstr ""
+
+#: paperless_mail/models.py:148
+msgid "assign title from"
+msgstr ""
+
+#: paperless_mail/models.py:158
+msgid "assign this tag"
+msgstr ""
+
+#: paperless_mail/models.py:166
+msgid "assign this document type"
+msgstr ""
+
+#: paperless_mail/models.py:170
+msgid "assign correspondent from"
+msgstr ""
+
+#: paperless_mail/models.py:180
+msgid "assign this correspondent"
+msgstr ""

src/locale/fr/LC_MESSAGES/django.po

@@ -0,0 +1,569 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+# 
+# Translators:
+# Jonas Winkler <dev@jpwinkler.de>, 2020
+# Philmo67, 2021
+# 
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2021-01-02 00:26+0000\n"
+"PO-Revision-Date: 2020-12-30 19:27+0000\n"
+"Last-Translator: Philmo67, 2021\n"
+"Language-Team: French (https://www.transifex.com/paperless/teams/115905/fr/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: fr\n"
+"Plural-Forms: nplurals=2; plural=(n > 1);\n"
+
+#: documents/apps.py:10
+msgid "Documents"
+msgstr "Documents"
+
+#: documents/models.py:32
+msgid "Any word"
+msgstr "Un des mots"
+
+#: documents/models.py:33
+msgid "All words"
+msgstr "Tous les mots"
+
+#: documents/models.py:34
+msgid "Exact match"
+msgstr "Concordance exacte"
+
+#: documents/models.py:35
+msgid "Regular expression"
+msgstr "Expression régulière"
+
+#: documents/models.py:36
+msgid "Fuzzy word"
+msgstr "Mot approximatif"
+
+#: documents/models.py:37
+msgid "Automatic"
+msgstr "Automatique"
+
+#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
+#: paperless_mail/models.py:100
+msgid "name"
+msgstr "nom"
+
+#: documents/models.py:45
+msgid "match"
+msgstr "rapprochement"
+
+#: documents/models.py:49
+msgid "matching algorithm"
+msgstr "algorithme de rapprochement"
+
+#: documents/models.py:55
+msgid "is insensitive"
+msgstr "est insensible à la casse"
+
+#: documents/models.py:80 documents/models.py:140
+msgid "correspondent"
+msgstr "correspondant"
+
+#: documents/models.py:81
+msgid "correspondents"
+msgstr "correspondants"
+
+#: documents/models.py:103
+msgid "color"
+msgstr "couleur"
+
+#: documents/models.py:107
+msgid "is inbox tag"
+msgstr "est une étiquette de boîte de réception"
+
+#: documents/models.py:109
+msgid ""
+"Marks this tag as an inbox tag: All newly consumed documents will be tagged "
+"with inbox tags."
+msgstr ""
+"Marque cette étiquette comme étiquette de boîte de réception : ces "
+"étiquettes sont affectées à tous les documents nouvellement traités."
+
+#: documents/models.py:114
+msgid "tag"
+msgstr "étiquette"
+
+#: documents/models.py:115 documents/models.py:171
+msgid "tags"
+msgstr "étiquettes"
+
+#: documents/models.py:121 documents/models.py:153
+msgid "document type"
+msgstr "type de document"
+
+#: documents/models.py:122
+msgid "document types"
+msgstr "types de document"
+
+#: documents/models.py:130
+msgid "Unencrypted"
+msgstr "Non chiffré"
+
+#: documents/models.py:131
+msgid "Encrypted with GNU Privacy Guard"
+msgstr "Chiffré avec GNU Privacy Guard"
+
+#: documents/models.py:144
+msgid "title"
+msgstr "titre"
+
+#: documents/models.py:157
+msgid "content"
+msgstr "contenu"
+
+#: documents/models.py:159
+msgid ""
+"The raw, text-only data of the document. This field is primarily used for "
+"searching."
+msgstr ""
+"Les données brutes du document, en format texte uniquement. Ce champ est "
+"principalement utilisé pour la recherche."
+
+#: documents/models.py:164
+msgid "mime type"
+msgstr "type mime"
+
+#: documents/models.py:175
+msgid "checksum"
+msgstr "somme de contrôle"
+
+#: documents/models.py:179
+msgid "The checksum of the original document."
+msgstr "La somme de contrôle du document original."
+
+#: documents/models.py:183
+msgid "archive checksum"
+msgstr "somme de contrôle de l'archive"
+
+#: documents/models.py:188
+msgid "The checksum of the archived document."
+msgstr "La somme de contrôle du document archivé."
+
+#: documents/models.py:192 documents/models.py:332
+msgid "created"
+msgstr "créé le"
+
+#: documents/models.py:196
+msgid "modified"
+msgstr "modifié"
+
+#: documents/models.py:200
+msgid "storage type"
+msgstr "forme d'enregistrement :"
+
+#: documents/models.py:208
+msgid "added"
+msgstr "date d'ajout"
+
+#: documents/models.py:212
+msgid "filename"
+msgstr "nom du fichier"
+
+#: documents/models.py:217
+msgid "Current filename in storage"
+msgstr "Nom du fichier courant en base de données"
+
+#: documents/models.py:221
+msgid "archive serial number"
+msgstr "numéro de série de l'archive"
+
+#: documents/models.py:226
+msgid "The position of this document in your physical document archive."
+msgstr ""
+"Le classement de ce document dans votre archive de documents physiques."
+
+#: documents/models.py:232
+msgid "document"
+msgstr "document"
+
+#: documents/models.py:233
+msgid "documents"
+msgstr "documents"
+
+#: documents/models.py:315
+msgid "debug"
+msgstr "débogage"
+
+#: documents/models.py:316
+msgid "information"
+msgstr "information"
+
+#: documents/models.py:317
+msgid "warning"
+msgstr "avertissement"
+
+#: documents/models.py:318
+msgid "error"
+msgstr "erreur"
+
+#: documents/models.py:319
+msgid "critical"
+msgstr "critique"
+
+#: documents/models.py:323
+msgid "group"
+msgstr "groupe"
+
+#: documents/models.py:326
+msgid "message"
+msgstr "message"
+
+#: documents/models.py:329
+msgid "level"
+msgstr "niveau"
+
+#: documents/models.py:336
+msgid "log"
+msgstr "rapport"
+
+#: documents/models.py:337
+msgid "logs"
+msgstr "rapports"
+
+#: documents/models.py:348 documents/models.py:398
+msgid "saved view"
+msgstr "vue enregistrée"
+
+#: documents/models.py:349
+msgid "saved views"
+msgstr "vues enregistrées"
+
+#: documents/models.py:352
+msgid "user"
+msgstr "utilisateur"
+
+#: documents/models.py:358
+msgid "show on dashboard"
+msgstr "montrer sur le tableau de bord"
+
+#: documents/models.py:361
+msgid "show in sidebar"
+msgstr "montrer dans la barre latérale"
+
+#: documents/models.py:365
+msgid "sort field"
+msgstr "champ de tri"
+
+#: documents/models.py:368
+msgid "sort reverse"
+msgstr "tri inverse"
+
+#: documents/models.py:374
+msgid "title contains"
+msgstr "le titre contient"
+
+#: documents/models.py:375
+msgid "content contains"
+msgstr "le contenu contient"
+
+#: documents/models.py:376
+msgid "ASN is"
+msgstr "le NSA est"
+
+#: documents/models.py:377
+msgid "correspondent is"
+msgstr "le correspondant est"
+
+#: documents/models.py:378
+msgid "document type is"
+msgstr "le type de document est"
+
+#: documents/models.py:379
+msgid "is in inbox"
+msgstr "est dans la boîte de réception"
+
+#: documents/models.py:380
+msgid "has tag"
+msgstr "porte l'étiquette"
+
+#: documents/models.py:381
+msgid "has any tag"
+msgstr "porte l'une des étiquettes"
+
+#: documents/models.py:382
+msgid "created before"
+msgstr "créé avant"
+
+#: documents/models.py:383
+msgid "created after"
+msgstr "créé après"
+
+#: documents/models.py:384
+msgid "created year is"
+msgstr "l'année de création est"
+
+#: documents/models.py:385
+msgid "created month is"
+msgstr "le mois de création est"
+
+#: documents/models.py:386
+msgid "created day is"
+msgstr "le jour de création est"
+
+#: documents/models.py:387
+msgid "added before"
+msgstr "ajouté avant"
+
+#: documents/models.py:388
+msgid "added after"
+msgstr "ajouté après"
+
+#: documents/models.py:389
+msgid "modified before"
+msgstr "modifié avant"
+
+#: documents/models.py:390
+msgid "modified after"
+msgstr "modifié après"
+
+#: documents/models.py:391
+msgid "does not have tag"
+msgstr "ne porte pas d'étiquette"
+
+#: documents/models.py:402
+msgid "rule type"
+msgstr "type de règle"
+
+#: documents/models.py:406
+msgid "value"
+msgstr "valeur"
+
+#: documents/models.py:412
+msgid "filter rule"
+msgstr "règle de filtrage"
+
+#: documents/models.py:413
+msgid "filter rules"
+msgstr "règles de filtrage"
+
+#: paperless/settings.py:254
+msgid "English"
+msgstr "Anglais"
+
+#: paperless/settings.py:255
+msgid "German"
+msgstr "Allemand"
+
+#: paperless/urls.py:108
+msgid "Paperless-ng administration"
+msgstr "Administration de Paperless-ng"
+
+#: paperless_mail/admin.py:24
+msgid "Filter"
+msgstr "Filtrage"
+
+#: paperless_mail/admin.py:26
+msgid ""
+"Paperless will only process mails that match ALL of the filters given below."
+msgstr ""
+"Paperless-ng ne traitera que les courriers qui correspondent à TOUS les "
+"filtres ci-dessous."
+
+#: paperless_mail/admin.py:34
+msgid "Actions"
+msgstr "Actions"
+
+#: paperless_mail/admin.py:36
+msgid ""
+"The action applied to the mail. This action is only performed when documents"
+" were consumed from the mail. Mails without attachments will remain entirely"
+" untouched."
+msgstr ""
+"Action appliquée au courriel. Cette action n'est exécutée que lorsque les "
+"documents ont été traités depuis des courriels. Les courriels sans pièces "
+"jointes demeurent totalement inchangés."
+
+#: paperless_mail/admin.py:43
+msgid "Metadata"
+msgstr "Métadonnées"
+
+#: paperless_mail/admin.py:45
+msgid ""
+"Assign metadata to documents consumed from this rule automatically. If you "
+"do not assign tags, types or correspondents here, paperless will still "
+"process all matching rules that you have defined."
+msgstr ""
+"Affecter automatiquement des métadonnées aux documents traités à partir de "
+"cette règle. Si vous n'affectez pas d'étiquettes, de types ou de "
+"correspondants ici, Paperless-ng traitera quand même toutes les règles de "
+"rapprochement que vous avez définies."
+
+#: paperless_mail/apps.py:9
+msgid "Paperless mail"
+msgstr "Paperless-ng pour le courriel"
+
+#: paperless_mail/models.py:11
+msgid "mail account"
+msgstr "compte de messagerie"
+
+#: paperless_mail/models.py:12
+msgid "mail accounts"
+msgstr "comptes de messagerie"
+
+#: paperless_mail/models.py:19
+msgid "No encryption"
+msgstr "Pas de chiffrement"
+
+#: paperless_mail/models.py:20
+msgid "Use SSL"
+msgstr "Utiliser SSL"
+
+#: paperless_mail/models.py:21
+msgid "Use STARTTLS"
+msgstr "Utiliser STARTTLS"
+
+#: paperless_mail/models.py:29
+msgid "IMAP server"
+msgstr "Serveur IMAP"
+
+#: paperless_mail/models.py:33
+msgid "IMAP port"
+msgstr "Port IMAP"
+
+#: paperless_mail/models.py:36
+msgid ""
+"This is usually 143 for unencrypted and STARTTLS connections, and 993 for "
+"SSL connections."
+msgstr ""
+"Généralement 143 pour les connexions non chiffrées et STARTTLS, et 993 pour "
+"les connexions SSL."
+
+#: paperless_mail/models.py:40
+msgid "IMAP security"
+msgstr "Sécurité IMAP"
+
+#: paperless_mail/models.py:46
+msgid "username"
+msgstr "nom d'utilisateur"
+
+#: paperless_mail/models.py:50
+msgid "password"
+msgstr "mot de passe"
+
+#: paperless_mail/models.py:60
+msgid "mail rule"
+msgstr "règle de courriel"
+
+#: paperless_mail/models.py:61
+msgid "mail rules"
+msgstr "règles de courriel"
+
+#: paperless_mail/models.py:69
+msgid "Mark as read, don't process read mails"
+msgstr "Marquer comme lu, ne pas traiter les courriels lus"
+
+#: paperless_mail/models.py:70
+msgid "Flag the mail, don't process flagged mails"
+msgstr "Marquer le courriel, ne pas traiter les courriels marqués"
+
+#: paperless_mail/models.py:71
+msgid "Move to specified folder"
+msgstr "Déplacer vers le dossier spécifié"
+
+#: paperless_mail/models.py:72
+msgid "Delete"
+msgstr "Supprimer"
+
+#: paperless_mail/models.py:79
+msgid "Use subject as title"
+msgstr "Utiliser le sujet en tant que titre"
+
+#: paperless_mail/models.py:80
+msgid "Use attachment filename as title"
+msgstr "Utiliser le nom de la pièce jointe en tant que titre"
+
+#: paperless_mail/models.py:90
+msgid "Do not assign a correspondent"
+msgstr "Ne pas affecter de correspondant"
+
+#: paperless_mail/models.py:92
+msgid "Use mail address"
+msgstr "Utiliser l'adresse électronique"
+
+#: paperless_mail/models.py:94
+msgid "Use name (or mail address if not available)"
+msgstr "Utiliser le nom (ou l'adresse électronique s'il n'est pas disponible)"
+
+#: paperless_mail/models.py:96
+msgid "Use correspondent selected below"
+msgstr "Utiliser le correspondant sélectionné ci-dessous"
+
+#: paperless_mail/models.py:104
+msgid "order"
+msgstr "ordre"
+
+#: paperless_mail/models.py:111
+msgid "account"
+msgstr "compte"
+
+#: paperless_mail/models.py:115
+msgid "folder"
+msgstr "répertoire"
+
+#: paperless_mail/models.py:119
+msgid "filter from"
+msgstr "filtrer l'expéditeur"
+
+#: paperless_mail/models.py:122
+msgid "filter subject"
+msgstr "filtrer le sujet"
+
+#: paperless_mail/models.py:125
+msgid "filter body"
+msgstr "filtrer le corps du message"
+
+#: paperless_mail/models.py:129
+msgid "maximum age"
+msgstr "âge maximum"
+
+#: paperless_mail/models.py:131
+msgid "Specified in days."
+msgstr "En jours."
+
+#: paperless_mail/models.py:134
+msgid "action"
+msgstr "action"
+
+#: paperless_mail/models.py:140
+msgid "action parameter"
+msgstr "paramètre d'action"
+
+#: paperless_mail/models.py:142
+msgid ""
+"Additional parameter for the action selected above, i.e., the target folder "
+"of the move to folder action."
+msgstr ""
+"Paramètre supplémentaire pour l'action sélectionnée ci-dessus, par exemple "
+"le dossier cible de l'action de déplacement vers un dossier."
+
+#: paperless_mail/models.py:148
+msgid "assign title from"
+msgstr "affecter le titre depuis"
+
+#: paperless_mail/models.py:158
+msgid "assign this tag"
+msgstr "affecter cette étiquette"
+
+#: paperless_mail/models.py:166
+msgid "assign this document type"
+msgstr "affecter ce type de document"
+
+#: paperless_mail/models.py:170
+msgid "assign correspondent from"
+msgstr "affecter le correspondant depuis"
+
+#: paperless_mail/models.py:180
+msgid "assign this correspondent"
+msgstr "affecter ce correspondant"

src/locale/nl_NL/LC_MESSAGES/django.po

@@ -0,0 +1,566 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
+# This file is distributed under the same license as the PACKAGE package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+# 
+# Translators:
+# Ben Zweekhorst <bzweekhorst@gmail.com>, 2021
+# J V <bugs.github@dwarfy.be>, 2021
+# 
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2021-01-02 00:26+0000\n"
+"PO-Revision-Date: 2020-12-30 19:27+0000\n"
+"Last-Translator: J V <bugs.github@dwarfy.be>, 2021\n"
+"Language-Team: Dutch (Netherlands) (https://www.transifex.com/paperless/teams/115905/nl_NL/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: nl_NL\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: documents/apps.py:10
+msgid "Documents"
+msgstr "Documenten"
+
+#: documents/models.py:32
+msgid "Any word"
+msgstr "Eender welk woord"
+
+#: documents/models.py:33
+msgid "All words"
+msgstr "Alle woorden"
+
+#: documents/models.py:34
+msgid "Exact match"
+msgstr "Exacte overeenkomst"
+
+#: documents/models.py:35
+msgid "Regular expression"
+msgstr "Reguliere expressie"
+
+#: documents/models.py:36
+msgid "Fuzzy word"
+msgstr "Vaag woord"
+
+#: documents/models.py:37
+msgid "Automatic"
+msgstr "Automatisch"
+
+#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
+#: paperless_mail/models.py:100
+msgid "name"
+msgstr "naam"
+
+#: documents/models.py:45
+msgid "match"
+msgstr "Overeenkomst"
+
+#: documents/models.py:49
+msgid "matching algorithm"
+msgstr "Algoritme voor het bepalen van de overeenkomst"
+
+#: documents/models.py:55
+msgid "is insensitive"
+msgstr "is niet hoofdlettergevoelig"
+
+#: documents/models.py:80 documents/models.py:140
+msgid "correspondent"
+msgstr "correspondent"
+
+#: documents/models.py:81
+msgid "correspondents"
+msgstr "correspondenten"
+
+#: documents/models.py:103
+msgid "color"
+msgstr "Kleur"
+
+#: documents/models.py:107
+msgid "is inbox tag"
+msgstr "is \"Postvak in\"-etiket"
+
+#: documents/models.py:109
+msgid ""
+"Marks this tag as an inbox tag: All newly consumed documents will be tagged "
+"with inbox tags."
+msgstr ""
+"Markeer dit etiket als een \"Postvak in\"-etiket: alle nieuw verwerkte "
+"documenten krijgen de \"Postvak in\"-etiketten."
+
+#: documents/models.py:114
+msgid "tag"
+msgstr "tag"
+
+#: documents/models.py:115 documents/models.py:171
+msgid "tags"
+msgstr "tags"
+
+#: documents/models.py:121 documents/models.py:153
+msgid "document type"
+msgstr "documenttype"
+
+#: documents/models.py:122
+msgid "document types"
+msgstr "documenttypen"
+
+#: documents/models.py:130
+msgid "Unencrypted"
+msgstr "Niet versleuteld"
+
+#: documents/models.py:131
+msgid "Encrypted with GNU Privacy Guard"
+msgstr "Versleuteld met GNU Privacy Guard"
+
+#: documents/models.py:144
+msgid "title"
+msgstr "titel"
+
+#: documents/models.py:157
+msgid "content"
+msgstr "inhoud"
+
+#: documents/models.py:159
+msgid ""
+"The raw, text-only data of the document. This field is primarily used for "
+"searching."
+msgstr ""
+"De onbewerkte gegevens van het document. Dit veld wordt voornamelijk "
+"gebruikt om te zoeken."
+
+#: documents/models.py:164
+msgid "mime type"
+msgstr "mimetype"
+
+#: documents/models.py:175
+msgid "checksum"
+msgstr "checksum"
+
+#: documents/models.py:179
+msgid "The checksum of the original document."
+msgstr "Het controlecijfer van het originele document."
+
+#: documents/models.py:183
+msgid "archive checksum"
+msgstr "archief checksum"
+
+#: documents/models.py:188
+msgid "The checksum of the archived document."
+msgstr "De checksum van het gearchiveerde document."
+
+#: documents/models.py:192 documents/models.py:332
+msgid "created"
+msgstr "aangemaakt"
+
+#: documents/models.py:196
+msgid "modified"
+msgstr "gewijzigd"
+
+#: documents/models.py:200
+msgid "storage type"
+msgstr "type opslag"
+
+#: documents/models.py:208
+msgid "added"
+msgstr "toegevoegd"
+
+#: documents/models.py:212
+msgid "filename"
+msgstr "bestandsnaam"
+
+#: documents/models.py:217
+msgid "Current filename in storage"
+msgstr "Huidige bestandsnaam in opslag"
+
+#: documents/models.py:221
+msgid "archive serial number"
+msgstr "serienummer in archief"
+
+#: documents/models.py:226
+msgid "The position of this document in your physical document archive."
+msgstr "De positie van dit document in je fysieke documentenarchief."
+
+#: documents/models.py:232
+msgid "document"
+msgstr "document"
+
+#: documents/models.py:233
+msgid "documents"
+msgstr "documenten"
+
+#: documents/models.py:315
+msgid "debug"
+msgstr "debug"
+
+#: documents/models.py:316
+msgid "information"
+msgstr "informatie"
+
+#: documents/models.py:317
+msgid "warning"
+msgstr "waarschuwing"
+
+#: documents/models.py:318
+msgid "error"
+msgstr "fout"
+
+#: documents/models.py:319
+msgid "critical"
+msgstr "kritisch"
+
+#: documents/models.py:323
+msgid "group"
+msgstr "groep"
+
+#: documents/models.py:326
+msgid "message"
+msgstr "bericht"
+
+#: documents/models.py:329
+msgid "level"
+msgstr "niveau"
+
+#: documents/models.py:336
+msgid "log"
+msgstr "bericht"
+
+#: documents/models.py:337
+msgid "logs"
+msgstr "berichten"
+
+#: documents/models.py:348 documents/models.py:398
+msgid "saved view"
+msgstr "opgeslagen view"
+
+#: documents/models.py:349
+msgid "saved views"
+msgstr "opgeslagen views"
+
+#: documents/models.py:352
+msgid "user"
+msgstr "gebruiker"
+
+#: documents/models.py:358
+msgid "show on dashboard"
+msgstr "weergeven op dashboard"
+
+#: documents/models.py:361
+msgid "show in sidebar"
+msgstr "weergeven in zijbalk"
+
+#: documents/models.py:365
+msgid "sort field"
+msgstr "sorteerveld"
+
+#: documents/models.py:368
+msgid "sort reverse"
+msgstr "omgekeerd sorteren"
+
+#: documents/models.py:374
+msgid "title contains"
+msgstr "titel bevat"
+
+#: documents/models.py:375
+msgid "content contains"
+msgstr "inhoud bevat"
+
+#: documents/models.py:376
+msgid "ASN is"
+msgstr "ASN is"
+
+#: documents/models.py:377
+msgid "correspondent is"
+msgstr "correspondent is"
+
+#: documents/models.py:378
+msgid "document type is"
+msgstr "documenttype is"
+
+#: documents/models.py:379
+msgid "is in inbox"
+msgstr "zit in \"Postvak in\""
+
+#: documents/models.py:380
+msgid "has tag"
+msgstr "heeft tag"
+
+#: documents/models.py:381
+msgid "has any tag"
+msgstr "heeft elke tag"
+
+#: documents/models.py:382
+msgid "created before"
+msgstr "aangemaakt voor"
+
+#: documents/models.py:383
+msgid "created after"
+msgstr "aangemaakt na"
+
+#: documents/models.py:384
+msgid "created year is"
+msgstr "aangemaakt jaar is"
+
+#: documents/models.py:385
+msgid "created month is"
+msgstr "aangemaakte maand is"
+
+#: documents/models.py:386
+msgid "created day is"
+msgstr "aangemaakte dag is"
+
+#: documents/models.py:387
+msgid "added before"
+msgstr "toegevoegd voor"
+
+#: documents/models.py:388
+msgid "added after"
+msgstr "toegevoegd na"
+
+#: documents/models.py:389
+msgid "modified before"
+msgstr "gewijzigd voor"
+
+#: documents/models.py:390
+msgid "modified after"
+msgstr "gewijzigd na"
+
+#: documents/models.py:391
+msgid "does not have tag"
+msgstr "heeft geen tag"
+
+#: documents/models.py:402
+msgid "rule type"
+msgstr "type regel"
+
+#: documents/models.py:406
+msgid "value"
+msgstr "waarde"
+
+#: documents/models.py:412
+msgid "filter rule"
+msgstr "filterregel"
+
+#: documents/models.py:413
+msgid "filter rules"
+msgstr "filterregels"
+
+#: paperless/settings.py:254
+msgid "English"
+msgstr "Engels"
+
+#: paperless/settings.py:255
+msgid "German"
+msgstr "Duits"
+
+#: paperless/urls.py:108
+msgid "Paperless-ng administration"
+msgstr "Paperless-ng administratie"
+
+#: paperless_mail/admin.py:24
+msgid "Filter"
+msgstr "Filter"
+
+#: paperless_mail/admin.py:26
+msgid ""
+"Paperless will only process mails that match ALL of the filters given below."
+msgstr ""
+"Paperless verwerkt alleen e-mails die voldoen aan ALLE onderstaande filters."
+
+#: paperless_mail/admin.py:34
+msgid "Actions"
+msgstr "Acties"
+
+#: paperless_mail/admin.py:36
+msgid ""
+"The action applied to the mail. This action is only performed when documents"
+" were consumed from the mail. Mails without attachments will remain entirely"
+" untouched."
+msgstr ""
+"De actie die wordt toegepast op de mail. Deze actie wordt alleen uitgevoerd "
+"wanneer documenten verwerkt werden uit de mail. Mails zonder bijlage blijven"
+" onaangeroerd."
+
+#: paperless_mail/admin.py:43
+msgid "Metadata"
+msgstr "Metadata"
+
+#: paperless_mail/admin.py:45
+msgid ""
+"Assign metadata to documents consumed from this rule automatically. If you "
+"do not assign tags, types or correspondents here, paperless will still "
+"process all matching rules that you have defined."
+msgstr ""
+"Automatisch metadata toewijzen aan documenten vanuit deze regel. Indien je "
+"geen etiketten, documenttypes of correspondenten toewijst, zal Paperless nog"
+" steeds alle regels verwerken die je hebt gedefinieerd."
+
+#: paperless_mail/apps.py:9
+msgid "Paperless mail"
+msgstr "Paperless email"
+
+#: paperless_mail/models.py:11
+msgid "mail account"
+msgstr "email account"
+
+#: paperless_mail/models.py:12
+msgid "mail accounts"
+msgstr "email accounts"
+
+#: paperless_mail/models.py:19
+msgid "No encryption"
+msgstr "Geen versleuteling"
+
+#: paperless_mail/models.py:20
+msgid "Use SSL"
+msgstr "Gebruik SSL"
+
+#: paperless_mail/models.py:21
+msgid "Use STARTTLS"
+msgstr "Gebruik STARTTLS"
+
+#: paperless_mail/models.py:29
+msgid "IMAP server"
+msgstr "IMAP-server"
+
+#: paperless_mail/models.py:33
+msgid "IMAP port"
+msgstr "IMAP-poort"
+
+#: paperless_mail/models.py:36
+msgid ""
+"This is usually 143 for unencrypted and STARTTLS connections, and 993 for "
+"SSL connections."
+msgstr ""
+"Dit is gewoonlijk 143 voor onversleutelde of STARTTLS verbindingen, en 993 "
+"voor SSL verbindingen."
+
+#: paperless_mail/models.py:40
+msgid "IMAP security"
+msgstr "IMAP-beveiliging"
+
+#: paperless_mail/models.py:46
+msgid "username"
+msgstr "gebruikersnaam"
+
+#: paperless_mail/models.py:50
+msgid "password"
+msgstr "wachtwoord"
+
+#: paperless_mail/models.py:60
+msgid "mail rule"
+msgstr "email-regel"
+
+#: paperless_mail/models.py:61
+msgid "mail rules"
+msgstr "email-regels"
+
+#: paperless_mail/models.py:69
+msgid "Mark as read, don't process read mails"
+msgstr "Markeer als gelezen, verwerk geen gelezen mails"
+
+#: paperless_mail/models.py:70
+msgid "Flag the mail, don't process flagged mails"
+msgstr "Markeer de mail, verwerk geen mails met markering"
+
+#: paperless_mail/models.py:71
+msgid "Move to specified folder"
+msgstr "Verplaats naar gegeven map"
+
+#: paperless_mail/models.py:72
+msgid "Delete"
+msgstr "Verwijder"
+
+#: paperless_mail/models.py:79
+msgid "Use subject as title"
+msgstr "Gebruik onderwerp als titel"
+
+#: paperless_mail/models.py:80
+msgid "Use attachment filename as title"
+msgstr "Gebruik naam van bijlage als titel"
+
+#: paperless_mail/models.py:90
+msgid "Do not assign a correspondent"
+msgstr "Wijs geen correspondent toe"
+
+#: paperless_mail/models.py:92
+msgid "Use mail address"
+msgstr "Gebruik het email-adres"
+
+#: paperless_mail/models.py:94
+msgid "Use name (or mail address if not available)"
+msgstr "Gebruik de naam, en anders het email-adres"
+
+#: paperless_mail/models.py:96
+msgid "Use correspondent selected below"
+msgstr "Gebruik de hieronder aangeduide correspondent"
+
+#: paperless_mail/models.py:104
+msgid "order"
+msgstr "volgorde"
+
+#: paperless_mail/models.py:111
+msgid "account"
+msgstr "account"
+
+#: paperless_mail/models.py:115
+msgid "folder"
+msgstr "map"
+
+#: paperless_mail/models.py:119
+msgid "filter from"
+msgstr "filter afzender"
+
+#: paperless_mail/models.py:122
+msgid "filter subject"
+msgstr "filter onderwerp"
+
+#: paperless_mail/models.py:125
+msgid "filter body"
+msgstr "filter inhoud"
+
+#: paperless_mail/models.py:129
+msgid "maximum age"
+msgstr "Maximale leeftijd"
+
+#: paperless_mail/models.py:131
+msgid "Specified in days."
+msgstr "Aangegeven in dagen"
+
+#: paperless_mail/models.py:134
+msgid "action"
+msgstr "actie"
+
+#: paperless_mail/models.py:140
+msgid "action parameter"
+msgstr "actie parameters"
+
+#: paperless_mail/models.py:142
+msgid ""
+"Additional parameter for the action selected above, i.e., the target folder "
+"of the move to folder action."
+msgstr ""
+"Extra parameters voor de hierboven gekozen actie, met andere woorden: de "
+"bestemmingsmap voor de verplaats-actie."
+
+#: paperless_mail/models.py:148
+msgid "assign title from"
+msgstr "wijs titel toe van"
+
+#: paperless_mail/models.py:158
+msgid "assign this tag"
+msgstr "wijs dit etiket toe"
+
+#: paperless_mail/models.py:166
+msgid "assign this document type"
+msgstr "wijs dit documenttype toe"
+
+#: paperless_mail/models.py:170
+msgid "assign correspondent from"
+msgstr "wijs correspondent toe van"
+
+#: paperless_mail/models.py:180
+msgid "assign this correspondent"
+msgstr "wijs deze correspondent toe"

src/paperless/checks.py

@@ -13,18 +13,17 @@ writeable_hint = (
 )
 
 
-def path_check(env_var):
+def path_check(var, directory):
     messages = []
-    directory = os.getenv(env_var)
     if directory:
         if not os.path.exists(directory):
             messages.append(Error(
-                exists_message.format(env_var),
+                exists_message.format(var),
                 exists_hint.format(directory)
             ))
         elif not os.access(directory, os.W_OK | os.X_OK):
             messages.append(Error(
-                writeable_message.format(env_var),
+                writeable_message.format(var),
                 writeable_hint.format(directory)
             ))
     return messages
@@ -36,12 +35,9 @@ def paths_check(app_configs, **kwargs):
     Check the various paths for existence, readability and writeability
     """
 
-    check_messages = path_check("PAPERLESS_DATA_DIR") + \
-        path_check("PAPERLESS_MEDIA_ROOT") + \
-        path_check("PAPERLESS_CONSUMPTION_DIR") + \
-        path_check("PAPERLESS_STATICDIR")
-
-    return check_messages
+    return path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \
+        path_check("PAPERLESS_MEDIA_ROOT", settings.MEDIA_ROOT) + \
+        path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR)
 
 
 @register()

src/paperless/settings.py

@@ -6,6 +6,8 @@ import re
 
 from dotenv import load_dotenv
 
+from django.utils.translation import gettext_lazy as _
+
 # Tap paperless.conf if it's available
 if os.path.exists("../paperless.conf"):
     load_dotenv("../paperless.conf")
@@ -53,6 +55,10 @@ ARCHIVE_DIR = os.path.join(MEDIA_ROOT, "documents", "archive")
 THUMBNAIL_DIR = os.path.join(MEDIA_ROOT, "documents", "thumbnails")
 
 DATA_DIR = os.getenv('PAPERLESS_DATA_DIR', os.path.join(BASE_DIR, "..", "data"))
+
+# Lock file for synchronizing changes to the MEDIA directory across multiple
+# threads.
+MEDIA_LOCK = os.path.join(MEDIA_ROOT, "media.lock")
 INDEX_DIR = os.path.join(DATA_DIR, "index")
 MODEL_FILE = os.path.join(DATA_DIR, "classification_model.pickle")
 
@@ -65,6 +71,8 @@ SCRATCH_DIR = os.getenv("PAPERLESS_SCRATCH_DIR", "/tmp/paperless")
 # Application Definition                                                      #
 ###############################################################################
 
+env_apps = os.getenv("PAPERLESS_APPS").split(",") if os.getenv("PAPERLESS_APPS") else []
+
 INSTALLED_APPS = [
     "whitenoise.runserver_nostatic",
 
@@ -81,6 +89,7 @@ INSTALLED_APPS = [
     "documents.apps.DocumentsConfig",
     "paperless_tesseract.apps.PaperlessTesseractConfig",
     "paperless_text.apps.PaperlessTextConfig",
+    "paperless_tika.apps.PaperlessTikaConfig",
     "paperless_mail.apps.PaperlessMailConfig",
 
     "django.contrib.admin",
@@ -93,7 +102,7 @@ INSTALLED_APPS = [
 
     "channels",
 
-]
+] + env_apps
 
 REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': [
@@ -113,6 +122,7 @@ MIDDLEWARE = [
     'whitenoise.middleware.WhiteNoiseMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'corsheaders.middleware.CorsMiddleware',
+    'django.middleware.locale.LocaleMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
@@ -168,13 +178,6 @@ if AUTO_LOGIN_USERNAME:
     MIDDLEWARE.insert(_index+1, 'paperless.auth.AutoLoginMiddleware')
 
 
-if DEBUG:
-    X_FRAME_OPTIONS = ''
-    # this should really be 'allow-from uri' but its not supported in any mayor
-    # browser.
-else:
-    X_FRAME_OPTIONS = 'SAMEORIGIN'
-
 # We allow CORS from localhost:8080
 CORS_ALLOWED_ORIGINS = tuple(os.getenv("PAPERLESS_CORS_ALLOWED_HOSTS", "http://localhost:8000").split(","))
 
@@ -218,6 +221,12 @@ AUTH_PASSWORD_VALIDATORS = [
 
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None
 
+COOKIE_PREFIX = os.getenv("PAPERLESS_COOKIE_PREFIX", "")
+
+CSRF_COOKIE_NAME = f"{COOKIE_PREFIX}csrftoken"
+SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid"
+LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language"
+
 ###############################################################################
 # Database                                                                    #
 ###############################################################################
@@ -253,6 +262,17 @@ if os.getenv("PAPERLESS_DBHOST"):
 
 LANGUAGE_CODE = 'en-us'
 
+LANGUAGES = [
+    ("en-us", _("English")),
+    ("de", _("German")),
+    ("nl-nl", _("Dutch")),
+    ("fr", _("French"))
+]
+
+LOCALE_PATHS = [
+    os.path.join(BASE_DIR, "locale")
+]
+
 TIME_ZONE = os.getenv("PAPERLESS_TIME_ZONE", "UTC")
 
 USE_I18N = True
@@ -429,3 +449,12 @@ for t in json.loads(os.getenv("PAPERLESS_FILENAME_PARSE_TRANSFORMS", "[]")):
 # TODO: this should not have a prefix.
 # Specify the filename format for out files
 PAPERLESS_FILENAME_FORMAT = os.getenv("PAPERLESS_FILENAME_FORMAT")
+
+THUMBNAIL_FONT_NAME = os.getenv("PAPERLESS_THUMBNAIL_FONT_NAME", "/usr/share/fonts/liberation/LiberationSerif-Regular.ttf")
+
+# Tika settings
+PAPERLESS_TIKA_ENABLED = __get_boolean("PAPERLESS_TIKA_ENABLED", "NO")
+PAPERLESS_TIKA_ENDPOINT = os.getenv("PAPERLESS_TIKA_ENDPOINT", "http://localhost:9998")
+PAPERLESS_TIKA_GOTENBERG_ENDPOINT = os.getenv(
+    "PAPERLESS_TIKA_GOTENBERG_ENDPOINT", "http://localhost:3000"
+)

src/paperless/tests/test_checks.py

@@ -0,0 +1,54 @@
+import os
+import shutil
+
+from django.test import TestCase, override_settings
+
+from documents.tests.utils import DirectoriesMixin
+from paperless import binaries_check, paths_check
+from paperless.checks import debug_mode_check
+
+
+class TestChecks(DirectoriesMixin, TestCase):
+
+    def test_binaries(self):
+        self.assertEqual(binaries_check(None), [])
+
+    @override_settings(CONVERT_BINARY="uuuhh", OPTIPNG_BINARY="forgot")
+    def test_binaries_fail(self):
+        self.assertEqual(len(binaries_check(None)), 2)
+
+    def test_paths_check(self):
+        self.assertEqual(paths_check(None), [])
+
+    @override_settings(MEDIA_ROOT="uuh",
+                       DATA_DIR="whatever",
+                       CONSUMPTION_DIR="idontcare")
+    def test_paths_check_dont_exist(self):
+        msgs = paths_check(None)
+        self.assertEqual(len(msgs), 3, str(msgs))
+
+        for msg in msgs:
+            self.assertTrue(msg.msg.endswith("is set but doesn't exist."))
+
+    def test_paths_check_no_access(self):
+        os.chmod(self.dirs.data_dir, 0o000)
+        os.chmod(self.dirs.media_dir, 0o000)
+        os.chmod(self.dirs.consumption_dir, 0o000)
+
+        self.addCleanup(os.chmod, self.dirs.data_dir, 0o777)
+        self.addCleanup(os.chmod, self.dirs.media_dir, 0o777)
+        self.addCleanup(os.chmod, self.dirs.consumption_dir, 0o777)
+
+        msgs = paths_check(None)
+        self.assertEqual(len(msgs), 3)
+
+        for msg in msgs:
+            self.assertTrue(msg.msg.endswith("is not writeable"))
+
+    @override_settings(DEBUG=False)
+    def test_debug_disabled(self):
+        self.assertEqual(debug_mode_check(None), [])
+
+    @override_settings(DEBUG=True)
+    def test_debug_enabled(self):
+        self.assertEqual(len(debug_mode_check(None)), 1)

src/paperless/urls.py

@@ -7,6 +7,8 @@ from django.views.generic import RedirectView
 from rest_framework.authtoken import views
 from rest_framework.routers import DefaultRouter
 
+from django.utils.translation import gettext_lazy as _
+
 from paperless.consumers import StatusConsumer
 from documents.views import (
     CorrespondentViewSet,
@@ -18,7 +20,10 @@ from documents.views import (
     IndexView,
     SearchAutoCompleteView,
     StatisticsView,
-    PostDocumentView
+    PostDocumentView,
+    SavedViewViewSet,
+    BulkEditView,
+    SelectionDataView
 )
 from paperless.views import FaviconView
 
@@ -28,6 +33,7 @@ api_router.register(r"document_types", DocumentTypeViewSet)
 api_router.register(r"documents", DocumentViewSet)
 api_router.register(r"logs", LogViewSet)
 api_router.register(r"tags", TagViewSet)
+api_router.register(r"saved_views", SavedViewViewSet)
 
 
 urlpatterns = [
@@ -51,6 +57,12 @@ urlpatterns = [
         re_path(r"^documents/post_document/", PostDocumentView.as_view(),
                 name="post_document"),
 
+        re_path(r"^documents/bulk_edit/", BulkEditView.as_view(),
+                name="bulk_edit"),
+
+        re_path(r"^documents/selection_data/", SelectionDataView.as_view(),
+                name="selection_data"),
+
         path('token/', views.obtain_auth_token)
 
     ] + api_router.urls)),
@@ -79,7 +91,8 @@ urlpatterns = [
 
     # Frontend assets TODO: this is pretty bad, but it works.
     path('assets/<path:path>',
-         RedirectView.as_view(url='/static/frontend/assets/%(path)s')),
+         RedirectView.as_view(url='/static/frontend/en-US/assets/%(path)s')),
+    # TODO: with localization, this is even worse! :/
 
     # login, logout
     path('accounts/', include('django.contrib.auth.urls')),
@@ -98,4 +111,4 @@ admin.site.site_header = 'Paperless-ng'
 # Text at the end of each page's <title>.
 admin.site.site_title = 'Paperless-ng'
 # Text at the top of the admin index page.
-admin.site.index_title = 'Paperless-ng administration'
+admin.site.index_title = _('Paperless-ng administration')

src/paperless/version.py

@@ -1 +1 @@
-__version__ = (0, 9, 5)
+__version__ = (0, 9, 11)

src/paperless_mail/admin.py

@@ -1,6 +1,8 @@
 from django.contrib import admin
 from paperless_mail.models import MailAccount, MailRule
 
+from django.utils.translation import gettext_lazy as _
+
 
 class MailAccountAdmin(admin.ModelAdmin):
 
@@ -19,31 +21,31 @@ class MailRuleAdmin(admin.ModelAdmin):
         (None, {
             'fields': ('name', 'order', 'account', 'folder')
         }),
-        ("Filter", {
+        (_("Filter"), {
             'description':
-                "Paperless will only process mails that match ALL of the "
-                "filters given below.",
+                _("Paperless will only process mails that match ALL of the "
+                  "filters given below."),
             'fields':
                 ('filter_from',
                  'filter_subject',
                  'filter_body',
                  'maximum_age')
         }),
-        ("Actions", {
+        (_("Actions"), {
             'description':
-                "The action applied to the mail. This action is only "
-                "performed when documents were consumed from the mail. Mails "
-                "without attachments will remain entirely untouched.",
+                _("The action applied to the mail. This action is only "
+                  "performed when documents were consumed from the mail. "
+                  "Mails without attachments will remain entirely untouched."),
             'fields': (
                 'action',
                 'action_parameter')
         }),
-        ("Metadata", {
+        (_("Metadata"), {
             'description':
-                "Assign metadata to documents consumed from this rule "
-                "automatically. If you do not assign tags, types or "
-                "correspondents here, paperless will still process all "
-                "matching rules that you have defined.",
+                _("Assign metadata to documents consumed from this rule "
+                  "automatically. If you do not assign tags, types or "
+                  "correspondents here, paperless will still process all "
+                  "matching rules that you have defined."),
             "fields": (
                 'assign_title_from',
                 'assign_tag',

src/paperless_mail/apps.py

@@ -1,7 +1,9 @@
 from django.apps import AppConfig
 
+from django.utils.translation import gettext_lazy as _
+
 
 class PaperlessMailConfig(AppConfig):
     name = 'paperless_mail'
 
-    verbose_name = 'Paperless Mail'
+    verbose_name = _('Paperless mail')

src/paperless_mail/mail.py

@@ -3,9 +3,9 @@ import tempfile
 from datetime import timedelta, date
 
 import magic
+import pathvalidate
 from django.conf import settings
 from django.db import DatabaseError
-from django.utils.text import slugify
 from django_q.tasks import async_task
 from imap_tools import MailBox, MailBoxUnencrypted, AND, MailMessageFlags, \
     MailboxFolderSelectError
@@ -26,7 +26,7 @@ class BaseMailAction:
         return {}
 
     def post_consume(self, M, message_uids, parameter):
-        pass
+        pass  # pragma: nocover
 
 
 class DeleteMailAction(BaseMailAction):
@@ -69,7 +69,7 @@ def get_rule_action(rule):
     elif rule.action == MailRule.ACTION_MARK_READ:
         return MarkReadMailAction()
     else:
-        raise ValueError("Unknown action.")
+        raise NotImplementedError("Unknown action.")  # pragma: nocover
 
 
 def make_criterias(rule):
@@ -95,7 +95,7 @@ def get_mailbox(server, port, security):
     elif security == MailAccount.IMAP_SECURITY_SSL:
         mailbox = MailBox(server, port)
     else:
-        raise ValueError("Unknown IMAP security")
+        raise NotImplementedError("Unknown IMAP security")  # pragma: nocover
     return mailbox
 
 
@@ -103,10 +103,7 @@ class MailAccountHandler(LoggingMixin):
 
     def _correspondent_from_name(self, name):
         try:
-            return Correspondent.objects.get_or_create(
-                name=name, defaults={
-                    "slug": slugify(name)
-                })[0]
+            return Correspondent.objects.get_or_create(name=name)[0]
         except DatabaseError as e:
             self.log(
                 "error",
@@ -122,7 +119,7 @@ class MailAccountHandler(LoggingMixin):
             return os.path.splitext(os.path.basename(att.filename))[0]
 
         else:
-            raise ValueError("Unknown title selector.")
+            raise NotImplementedError("Unknown title selector.")  # pragma: nocover  # NOQA: E501
 
     def get_correspondent(self, message, rule):
         c_from = rule.assign_correspondent_from
@@ -144,7 +141,7 @@ class MailAccountHandler(LoggingMixin):
             return rule.assign_correspondent
 
         else:
-            raise ValueError("Unknwown correspondent selector")
+            raise NotImplementedError("Unknwown correspondent selector")  # pragma: nocover  # NOQA: E501
 
     def handle_mail_account(self, account):
 
@@ -297,7 +294,7 @@ class MailAccountHandler(LoggingMixin):
                 async_task(
                     "documents.tasks.consume_file",
                     path=temp_filename,
-                    override_filename=att.filename,
+                    override_filename=pathvalidate.sanitize_filename(att.filename),  # NOQA: E501
                     override_title=title,
                     override_correspondent_id=correspondent.id if correspondent else None,  # NOQA: E501
                     override_document_type_id=doc_type.id if doc_type else None,  # NOQA: E501

src/paperless_mail/migrations/0006_auto_20210101_2340.py

@@ -0,0 +1,128 @@
+# Generated by Django 3.1.4 on 2021-01-01 23:40
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '1011_auto_20210101_2340'),
+        ('paperless_mail', '0005_help_texts'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='mailaccount',
+            options={'verbose_name': 'mail account', 'verbose_name_plural': 'mail accounts'},
+        ),
+        migrations.AlterModelOptions(
+            name='mailrule',
+            options={'verbose_name': 'mail rule', 'verbose_name_plural': 'mail rules'},
+        ),
+        migrations.AlterField(
+            model_name='mailaccount',
+            name='imap_port',
+            field=models.IntegerField(blank=True, help_text='This is usually 143 for unencrypted and STARTTLS connections, and 993 for SSL connections.', null=True, verbose_name='IMAP port'),
+        ),
+        migrations.AlterField(
+            model_name='mailaccount',
+            name='imap_security',
+            field=models.PositiveIntegerField(choices=[(1, 'No encryption'), (2, 'Use SSL'), (3, 'Use STARTTLS')], default=2, verbose_name='IMAP security'),
+        ),
+        migrations.AlterField(
+            model_name='mailaccount',
+            name='imap_server',
+            field=models.CharField(max_length=256, verbose_name='IMAP server'),
+        ),
+        migrations.AlterField(
+            model_name='mailaccount',
+            name='name',
+            field=models.CharField(max_length=256, unique=True, verbose_name='name'),
+        ),
+        migrations.AlterField(
+            model_name='mailaccount',
+            name='password',
+            field=models.CharField(max_length=256, verbose_name='password'),
+        ),
+        migrations.AlterField(
+            model_name='mailaccount',
+            name='username',
+            field=models.CharField(max_length=256, verbose_name='username'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='account',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rules', to='paperless_mail.mailaccount', verbose_name='account'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='action',
+            field=models.PositiveIntegerField(choices=[(3, "Mark as read, don't process read mails"), (4, "Flag the mail, don't process flagged mails"), (2, 'Move to specified folder'), (1, 'Delete')], default=3, verbose_name='action'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='action_parameter',
+            field=models.CharField(blank=True, help_text='Additional parameter for the action selected above, i.e., the target folder of the move to folder action.', max_length=256, null=True, verbose_name='action parameter'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='assign_correspondent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.correspondent', verbose_name='assign this correspondent'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='assign_correspondent_from',
+            field=models.PositiveIntegerField(choices=[(1, 'Do not assign a correspondent'), (2, 'Use mail address'), (3, 'Use name (or mail address if not available)'), (4, 'Use correspondent selected below')], default=1, verbose_name='assign correspondent from'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='assign_document_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.documenttype', verbose_name='assign this document type'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='assign_tag',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='documents.tag', verbose_name='assign this tag'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='assign_title_from',
+            field=models.PositiveIntegerField(choices=[(1, 'Use subject as title'), (2, 'Use attachment filename as title')], default=1, verbose_name='assign title from'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='filter_body',
+            field=models.CharField(blank=True, max_length=256, null=True, verbose_name='filter body'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='filter_from',
+            field=models.CharField(blank=True, max_length=256, null=True, verbose_name='filter from'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='filter_subject',
+            field=models.CharField(blank=True, max_length=256, null=True, verbose_name='filter subject'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='folder',
+            field=models.CharField(default='INBOX', max_length=256, verbose_name='folder'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='maximum_age',
+            field=models.PositiveIntegerField(default=30, help_text='Specified in days.', verbose_name='maximum age'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='name',
+            field=models.CharField(max_length=256, unique=True, verbose_name='name'),
+        ),
+        migrations.AlterField(
+            model_name='mailrule',
+            name='order',
+            field=models.IntegerField(default=0, verbose_name='order'),
+        ),
+    ]

src/paperless_mail/models.py

@@ -2,37 +2,53 @@ from django.db import models
 
 import documents.models as document_models
 
+from django.utils.translation import gettext_lazy as _
+
 
 class MailAccount(models.Model):
 
+    class Meta:
+        verbose_name = _("mail account")
+        verbose_name_plural = _("mail accounts")
+
     IMAP_SECURITY_NONE = 1
     IMAP_SECURITY_SSL = 2
     IMAP_SECURITY_STARTTLS = 3
 
     IMAP_SECURITY_OPTIONS = (
-        (IMAP_SECURITY_NONE, "No encryption"),
-        (IMAP_SECURITY_SSL, "Use SSL"),
-        (IMAP_SECURITY_STARTTLS, "Use STARTTLS"),
+        (IMAP_SECURITY_NONE, _("No encryption")),
+        (IMAP_SECURITY_SSL, _("Use SSL")),
+        (IMAP_SECURITY_STARTTLS, _("Use STARTTLS")),
     )
 
-    name = models.CharField(max_length=256, unique=True)
+    name = models.CharField(
+        _("name"),
+        max_length=256, unique=True)
 
-    imap_server = models.CharField(max_length=256)
+    imap_server = models.CharField(
+        _("IMAP server"),
+        max_length=256)
 
     imap_port = models.IntegerField(
+        _("IMAP port"),
         blank=True,
         null=True,
-        help_text="This is usually 143 for unencrypted and STARTTLS "
-                  "connections, and 993 for SSL connections.")
+        help_text=_("This is usually 143 for unencrypted and STARTTLS "
+                    "connections, and 993 for SSL connections."))
 
     imap_security = models.PositiveIntegerField(
+        _("IMAP security"),
         choices=IMAP_SECURITY_OPTIONS,
         default=IMAP_SECURITY_SSL
     )
 
-    username = models.CharField(max_length=256)
+    username = models.CharField(
+        _("username"),
+        max_length=256)
 
-    password = models.CharField(max_length=256)
+    password = models.CharField(
+        _("password"),
+        max_length=256)
 
     def __str__(self):
         return self.name
@@ -40,24 +56,28 @@ class MailAccount(models.Model):
 
 class MailRule(models.Model):
 
+    class Meta:
+        verbose_name = _("mail rule")
+        verbose_name_plural = _("mail rules")
+
     ACTION_DELETE = 1
     ACTION_MOVE = 2
     ACTION_MARK_READ = 3
     ACTION_FLAG = 4
 
     ACTIONS = (
-        (ACTION_MARK_READ, "Mark as read, don't process read mails"),
-        (ACTION_FLAG, "Flag the mail, don't process flagged mails"),
-        (ACTION_MOVE, "Move to specified folder"),
-        (ACTION_DELETE, "Delete"),
+        (ACTION_MARK_READ, _("Mark as read, don't process read mails")),
+        (ACTION_FLAG, _("Flag the mail, don't process flagged mails")),
+        (ACTION_MOVE, _("Move to specified folder")),
+        (ACTION_DELETE, _("Delete")),
     )
 
     TITLE_FROM_SUBJECT = 1
     TITLE_FROM_FILENAME = 2
 
     TITLE_SELECTOR = (
-        (TITLE_FROM_SUBJECT, "Use subject as title"),
-        (TITLE_FROM_FILENAME, "Use attachment filename as title")
+        (TITLE_FROM_SUBJECT, _("Use subject as title")),
+        (TITLE_FROM_FILENAME, _("Use attachment filename as title"))
     )
 
     CORRESPONDENT_FROM_NOTHING = 1
@@ -67,47 +87,65 @@ class MailRule(models.Model):
 
     CORRESPONDENT_SELECTOR = (
         (CORRESPONDENT_FROM_NOTHING,
-         "Do not assign a correspondent"),
+         _("Do not assign a correspondent")),
         (CORRESPONDENT_FROM_EMAIL,
-         "Use mail address"),
+         _("Use mail address")),
         (CORRESPONDENT_FROM_NAME,
-         "Use name (or mail address if not available)"),
+         _("Use name (or mail address if not available)")),
         (CORRESPONDENT_FROM_CUSTOM,
-         "Use correspondent selected below")
+         _("Use correspondent selected below"))
     )
 
-    name = models.CharField(max_length=256, unique=True)
+    name = models.CharField(
+        _("name"),
+        max_length=256, unique=True)
 
-    order = models.IntegerField(default=0)
+    order = models.IntegerField(
+        _("order"),
+        default=0)
 
     account = models.ForeignKey(
         MailAccount,
         related_name="rules",
-        on_delete=models.CASCADE
+        on_delete=models.CASCADE,
+        verbose_name=_("account")
     )
 
-    folder = models.CharField(default='INBOX', max_length=256)
+    folder = models.CharField(
+        _("folder"),
+        default='INBOX', max_length=256)
 
-    filter_from = models.CharField(max_length=256, null=True, blank=True)
-    filter_subject = models.CharField(max_length=256, null=True, blank=True)
-    filter_body = models.CharField(max_length=256, null=True, blank=True)
+    filter_from = models.CharField(
+        _("filter from"),
+        max_length=256, null=True, blank=True)
+    filter_subject = models.CharField(
+        _("filter subject"),
+        max_length=256, null=True, blank=True)
+    filter_body = models.CharField(
+        _("filter body"),
+        max_length=256, null=True, blank=True)
 
     maximum_age = models.PositiveIntegerField(
+        _("maximum age"),
         default=30,
-        help_text="Specified in days.")
+        help_text=_("Specified in days."))
 
     action = models.PositiveIntegerField(
+        _("action"),
         choices=ACTIONS,
         default=ACTION_MARK_READ,
     )
 
     action_parameter = models.CharField(
+        _("action parameter"),
         max_length=256, blank=True, null=True,
-        help_text="Additional parameter for the action selected above, i.e., "
-                  "the target folder of the move to folder action."
+        help_text=_("Additional parameter for the action selected above, "
+                    "i.e., "
+                    "the target folder of the move to folder action.")
     )
 
     assign_title_from = models.PositiveIntegerField(
+        _("assign title from"),
         choices=TITLE_SELECTOR,
         default=TITLE_FROM_SUBJECT
     )
@@ -116,17 +154,20 @@ class MailRule(models.Model):
         document_models.Tag,
         null=True,
         blank=True,
-        on_delete=models.SET_NULL
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this tag"),
     )
 
     assign_document_type = models.ForeignKey(
         document_models.DocumentType,
         null=True,
         blank=True,
-        on_delete=models.SET_NULL
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this document type"),
     )
 
     assign_correspondent_from = models.PositiveIntegerField(
+        _("assign correspondent from"),
         choices=CORRESPONDENT_SELECTOR,
         default=CORRESPONDENT_FROM_NOTHING
     )
@@ -135,7 +176,8 @@ class MailRule(models.Model):
         document_models.Correspondent,
         null=True,
         blank=True,
-        on_delete=models.SET_NULL
+        on_delete=models.SET_NULL,
+        verbose_name=_("assign this correspondent")
     )
 
     def __str__(self):

src/paperless_mail/tests/test_mail.py

@@ -399,7 +399,7 @@ class TestMail(TestCase):
 
         c = Correspondent.objects.get(name="amazon@amazon.de")
         # should work
-        self.assertEquals(kwargs['override_correspondent_id'], c.id)
+        self.assertEqual(kwargs['override_correspondent_id'], c.id)
 
         self.async_task.reset_mock()
         self.reset_bogus_mailbox()
@@ -411,7 +411,7 @@ class TestMail(TestCase):
 
         args, kwargs = self.async_task.call_args
         self.async_task.assert_called_once()
-        self.assertEquals(kwargs['override_correspondent_id'], None)
+        self.assertEqual(kwargs['override_correspondent_id'], None)
 
 
     def test_filters(self):

src/paperless_tesseract/checks.py

@@ -1,7 +1,7 @@
 import subprocess
 
 from django.conf import settings
-from django.core.checks import Error, register
+from django.core.checks import Error, Warning, register
 
 
 def get_tesseract_langs():

src/paperless_tesseract/languages.py

@@ -1,194 +0,0 @@
-# Thanks to the Library of Congress and some creative use of sed and awk:
-# http://www.loc.gov/standards/iso639-2/php/English_list.php
-
-ISO639 = {
-
-    "aa": "aar",
-    "ab": "abk",
-    "ae": "ave",
-    "af": "afr",
-    "ak": "aka",
-    "am": "amh",
-    "an": "arg",
-    "ar": "ara",
-    "as": "asm",
-    "av": "ava",
-    "ay": "aym",
-    "az": "aze",
-    "ba": "bak",
-    "be": "bel",
-    "bg": "bul",
-    "bh": "bih",
-    "bi": "bis",
-    "bm": "bam",
-    "bn": "ben",
-    "bo": "bod",
-    "br": "bre",
-    "bs": "bos",
-    "ca": "cat",
-    "ce": "che",
-    "ch": "cha",
-    "co": "cos",
-    "cr": "cre",
-    "cs": "ces",
-    "cu": "chu",
-    "cv": "chv",
-    "cy": "cym",
-    "da": "dan",
-    "de": "deu",
-    "dv": "div",
-    "dz": "dzo",
-    "ee": "ewe",
-    "el": "ell",
-    "en": "eng",
-    "eo": "epo",
-    "es": "spa",
-    "et": "est",
-    "eu": "eus",
-    "fa": "fas",
-    "ff": "ful",
-    "fi": "fin",
-    "fj": "fij",
-    "fo": "fao",
-    "fr": "fra",
-    "fy": "fry",
-    "ga": "gle",
-    "gd": "gla",
-    "gl": "glg",
-    "gn": "grn",
-    "gu": "guj",
-    "gv": "glv",
-    "ha": "hau",
-    "he": "heb",
-    "hi": "hin",
-    "ho": "hmo",
-    "hr": "hrv",
-    "ht": "hat",
-    "hu": "hun",
-    "hy": "hye",
-    "hz": "her",
-    "ia": "ina",
-    "id": "ind",
-    "ie": "ile",
-    "ig": "ibo",
-    "ii": "iii",
-    "ik": "ipk",
-    "io": "ido",
-    "is": "isl",
-    "it": "ita",
-    "iu": "iku",
-    "ja": "jpn",
-    "jv": "jav",
-    "ka": "kat",
-    "kg": "kon",
-    "ki": "kik",
-    "kj": "kua",
-    "kk": "kaz",
-    "kl": "kal",
-    "km": "khm",
-    "kn": "kan",
-    "ko": "kor",
-    "kr": "kau",
-    "ks": "kas",
-    "ku": "kur",
-    "kv": "kom",
-    "kw": "cor",
-    "ky": "kir",
-    "la": "lat",
-    "lb": "ltz",
-    "lg": "lug",
-    "li": "lim",
-    "ln": "lin",
-    "lo": "lao",
-    "lt": "lit",
-    "lu": "lub",
-    "lv": "lav",
-    "mg": "mlg",
-    "mh": "mah",
-    "mi": "mri",
-    "mk": "mkd",
-    "ml": "mal",
-    "mn": "mon",
-    "mr": "mar",
-    "ms": "msa",
-    "mt": "mlt",
-    "my": "mya",
-    "na": "nau",
-    "nb": "nob",
-    "nd": "nde",
-    "ne": "nep",
-    "ng": "ndo",
-    "nl": "nld",
-    "no": "nor",
-    "nr": "nbl",
-    "nv": "nav",
-    "ny": "nya",
-    "oc": "oci",
-    "oj": "oji",
-    "om": "orm",
-    "or": "ori",
-    "os": "oss",
-    "pa": "pan",
-    "pi": "pli",
-    "pl": "pol",
-    "ps": "pus",
-    "pt": "por",
-    "qu": "que",
-    "rm": "roh",
-    "rn": "run",
-    "ro": "ron",
-    "ru": "rus",
-    "rw": "kin",
-    "sa": "san",
-    "sc": "srd",
-    "sd": "snd",
-    "se": "sme",
-    "sg": "sag",
-    "si": "sin",
-    "sk": "slk",
-    "sl": "slv",
-    "sm": "smo",
-    "sn": "sna",
-    "so": "som",
-    "sq": "sqi",
-    "sr": "srp",
-    "ss": "ssw",
-    "st": "sot",
-    "su": "sun",
-    "sv": "swe",
-    "sw": "swa",
-    "ta": "tam",
-    "te": "tel",
-    "tg": "tgk",
-    "th": "tha",
-    "ti": "tir",
-    "tk": "tuk",
-    "tl": "tgl",
-    "tn": "tsn",
-    "to": "ton",
-    "tr": "tur",
-    "ts": "tso",
-    "tt": "tat",
-    "tw": "twi",
-    "ty": "tah",
-    "ug": "uig",
-    "uk": "ukr",
-    "ur": "urd",
-    "uz": "uzb",
-    "ve": "ven",
-    "vi": "vie",
-    "vo": "vol",
-    "wa": "wln",
-    "wo": "wol",
-    "xh": "xho",
-    "yi": "yid",
-    "yo": "yor",
-    "za": "zha",
-
-    # Tessdata contains two values for Chinese, "chi_sim" and "chi_tra".  I
-    # have no idea which one is better, so I just picked the bigger file.
-    "zh": "chi_tra",
-
-    "zu": "zul"
-
-}

src/paperless_tesseract/parsers.py

@@ -1,15 +1,16 @@
 import json
 import os
 import re
-import subprocess
 
 import ocrmypdf
 import pdftotext
+import pikepdf
 from PIL import Image
 from django.conf import settings
 from ocrmypdf import InputFileError, EncryptedPdfError
 
-from documents.parsers import DocumentParser, ParseError, run_convert
+from documents.parsers import DocumentParser, ParseError, \
+    make_thumbnail_from_pdf
 
 
 class RasterisedDocumentParser(DocumentParser):
@@ -18,49 +19,36 @@ class RasterisedDocumentParser(DocumentParser):
     image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.)
     """
 
+    def extract_metadata(self, document_path, mime_type):
+        namespace_pattern = re.compile(r"\{(.*)\}(.*)")
+
+        result = []
+        if mime_type == 'application/pdf':
+            pdf = pikepdf.open(document_path)
+            meta = pdf.open_metadata()
+            for key, value in meta.items():
+                if isinstance(value, list):
+                    value = " ".join([str(e) for e in value])
+                value = str(value)
+                try:
+                    m = namespace_pattern.match(key)
+                    result.append({
+                        "namespace": m.group(1),
+                        "prefix": meta.REVERSE_NS[m.group(1)],
+                        "key": m.group(2),
+                        "value": value
+                    })
+                except Exception as e:
+                    self.log(
+                        "warning",
+                        f"Error while reading metadata {key}: {value}. Error: "
+                        f"{e}"
+                    )
+        return result
+
     def get_thumbnail(self, document_path, mime_type):
-        """
-        The thumbnail of a PDF is just a 500px wide image of the first page.
-        """
-
-        out_path = os.path.join(self.tempdir, "convert.png")
-
-        # Run convert to get a decent thumbnail
-        try:
-            run_convert(density=300,
-                        scale="500x5000>",
-                        alpha="remove",
-                        strip=True,
-                        trim=False,
-                        input_file="{}[0]".format(document_path),
-                        output_file=out_path,
-                        logging_group=self.logging_group)
-        except ParseError:
-            # if convert fails, fall back to extracting
-            # the first PDF page as a PNG using Ghostscript
-            self.log(
-                'warning',
-                "Thumbnail generation with ImageMagick failed, falling back "
-                "to ghostscript. Check your /etc/ImageMagick-x/policy.xml!")
-            gs_out_path = os.path.join(self.tempdir, "gs_out.png")
-            cmd = [settings.GS_BINARY,
-                   "-q",
-                   "-sDEVICE=pngalpha",
-                   "-o", gs_out_path,
-                   document_path]
-            if not subprocess.Popen(cmd).wait() == 0:
-                raise ParseError("Thumbnail (gs) failed at {}".format(cmd))
-            # then run convert on the output from gs
-            run_convert(density=300,
-                        scale="500x5000>",
-                        alpha="remove",
-                        strip=True,
-                        trim=False,
-                        input_file=gs_out_path,
-                        output_file=out_path,
-                        logging_group=self.logging_group)
-
-        return out_path
+        return make_thumbnail_from_pdf(
+            document_path, self.tempdir, self.logging_group)
 
     def is_image(self, mime_type):
         return mime_type in [
@@ -82,7 +70,25 @@ class RasterisedDocumentParser(DocumentParser):
                 f"Error while getting DPI from image {image}: {e}")
             return None
 
-    def parse(self, document_path, mime_type):
+    def calculate_a4_dpi(self, image):
+        try:
+            with Image.open(image) as im:
+                width, height = im.size
+                # divide image width by A4 width (210mm) in inches.
+                dpi = int(width / (21 / 2.54))
+                self.log(
+                    'debug',
+                    f"Estimated DPI {dpi} based on image width {width}"
+                )
+                return dpi
+
+        except Exception as e:
+            self.log(
+                'warning',
+                f"Error while calculating DPI for image {image}: {e}")
+            return None
+
+    def parse(self, document_path, mime_type, file_name=None):
         mode = settings.OCR_MODE
 
         text_original = get_text_from_pdf(document_path)
@@ -134,6 +140,7 @@ class RasterisedDocumentParser(DocumentParser):
 
         if self.is_image(mime_type):
             dpi = self.get_dpi(document_path)
+            a4_dpi = self.calculate_a4_dpi(document_path)
             if dpi:
                 self.log(
                     "debug",
@@ -142,6 +149,8 @@ class RasterisedDocumentParser(DocumentParser):
                 ocr_args['image_dpi'] = dpi
             elif settings.OCR_IMAGE_DPI:
                 ocr_args['image_dpi'] = settings.OCR_IMAGE_DPI
+            elif a4_dpi:
+                ocr_args['image_dpi'] = a4_dpi
             else:
                 raise ParseError(
                     f"Cannot produce archive PDF for image {document_path}, "
@@ -213,6 +222,9 @@ def strip_excess_whitespace(text):
 
 def get_text_from_pdf(pdf_file):
 
+    if not os.path.isfile(pdf_file):
+        return None
+
     with open(pdf_file, "rb") as f:
         try:
             pdf = pdftotext.PDF(f)

src/paperless_tesseract/tests/test_checks.py

@@ -0,0 +1,26 @@
+from unittest import mock
+
+from django.core.checks import ERROR
+from django.test import TestCase, override_settings
+
+from paperless_tesseract import check_default_language_available
+
+
+class TestChecks(TestCase):
+
+    def test_default_language(self):
+        msgs = check_default_language_available(None)
+
+    @override_settings(OCR_LANGUAGE="")
+    def test_no_language(self):
+        msgs = check_default_language_available(None)
+        self.assertEqual(len(msgs), 1)
+        self.assertTrue(msgs[0].msg.startswith("No OCR language has been specified with PAPERLESS_OCR_LANGUAGE"))
+
+    @override_settings(OCR_LANGUAGE="ita")
+    @mock.patch("paperless_tesseract.checks.get_tesseract_langs")
+    def test_invalid_language(self, m):
+        m.return_value = ["deu", "eng"]
+        msgs = check_default_language_available(None)
+        self.assertEqual(len(msgs), 1)
+        self.assertEqual(msgs[0].level, ERROR)

src/paperless_tesseract/tests/test_parser.py

@@ -78,7 +78,7 @@ class TestParser(DirectoriesMixin, TestCase):
         parser.get_thumbnail(os.path.join(self.SAMPLE_FILES, 'simple-digital.pdf'), "application/pdf")
         # dont really know how to test it, just call it and assert that it does not raise anything.
 
-    @mock.patch("paperless_tesseract.parsers.run_convert")
+    @mock.patch("documents.parsers.run_convert")
     def test_thumbnail_fallback(self, m):
 
         def call_convert(input_file, output_file, **kwargs):
@@ -164,8 +164,21 @@ class TestParser(DirectoriesMixin, TestCase):
 
         self.assertRaises(ParseError, f)
 
+    @mock.patch("paperless_tesseract.parsers.ocrmypdf.ocr")
+    def test_image_calc_a4_dpi(self, m):
+        parser = RasterisedDocumentParser(None)
 
-    def test_image_no_dpi_fail(self):
+        parser.parse(os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"), "image/png")
+
+        m.assert_called_once()
+
+        args, kwargs = m.call_args
+
+        self.assertEqual(kwargs['image_dpi'], 62)
+
+    @mock.patch("paperless_tesseract.parsers.RasterisedDocumentParser.calculate_a4_dpi")
+    def test_image_dpi_fail(self, m):
+        m.return_value = None
         parser = RasterisedDocumentParser(None)
 
         def f():

src/paperless_text/parsers.py

@@ -1,9 +1,9 @@
 import os
-import subprocess
 
+from PIL import ImageDraw, ImageFont, Image
 from django.conf import settings
 
-from documents.parsers import DocumentParser, ParseError
+from documents.parsers import DocumentParser
 
 
 class TextDocumentParser(DocumentParser):
@@ -12,78 +12,26 @@ class TextDocumentParser(DocumentParser):
     """
 
     def get_thumbnail(self, document_path, mime_type):
-        """
-        The thumbnail of a text file is just a 500px wide image of the text
-        rendered onto a letter-sized page.
-        """
-        # The below is heavily cribbed from https://askubuntu.com/a/590951
-
-        bg_color = "white"  # bg color
-        text_color = "black"  # text color
-        psize = [500, 647]  # icon size
-        n_lines = 50  # number of lines to show
-        out_path = os.path.join(self.tempdir, "convert.png")
-
-        temp_bg = os.path.join(self.tempdir, "bg.png")
-        temp_txlayer = os.path.join(self.tempdir, "tx.png")
-        picsize = "x".join([str(n) for n in psize])
-        txsize = "x".join([str(n - 8) for n in psize])
-
-        def create_bg():
-            work_size = ",".join([str(n - 1) for n in psize])
-            r = str(round(psize[0] / 10))
-            rounded = ",".join([r, r])
-            run_command(
-                settings.CONVERT_BINARY,
-                "-size ", picsize,
-                ' xc:none -draw ',
-                '"fill ', bg_color, ' roundrectangle 0,0,', work_size, ",", rounded, '" ',  # NOQA: E501
-                temp_bg
-            )
 
         def read_text():
             with open(document_path, 'r') as src:
                 lines = [line.strip() for line in src.readlines()]
-                text = "\n".join([line for line in lines[:n_lines]])
-                return text.replace('"', "'")
+                text = "\n".join(lines[:50])
+                return text
 
-        def create_txlayer():
-            run_command(
-                settings.CONVERT_BINARY,
-                "-background none",
-                "-fill",
-                text_color,
-                "-pointsize", "12",
-                "-border 4 -bordercolor none",
-                "-size ", txsize,
-                ' caption:"', read_text(), '" ',
-                temp_txlayer
-            )
+        img = Image.new("RGB", (500, 700), color="white")
+        draw = ImageDraw.Draw(img)
+        font = ImageFont.truetype(
+            font=settings.THUMBNAIL_FONT_NAME,
+            size=20,
+            layout_engine=ImageFont.LAYOUT_BASIC)
+        draw.text((5, 5), read_text(), font=font, fill="black")
 
-        create_txlayer()
-        create_bg()
-        run_command(
-            settings.CONVERT_BINARY,
-            temp_bg,
-            temp_txlayer,
-            "-background None -layers merge ",
-            out_path
-        )
+        out_path = os.path.join(self.tempdir, "thumb.png")
+        img.save(out_path)
 
         return out_path
 
-    def parse(self, document_path, mime_type):
+    def parse(self, document_path, mime_type, file_name=None):
         with open(document_path, 'r') as f:
             self.text = f.read()
-
-
-def run_command(*args):
-    environment = os.environ.copy()
-    if settings.CONVERT_MEMORY_LIMIT:
-        environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
-    if settings.CONVERT_TMPDIR:
-        environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR
-
-    if not subprocess.Popen(' '.join(args), env=environment,
-                            shell=True).wait() == 0:
-        raise ParseError("Convert failed at {}".format(args))

src/paperless_text/tests/samples/test.txt

@@ -0,0 +1 @@
+This is a test file.

src/paperless_text/tests/test_parser.py

@@ -0,0 +1,26 @@
+import os
+
+from django.test import TestCase
+
+from documents.tests.utils import DirectoriesMixin
+from paperless_text.parsers import TextDocumentParser
+
+
+class TestTextParser(DirectoriesMixin, TestCase):
+
+    def test_thumbnail(self):
+
+        parser = TextDocumentParser(None)
+
+        # just make sure that it does not crash
+        f = parser.get_thumbnail(os.path.join(os.path.dirname(__file__), "samples", "test.txt"), "text/plain")
+        self.assertTrue(os.path.isfile(f))
+
+    def test_parse(self):
+
+        parser = TextDocumentParser(None)
+
+        parser.parse(os.path.join(os.path.dirname(__file__), "samples", "test.txt"), "text/plain")
+
+        self.assertEqual(parser.get_text(), "This is a test file.\n")
+        self.assertIsNone(parser.get_archive_path())

src/paperless_tika/apps.py

@@ -0,0 +1,14 @@
+from django.apps import AppConfig
+from django.conf import settings
+from paperless_tika.signals import tika_consumer_declaration
+
+
+class PaperlessTikaConfig(AppConfig):
+    name = "paperless_tika"
+
+    def ready(self):
+        from documents.signals import document_consumer_declaration
+
+        if settings.PAPERLESS_TIKA_ENABLED:
+            document_consumer_declaration.connect(tika_consumer_declaration)
+        AppConfig.ready(self)

src/paperless_tika/parsers.py

@@ -0,0 +1,87 @@
+import os
+import requests
+import dateutil.parser
+
+from django.conf import settings
+
+from documents.parsers import DocumentParser, ParseError, \
+    make_thumbnail_from_pdf
+from tika import parser
+
+
+class TikaDocumentParser(DocumentParser):
+    """
+    This parser sends documents to a local tika server
+    """
+
+    def get_thumbnail(self, document_path, mime_type):
+        if not self.archive_path:
+            self.archive_path = self.convert_to_pdf(document_path)
+
+        return make_thumbnail_from_pdf(
+            self.archive_path, self.tempdir, self.logging_group)
+
+    def extract_metadata(self, document_path, mime_type):
+        tika_server = settings.PAPERLESS_TIKA_ENDPOINT
+        try:
+            parsed = parser.from_file(document_path, tika_server)
+        except Exception as e:
+            self.log("warning", f"Error while fetching document metadata for "
+                                f"{document_path}: {e}")
+            return []
+
+        return [
+            {
+                "namespace": "",
+                "prefix": "",
+                "key": key,
+                "value": parsed['metadata'][key]
+            } for key in parsed['metadata']
+        ]
+
+    def parse(self, document_path, mime_type, file_name=None):
+        self.log("info", f"Sending {document_path} to Tika server")
+        tika_server = settings.PAPERLESS_TIKA_ENDPOINT
+
+        try:
+            parsed = parser.from_file(document_path, tika_server)
+        except Exception as err:
+            raise ParseError(
+                f"Could not parse {document_path} with tika server at "
+                f"{tika_server}: {err}"
+            )
+
+        self.text = parsed["content"].strip()
+
+        try:
+            self.date = dateutil.parser.isoparse(
+                parsed["metadata"]["Creation-Date"])
+        except Exception as e:
+            self.log("warning", f"Unable to extract date for document "
+                                f"{document_path}: {e}")
+
+        self.archive_path = self.convert_to_pdf(document_path, file_name)
+
+    def convert_to_pdf(self, document_path, file_name):
+        pdf_path = os.path.join(self.tempdir, "convert.pdf")
+        gotenberg_server = settings.PAPERLESS_TIKA_GOTENBERG_ENDPOINT
+        url = gotenberg_server + "/convert/office"
+
+        self.log("info", f"Converting {document_path} to PDF as {pdf_path}")
+        files = {"files": (file_name or os.path.basename(document_path),
+                           open(document_path, "rb"))}
+        headers = {}
+
+        try:
+            response = requests.post(url, files=files, headers=headers)
+            response.raise_for_status()  # ensure we notice bad responses
+        except Exception as err:
+            raise ParseError(
+                f"Error while converting document to PDF: {err}"
+            )
+
+        file = open(pdf_path, "wb")
+        file.write(response.content)
+        file.close()
+
+        return pdf_path

src/paperless_tika/signals.py

@@ -0,0 +1,20 @@
+from .parsers import TikaDocumentParser
+
+
+def tika_consumer_declaration(sender, **kwargs):
+    return {
+        "parser": TikaDocumentParser,
+        "weight": 10,
+        "mime_types": {
+            "application/msword": ".doc",
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",  # NOQA: E501
+            "application/vnd.ms-excel": ".xls",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",  # NOQA: E501
+            "application/vnd.ms-powerpoint": ".ppt",
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",  # NOQA: E501
+            "application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx",  # NOQA: E501
+            "application/vnd.oasis.opendocument.presentation": ".odp",
+            "application/vnd.oasis.opendocument.spreadsheet": ".ods",
+            "application/vnd.oasis.opendocument.text": ".odt",
+        },
+    }

src/paperless_tika/tests/test_tika_parser.py

@@ -0,0 +1,60 @@
+import datetime
+import os
+from pathlib import Path
+from unittest import mock
+
+from django.test import TestCase
+from requests import Response
+
+from paperless_tika.parsers import TikaDocumentParser
+
+
+class TestTikaParser(TestCase):
+
+    def setUp(self) -> None:
+        self.parser = TikaDocumentParser(logging_group=None)
+
+    def tearDown(self) -> None:
+        self.parser.cleanup()
+
+    @mock.patch("paperless_tika.parsers.parser.from_file")
+    @mock.patch("paperless_tika.parsers.requests.post")
+    def test_parse(self, post, from_file):
+        from_file.return_value = {
+            "content": "the content",
+            "metadata": {
+                "Creation-Date": "2020-11-21"
+            }
+        }
+        response = Response()
+        response._content = b"PDF document"
+        response.status_code = 200
+        post.return_value = response
+
+        file = os.path.join(self.parser.tempdir, "input.odt")
+        Path(file).touch()
+        self.parser.parse(file, "application/vnd.oasis.opendocument.text")
+
+        self.assertEqual(self.parser.text, "the content")
+        self.assertIsNotNone(self.parser.archive_path)
+        with open(self.parser.archive_path, "rb") as f:
+            self.assertEqual(f.read(), b"PDF document")
+
+        self.assertEqual(self.parser.date, datetime.datetime(2020, 11, 21))
+
+    @mock.patch("paperless_tika.parsers.parser.from_file")
+    def test_metadata(self, from_file):
+        from_file.return_value = {
+            "metadata": {
+                "Creation-Date": "2020-11-21",
+                "Some-key": "value"
+            }
+        }
+
+        file = os.path.join(self.parser.tempdir, "input.odt")
+        Path(file).touch()
+
+        metadata = self.parser.extract_metadata(file, "application/vnd.oasis.opendocument.text")
+
+        self.assertTrue("Creation-Date" in [m['key'] for m in metadata])
+        self.assertTrue("Some-key" in [m['key'] for m in metadata])