Merge branch 'jonaswinkler-new-features'

.gitignore

@@ -81,3 +81,5 @@ docker-compose.env
 scripts/import-for-development
 scripts/nuke
 
+# Static files collected by the collectstatic command
+static/

paperless.conf.example

@@ -59,6 +59,11 @@ PAPERLESS_EMAIL_SECRET=""
 ####                              Security                                 ####
 ###############################################################################
 
+# Controls whether django's debug mode is enabled. Disable this on production
+# systems. Debug mode is enabled by default.
+PAPERLESS_DEBUG="false"
+
+
 # Paperless can be instructed to attempt to encrypt your PDF files with GPG
 # using the PAPERLESS_PASSPHRASE specified below.  If however you're not
 # concerned about encrypting these files (for example if you have disk
@@ -203,3 +208,8 @@ PAPERLESS_EMAIL_SECRET=""
 # positive integer, but if you don't define one in paperless.conf, a default of
 # 100 will be used.
 #PAPERLESS_LIST_PER_PAGE=100
+
+
+# The number of years for which a correspondent will be included in the recent
+# correspondents filter.
+#PAPERLESS_RECENT_CORRESPONDENT_YEARS=1

requirements.txt

@@ -1,10 +1,10 @@
 -i https://pypi.python.org/simple
-apipkg==1.5; python_version != '3.1.*'
+apipkg==1.5; python_version != '3.3.*'
-atomicwrites==1.2.1; python_version != '3.1.*'
+atomicwrites==1.2.1; python_version != '3.3.*'
 attrs==18.2.0
 certifi==2018.8.24
 chardet==3.0.4
-coverage==4.5.1; python_version != '3.1.*'
+coverage==4.5.1; python_version < '4'
 coveralls==1.5.0
 dateparser==0.7.0
 django-cors-headers==2.4.0
@@ -14,9 +14,9 @@ django-filter==2.0.0
 django==2.0.8
 djangorestframework==3.8.2
 docopt==0.6.2
-execnet==1.5.0; python_version != '3.1.*'
+execnet==1.5.0; python_version != '3.3.*'
 factory-boy==2.11.1
-faker==0.9.0
+faker==0.9.0; python_version >= '2.7'
 filemagic==1.6
 fuzzywuzzy==0.15.0
 gunicorn==19.9.0
@@ -26,17 +26,17 @@ langdetect==1.0.7
 more-itertools==4.3.0
 pdftotext==2.1.0
 pillow==5.2.0
-pluggy==0.7.1; python_version != '3.1.*'
+pluggy==0.7.1; python_version != '3.3.*'
-py==1.6.0; python_version != '3.1.*'
+py==1.6.0; python_version != '3.3.*'
 pycodestyle==2.4.0
 pyocr==0.5.3
-pytest-cov==2.5.1
+pytest-cov==2.6.0
 pytest-django==3.4.2
 pytest-env==0.6.2
-pytest-forked==0.2
+pytest-forked==0.2; python_version != '3.3.*'
 pytest-sugar==0.9.1
 pytest-xdist==1.23.0
-pytest==3.7.4
+pytest==3.8.0
 python-dateutil==2.7.3
 python-dotenv==0.9.1
 python-gnupg==0.4.3
@@ -48,4 +48,4 @@ six==1.11.0
 termcolor==1.1.0
 text-unidecode==1.2
 tzlocal==1.5.1
-urllib3==1.23; python_version != '3.0.*'
+urllib3==1.23; python_version != '3.3.*'

src/documents/actions.py

@@ -0,0 +1,146 @@
+from django.contrib import messages
+from django.contrib.admin import helpers
+from django.contrib.admin.utils import model_ngettext
+from django.core.exceptions import PermissionDenied
+from django.template.response import TemplateResponse
+
+from documents.models import Correspondent, Tag
+
+
+def select_action(
+        modeladmin, request, queryset, title, action, modelclass,
+        success_message="", document_action=None, queryset_action=None):
+
+    opts = modeladmin.model._meta
+    app_label = opts.app_label
+
+    if not modeladmin.has_change_permission(request):
+        raise PermissionDenied
+
+    if request.POST.get('post'):
+        n = queryset.count()
+        selected_object = modelclass.objects.get(id=request.POST.get('obj_id'))
+        if n:
+            for document in queryset:
+                if document_action:
+                    document_action(document, selected_object)
+                document_display = str(document)
+                modeladmin.log_change(request, document, document_display)
+            if queryset_action:
+                queryset_action(queryset, selected_object)
+
+            modeladmin.message_user(request, success_message % {
+                "selected_object": selected_object.name,
+                "count": n,
+                "items": model_ngettext(modeladmin.opts, n)
+            }, messages.SUCCESS)
+
+        # Return None to display the change list page again.
+        return None
+
+    context = dict(
+        modeladmin.admin_site.each_context(request),
+        title=title,
+        queryset=queryset,
+        opts=opts,
+        action_checkbox_name=helpers.ACTION_CHECKBOX_NAME,
+        media=modeladmin.media,
+        action=action,
+        objects=modelclass.objects.all(),
+        itemname=model_ngettext(modelclass, 1)
+    )
+
+    request.current_app = modeladmin.admin_site.name
+
+    return TemplateResponse(
+        request,
+        "admin/{}/{}/select_object.html".format(app_label, opts.model_name),
+        context
+    )
+
+
+def simple_action(
+        modeladmin, request, queryset, success_message="",
+        document_action=None, queryset_action=None):
+
+    if not modeladmin.has_change_permission(request):
+        raise PermissionDenied
+
+    n = queryset.count()
+    if n:
+        for document in queryset:
+            if document_action:
+                document_action(document)
+            document_display = str(document)
+            modeladmin.log_change(request, document, document_display)
+        if queryset_action:
+            queryset_action(queryset)
+        modeladmin.message_user(request, success_message % {
+            "count": n, "items": model_ngettext(modeladmin.opts, n)
+        }, messages.SUCCESS)
+
+    # Return None to display the change list page again.
+    return None
+
+
+def add_tag_to_selected(modeladmin, request, queryset):
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Add tag to multiple documents",
+        action="add_tag_to_selected",
+        modelclass=Tag,
+        success_message="Successfully added tag %(selected_object)s to "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.add(tag)
+    )
+
+
+def remove_tag_from_selected(modeladmin, request, queryset):
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Remove tag from multiple documents",
+        action="remove_tag_from_selected",
+        modelclass=Tag,
+        success_message="Successfully removed tag %(selected_object)s from "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.remove(tag)
+    )
+
+
+def set_correspondent_on_selected(modeladmin, request, queryset):
+
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Set correspondent on multiple documents",
+        action="set_correspondent_on_selected",
+        modelclass=Correspondent,
+        success_message="Successfully set correspondent %(selected_object)s "
+                        "on %(count)d %(items)s.",
+        queryset_action=lambda qs, corr: qs.update(correspondent=corr)
+    )
+
+
+def remove_correspondent_from_selected(modeladmin, request, queryset):
+    return simple_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        success_message="Successfully removed correspondent from %(count)d "
+                        "%(items)s.",
+        queryset_action=lambda qs: qs.update(correspondent=None)
+    )
+
+
+add_tag_to_selected.short_description = "Add tag to selected documents"
+remove_tag_from_selected.short_description = \
+    "Remove tag from selected documents"
+set_correspondent_on_selected.short_description = \
+    "Set correspondent on selected documents"
+remove_correspondent_from_selected.short_description = \
+    "Remove correspondent from selected documents"

src/documents/admin.py

@@ -1,42 +1,25 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from django.conf import settings
-from django.contrib import admin
+from django.contrib import admin, messages
-from django.contrib.auth.models import User, Group
+from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
-try:
+from django.contrib.auth.models import Group, User
-    from django.core.urlresolvers import reverse
+from django.db import models
-except ImportError:
+from django.http import HttpResponseRedirect
-    from django.urls import reverse
 from django.templatetags.static import static
-from django.utils.safestring import mark_safe
+from django.urls import reverse
 from django.utils.html import format_html, format_html_join
+from django.utils.http import urlquote
+from django.utils.safestring import mark_safe
 
-from .models import Correspondent, Tag, Document, Log
+from documents.actions import (
+    add_tag_to_selected,
+    remove_correspondent_from_selected,
+    remove_tag_from_selected,
+    set_correspondent_on_selected
+)
 
-
+from .models import Correspondent, Document, Log, Tag
-class MonthListFilter(admin.SimpleListFilter):
-
-    title = "Month"
-
-    # Parameter for the filter that will be used in the URL query.
-    parameter_name = "month"
-
-    def lookups(self, request, model_admin):
-        r = []
-        for document in Document.objects.all():
-            r.append((
-                document.created.strftime("%Y-%m"),
-                document.created.strftime("%B %Y")
-            ))
-        return sorted(set(r), key=lambda x: x[0], reverse=True)
-
-    def queryset(self, request, queryset):
-
-        if not self.value():
-            return None
-
-        year, month = self.value().split("-")
-        return queryset.filter(created__year=year, created__month=month)
 
 
 class FinancialYearFilter(admin.SimpleListFilter):
@@ -104,18 +87,59 @@ class FinancialYearFilter(admin.SimpleListFilter):
                                created__lte=self._fy_end(end))
 
 
+class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.title = "correspondent (recent)"
+
+    def field_choices(self, field, request, model_admin):
+
+        years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
+        days = 365 * years
+
+        lookups = []
+        if years and years > 0:
+            correspondents = Correspondent.objects.filter(
+                documents__created__gte=datetime.now() - timedelta(days=days)
+            ).distinct()
+            for c in correspondents:
+                lookups.append((c.id, c.name))
+
+        return lookups
+
+
 class CommonAdmin(admin.ModelAdmin):
     list_per_page = settings.PAPERLESS_LIST_PER_PAGE
 
 
 class CorrespondentAdmin(CommonAdmin):
 
-    list_display = ("name", "match", "matching_algorithm", "document_count")
+    list_display = (
+        "name",
+        "match",
+        "matching_algorithm",
+        "document_count",
+        "last_correspondence"
+    )
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
+    def get_queryset(self, request):
+        qs = super(CorrespondentAdmin, self).get_queryset(request)
+        qs = qs.annotate(
+            document_count=models.Count("documents"),
+            last_correspondence=models.Max("documents__created")
+        )
+        return qs
+
     def document_count(self, obj):
-        return obj.documents.count()
+        return obj.document_count
+    document_count.admin_order_field = "document_count"
+
+    def last_correspondence(self, obj):
+        return obj.last_correspondence
+    last_correspondence.admin_order_field = "last_correspondence"
 
 
 class TagAdmin(CommonAdmin):
@@ -125,8 +149,14 @@ class TagAdmin(CommonAdmin):
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
+    def get_queryset(self, request):
+        qs = super(TagAdmin, self).get_queryset(request)
+        qs = qs.annotate(document_count=models.Count("documents"))
+        return qs
+
     def document_count(self, obj):
-        return obj.documents.count()
+        return obj.document_count
+    document_count.admin_order_field = "document_count"
 
 
 class DocumentAdmin(CommonAdmin):
@@ -140,12 +170,30 @@ class DocumentAdmin(CommonAdmin):
     readonly_fields = ("added",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_")
-    list_filter = ("tags", "correspondent", FinancialYearFilter,
+    list_filter = (
-                   MonthListFilter)
+        "tags",
+        ("correspondent", RecentCorrespondentFilter),
+        "correspondent",
+        FinancialYearFilter
+    )
+
     filter_horizontal = ("tags",)
 
     ordering = ["-created", "correspondent"]
 
+    actions = [
+        add_tag_to_selected,
+        remove_tag_from_selected,
+        set_correspondent_on_selected,
+        remove_correspondent_from_selected
+    ]
+
+    date_hierarchy = "created"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.document_queue = []
+
     def has_add_permission(self, request):
         return False
 
@@ -153,6 +201,79 @@ class DocumentAdmin(CommonAdmin):
         return obj.created.date().strftime("%Y-%m-%d")
     created_.short_description = "Created"
 
+    def changelist_view(self, request, extra_context=None):
+
+        response = super().changelist_view(
+            request,
+            extra_context=extra_context
+        )
+
+        if request.method == "GET":
+            cl = self.get_changelist_instance(request)
+            self.document_queue = [doc.id for doc in cl.queryset]
+
+        return response
+
+    def change_view(self, request, object_id=None, form_url='',
+                    extra_context=None):
+
+        extra_context = extra_context or {}
+
+        if self.document_queue and object_id:
+            if int(object_id) in self.document_queue:
+                # There is a queue of documents
+                current_index = self.document_queue.index(int(object_id))
+                if current_index < len(self.document_queue) - 1:
+                    # ... and there are still documents in the queue
+                    extra_context["next_object"] = self.document_queue[
+                        current_index + 1
+                    ]
+
+        return super(DocumentAdmin, self).change_view(
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
+        )
+
+    def response_change(self, request, obj):
+
+        # This is mostly copied from ModelAdmin.response_change()
+        opts = self.model._meta
+        preserved_filters = self.get_preserved_filters(request)
+
+        msg_dict = {
+            "name": opts.verbose_name,
+            "obj": format_html(
+                '<a href="{}">{}</a>',
+                urlquote(request.path),
+                obj
+            ),
+        }
+        if "_saveandeditnext" in request.POST:
+            msg = format_html(
+                'The {name} "{obj}" was changed successfully. '
+                'Editing next object.',
+                **msg_dict
+            )
+            self.message_user(request, msg, messages.SUCCESS)
+            redirect_url = reverse(
+                "admin:{}_{}_change".format(opts.app_label, opts.model_name),
+                args=(request.POST["_next_object"],),
+                current_app=self.admin_site.name
+            )
+            redirect_url = add_preserved_filters(
+                {
+                    "preserved_filters": preserved_filters,
+                    "opts": opts
+                },
+                redirect_url
+            )
+            return HttpResponseRedirect(redirect_url)
+
+        return super().response_change(request, obj)
+
+    @mark_safe
     def thumbnail(self, obj):
         return self._html_tag(
             "a",
@@ -165,8 +286,8 @@ class DocumentAdmin(CommonAdmin):
             ),
             href=obj.download_url
         )
-    thumbnail.allow_tags = True
 
+    @mark_safe
     def tags_(self, obj):
         r = ""
         for tag in obj.tags.all():
@@ -183,10 +304,11 @@ class DocumentAdmin(CommonAdmin):
                     )
                 }
             )
-        return mark_safe(r)
+        return r
-    tags_.allow_tags = True
 
+    @mark_safe
     def document(self, obj):
+        # TODO: is this method even used anymore?
         return self._html_tag(
             "a",
             self._html_tag(
@@ -199,7 +321,6 @@ class DocumentAdmin(CommonAdmin):
             ),
             href=obj.download_url
         )
-    document.allow_tags = True
 
     @staticmethod
     def _html_tag(kind, inside=None, **kwargs):

src/documents/templates/admin/documents/document/change_form.html

@@ -1,5 +1,21 @@
 {% extends 'admin/change_form.html' %}
 
+{% block content %}
+
+{{ block.super }}
+
+{% if next_object %}
+	<script type="text/javascript">//<![CDATA[
+		(function($){
+			$('<input type="submit" value="Save and edit next" name="_saveandeditnext" />')
+			.prependTo('div.submit-row');
+			$('<input type="hidden" value="{{next_object}}" name="_next_object" />')
+			.prependTo('div.submit-row');
+		})(django.jQuery);
+	//]]></script>
+{% endif %}
+
+{% endblock content %}
 
 {% block footer %}
 

src/documents/templates/admin/documents/document/select_object.html

@@ -0,0 +1,50 @@
+{% extends "admin/base_site.html" %}
+
+
+{% load i18n l10n admin_urls static %}
+{% load staticfiles %}
+
+
+{% block extrahead %}
+	{{ block.super }}
+	{{ media }}
+	<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
+{% endblock %}
+
+
+{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
+
+
+{% block breadcrumbs %}
+	<div class="breadcrumbs">
+		<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+		&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+		&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+		&rsaquo; {{ title }}
+	</div>
+{% endblock %}
+
+{% block content %}
+	<p>Please select the {{itemname}}.</p>
+	<form method="post">{% csrf_token %}
+		<div>
+			{% for obj in queryset %}
+			<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
+			{% endfor %}
+			<p>
+				<select name="obj_id">
+					{% for obj in objects %}
+					<option value="{{ obj.id }}">{{ obj.name }}</option>
+					{% endfor %}
+				</select>
+			</p>
+
+			<input type="hidden" name="action" value="{{ action }}"/>
+			<input type="hidden" name="post" value="yes" />
+			<p>
+				<input type="submit" value="{% trans 'Confirm' %}" />
+				<a href="#" class="button cancel-link">{% trans "Go back" %}</a>
+			</p>
+		</div>
+	</form>
+{% endblock %}

src/paperless/settings.py

@@ -22,12 +22,12 @@ elif os.path.exists("/usr/local/etc/paperless.conf"):
     load_dotenv("/usr/local/etc/paperless.conf")
 
 
-def __get_boolean(key):
+def __get_boolean(key, default="NO"):
     """
     Return a boolean value based on whatever the user has supplied in the
     environment based on whether the value "looks like" it's True or not.
     """
-    return bool(os.getenv(key, "NO").lower() in ("yes", "y", "1", "t", "true"))
+    return bool(os.getenv(key, default).lower() in ("yes", "y", "1", "t", "true"))
 
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -47,7 +47,7 @@ SECRET_KEY = os.getenv(
 
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = __get_boolean("PAPERLESS_DEBUG", "YES")
 
 LOGIN_URL = "admin:login"
 
@@ -292,3 +292,9 @@ FY_END = os.getenv("PAPERLESS_FINANCIAL_YEAR_END")
 
 # Specify the default date order (for autodetected dates)
 DATE_ORDER = os.getenv("PAPERLESS_DATE_ORDER", "DMY")
+
+# Specify for how many years a correspondent is considered recent. Recent
+# correspondents will be shown in a separate "Recent correspondents" filter as
+# well. Set to 0 to disable this filter.
+PAPERLESS_RECENT_CORRESPONDENT_YEARS = int(os.getenv(
+    "PAPERLESS_RECENT_CORRESPONDENT_YEARS", 0))

src/paperless_tesseract/parsers.py

@@ -172,8 +172,8 @@ class RasterisedDocumentParser(DocumentParser):
                 raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
                 return raw_text
             raise OCRError(
-                "The guessed language is not available in this instance of "
+                "The guessed language ({}) is not available in this instance "
-                "Tesseract."
+                "of Tesseract.".format(guessed_language)
             )
 
     def _ocr(self, imgs, lang):