Merge branch 'master' into dev

src/documents/actions.py

@@ -5,10 +5,13 @@ from django.core.exceptions import PermissionDenied
 from django.template.response import TemplateResponse
 
 from documents.classifier import DocumentClassifier
-from documents.models import Tag, Correspondent, DocumentType
+from documents.models import Correspondent, DocumentType, Tag
 
 
-def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None):
+def select_action(
+        modeladmin, request, queryset, title, action, modelclass,
+        success_message="", document_action=None, queryset_action=None):
+
     opts = modeladmin.model._meta
     app_label = opts.app_label
 
@@ -28,7 +31,9 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
                 queryset_action(queryset, selected_object)
 
             modeladmin.message_user(request, success_message % {
-                "selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n)
+                "selected_object": selected_object.name,
+                "count": n,
+                "items": model_ngettext(modeladmin.opts, n)
             }, messages.SUCCESS)
 
         # Return None to display the change list page again.
@@ -48,10 +53,17 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
 
     request.current_app = modeladmin.admin_site.name
 
-    return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context)
+    return TemplateResponse(
+        request,
+        "admin/{}/{}/select_object.html".format(app_label, opts.model_name),
+        context
+    )
 
 
-def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None):
+def simple_action(
+        modeladmin, request, queryset, success_message="",
+        document_action=None, queryset_action=None):
+
     if not modeladmin.has_change_permission(request):
         raise PermissionDenied
 
@@ -73,40 +85,57 @@ def simple_action(modeladmin, request, queryset, success_message="", document_ac
 
 
 def add_tag_to_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Add tag to multiple documents",
-                         action="add_tag_to_selected",
-                         modelclass=Tag,
-                         success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.",
-                         document_action=lambda doc, tag: doc.tags.add(tag))
-add_tag_to_selected.short_description = "Add tag to selected documents"
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Add tag to multiple documents",
+        action="add_tag_to_selected",
+        modelclass=Tag,
+        success_message="Successfully added tag %(selected_object)s to "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.add(tag)
+    )
 
 
 def remove_tag_from_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Remove tag from multiple documents",
-                         action="remove_tag_from_selected",
-                         modelclass=Tag,
-                         success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.",
-                         document_action=lambda doc, tag: doc.tags.remove(tag))
-remove_tag_from_selected.short_description = "Remove tag from selected documents"
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Remove tag from multiple documents",
+        action="remove_tag_from_selected",
+        modelclass=Tag,
+        success_message="Successfully removed tag %(selected_object)s from "
+                        "%(count)d %(items)s.",
+        document_action=lambda doc, tag: doc.tags.remove(tag)
+    )
 
 
 def set_correspondent_on_selected(modeladmin, request, queryset):
-    return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         title="Set correspondent on multiple documents",
-                         action="set_correspondent_on_selected",
-                         modelclass=Correspondent,
-                         success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.",
-                         queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent))
-set_correspondent_on_selected.short_description = "Set correspondent on selected documents"
+
+    return select_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        title="Set correspondent on multiple documents",
+        action="set_correspondent_on_selected",
+        modelclass=Correspondent,
+        success_message="Successfully set correspondent %(selected_object)s "
+                        "on %(count)d %(items)s.",
+        queryset_action=lambda qs, corr: qs.update(correspondent=corr)
+    )
 
 
 def remove_correspondent_from_selected(modeladmin, request, queryset):
-    return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
-                         success_message="Successfully removed correspondent from %(count)d %(items)s.",
-                         queryset_action=lambda qs: qs.update(correspondent=None))
-remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents"
+    return simple_action(
+        modeladmin=modeladmin,
+        request=request,
+        queryset=queryset,
+        success_message="Successfully removed correspondent from %(count)d "
+                        "%(items)s.",
+        queryset_action=lambda qs: qs.update(correspondent=None)
+    )
 
 
 def set_document_type_on_selected(modeladmin, request, queryset):
@@ -116,14 +145,12 @@ def set_document_type_on_selected(modeladmin, request, queryset):
                          modelclass=DocumentType,
                          success_message="Successfully set document type %(selected_object)s on %(count)d %(items)s.",
                          queryset_action=lambda qs, document_type: qs.update(document_type=document_type))
-set_document_type_on_selected.short_description = "Set document type on selected documents"
 
 
 def remove_document_type_from_selected(modeladmin, request, queryset):
     return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
                          success_message="Successfully removed document type from %(count)d %(items)s.",
                          queryset_action=lambda qs: qs.update(document_type=None))
-remove_document_type_from_selected.short_description = "Remove document type from selected documents"
 
 
 def run_document_classifier_on_selected(modeladmin, request, queryset):
@@ -135,4 +162,16 @@ def run_document_classifier_on_selected(modeladmin, request, queryset):
     except FileNotFoundError:
         modeladmin.message_user(request, "Classifier model file not found.", messages.ERROR)
         return None
+
+
+add_tag_to_selected.short_description = "Add tag to selected documents"
+remove_tag_from_selected.short_description = \
+    "Remove tag from selected documents"
+set_correspondent_on_selected.short_description = \
+    "Set correspondent on selected documents"
+remove_correspondent_from_selected.short_description = \
+    "Remove correspondent from selected documents"
+set_document_type_on_selected.short_description = "Set document type on selected documents"
+remove_document_type_from_selected.short_description = "Remove document type from selected documents"
 run_document_classifier_on_selected.short_description = "Run document classifier on selected"
+

src/documents/admin.py

@@ -3,22 +3,26 @@ from datetime import datetime, timedelta
 from django.conf import settings
 from django.contrib import admin, messages
 from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
-from django.contrib.auth.models import User, Group
+from django.contrib.auth.models import Group, User
+from django.db import models
 from django.http import HttpResponseRedirect
-try:
-    from django.core.urlresolvers import reverse
-except ImportError:
-    from django.urls import reverse
 from django.templatetags.static import static
+from django.urls import reverse
 from django.utils.html import format_html, format_html_join
 from django.utils.http import urlquote
 from django.utils.safestring import mark_safe
-from django.db import models
 
-from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \
-    remove_correspondent_from_selected, set_document_type_on_selected, remove_document_type_from_selected, \
+from documents.actions import (
+    add_tag_to_selected,
+    remove_correspondent_from_selected,
+    remove_tag_from_selected,
+    set_correspondent_on_selected,
+    set_document_type_on_selected,
+    remove_document_type_from_selected,
     run_document_classifier_on_selected
-from .models import Correspondent, Tag, Document, Log, DocumentType
+)
+
+from .models import Correspondent, Document, DocumentType, Log, Tag
 
 
 class FinancialYearFilter(admin.SimpleListFilter):
@@ -93,11 +97,18 @@ class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
         self.title = "correspondent (recent)"
 
     def field_choices(self, field, request, model_admin):
+
+        years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
+        days = 365 * years
+
         lookups = []
-        if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0:
-            date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS)
-            for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct():
+        if years and years > 0:
+            correspondents = Correspondent.objects.filter(
+                documents__created__gte=datetime.now() - timedelta(days=days)
+            ).distinct()
+            for c in correspondents:
                 lookups.append((c.id, c.name))
+
         return lookups
 
 
@@ -107,12 +118,20 @@ class CommonAdmin(admin.ModelAdmin):
 
 class CorrespondentAdmin(CommonAdmin):
 
-    list_display = ("name", "automatic_classification", "document_count", "last_correspondence")
-    list_editable = ("automatic_classification",)
+    list_display = (
+        "name",
+        "automatic_classification",
+        "document_count",
+        "last_correspondence"
+    )
+    list_editable = ("automatic_classification")
 
     def get_queryset(self, request):
         qs = super(CorrespondentAdmin, self).get_queryset(request)
-        qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created"))
+        qs = qs.annotate(
+            document_count=models.Count("documents"),
+            last_correspondence=models.Max("documents__created")
+        )
         return qs
 
     def document_count(self, obj):
@@ -160,24 +179,39 @@ class DocumentAdmin(CommonAdmin):
     class Media:
         css = {
             "all": ("paperless.css",)
-
         }
 
     search_fields = ("correspondent__name", "title", "content", "tags__name")
     readonly_fields = ("added",)
     list_display = ("title", "created", "added", "thumbnail", "correspondent",
                     "tags_", "archive_serial_number", "document_type")
-    list_filter = ("document_type", "tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
+    list_filter = (
+        "document_type",
+        "tags",
+        ("correspondent", RecentCorrespondentFilter),
+        "correspondent",
+        FinancialYearFilter
+    )
 
     filter_horizontal = ("tags",)
 
     ordering = ["-created", "correspondent"]
 
-    actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected, set_document_type_on_selected, remove_document_type_from_selected, run_document_classifier_on_selected]
+    actions = [
+        add_tag_to_selected,
+        remove_tag_from_selected,
+        set_correspondent_on_selected,
+        remove_correspondent_from_selected,
+        set_document_type_on_selected,
+        remove_document_type_from_selected,
+        run_document_classifier_on_selected
+    ]
 
-    date_hierarchy = 'created'
+    date_hierarchy = "created"
 
-    document_queue = None
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.document_queue = []
 
     def has_add_permission(self, request):
         return False
@@ -187,27 +221,41 @@ class DocumentAdmin(CommonAdmin):
     created_.short_description = "Created"
 
     def changelist_view(self, request, extra_context=None):
-        response = super().changelist_view(request, extra_context)
 
-        if request.method == 'GET':
+        response = super().changelist_view(
+            request,
+            extra_context=extra_context
+        )
+
+        if request.method == "GET":
             cl = self.get_changelist_instance(request)
             self.document_queue = [doc.id for doc in cl.queryset]
 
         return response
 
-    def change_view(self, request, object_id=None, form_url='', extra_context=None):
+    def change_view(self, request, object_id=None, form_url='',
+                    extra_context=None):
+
         extra_context = extra_context or {}
         doc = Document.objects.get(id=object_id)
         extra_context['download_url'] = doc.download_url
         extra_context['file_type'] = doc.file_type
-        if self.document_queue and object_id and int(object_id) in self.document_queue:
-            # There is a queue of documents
-            current_index = self.document_queue.index(int(object_id))
-            if current_index < len(self.document_queue) - 1:
-                # ... and there are still documents in the queue
-                extra_context['next_object'] = self.document_queue[current_index + 1]
+
+        if self.document_queue and object_id:
+            if int(object_id) in self.document_queue:
+                # There is a queue of documents
+                current_index = self.document_queue.index(int(object_id))
+                if current_index < len(self.document_queue) - 1:
+                    # ... and there are still documents in the queue
+                    extra_context["next_object"] = self.document_queue[
+                        current_index + 1
+                    ]
+
         return super(DocumentAdmin, self).change_view(
-            request, object_id, form_url, extra_context=extra_context,
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
         )
 
     def response_change(self, request, obj):
@@ -217,25 +265,35 @@ class DocumentAdmin(CommonAdmin):
         preserved_filters = self.get_preserved_filters(request)
 
         msg_dict = {
-            'name': opts.verbose_name,
-            'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
+            "name": opts.verbose_name,
+            "obj": format_html(
+                '<a href="{}">{}</a>',
+                urlquote(request.path),
+                obj
+            ),
         }
         if "_saveandeditnext" in request.POST:
             msg = format_html(
-                'The {name} "{obj}" was changed successfully. Editing next object.',
+                'The {name} "{obj}" was changed successfully. '
+                'Editing next object.',
                 **msg_dict
             )
             self.message_user(request, msg, messages.SUCCESS)
-            redirect_url = reverse('admin:%s_%s_change' %
-                                   (opts.app_label, opts.model_name),
-                                   args=(request.POST['_next_object'],),
-                                   current_app=self.admin_site.name)
-            redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
-            response = HttpResponseRedirect(redirect_url)
-        else:
-            response = super().response_change(request, obj)
+            redirect_url = reverse(
+                "admin:{}_{}_change".format(opts.app_label, opts.model_name),
+                args=(request.POST["_next_object"],),
+                current_app=self.admin_site.name
+            )
+            redirect_url = add_preserved_filters(
+                {
+                    "preserved_filters": preserved_filters,
+                    "opts": opts
+                },
+                redirect_url
+            )
+            return HttpResponseRedirect(redirect_url)
 
-        return response
+        return super().response_change(request, obj)
 
     @mark_safe
     def thumbnail(self, obj):

src/documents/filters.py

@@ -3,6 +3,12 @@ from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter
 from .models import Correspondent, Document, Tag, DocumentType
 
 
+CHAR_KWARGS = (
+    "startswith", "endswith", "contains",
+    "istartswith", "iendswith", "icontains"
+)
+
+
 class CorrespondentFilterSet(FilterSet):
 
     class Meta:
@@ -44,38 +50,27 @@ class DocumentTypeFilterSet(FilterSet):
 
 class DocumentFilterSet(FilterSet):
 
-    CHAR_KWARGS = {
-        "lookup_expr": (
-            "startswith",
-            "endswith",
-            "contains",
-            "istartswith",
-            "iendswith",
-            "icontains"
-        )
-    }
-
-    correspondent__name = CharFilter(
-        field_name="correspondent__name", **CHAR_KWARGS)
-    correspondent__slug = CharFilter(
-        field_name="correspondent__slug", **CHAR_KWARGS)
-    tags__name = CharFilter(
-        field_name="tags__name", **CHAR_KWARGS)
-    tags__slug = CharFilter(
-        field_name="tags__slug", **CHAR_KWARGS)
-    tags__empty = BooleanFilter(
-        field_name="tags", lookup_expr="isnull", distinct=True)
-    document_type__name = CharFilter(
-        field_name="document_type__name", **CHAR_KWARGS)
-    document_type__slug = CharFilter(
-        field_name="document_type__slug", **CHAR_KWARGS)
+    tags_empty = BooleanFilter(
+        label="Is tagged",
+        field_name="tags",
+        lookup_expr="isnull",
+        exclude=True
+    )
 
     class Meta:
         model = Document
         fields = {
-            "title": [
-                "startswith", "endswith", "contains",
-                "istartswith", "iendswith", "icontains"
-            ],
-            "content": ["contains", "icontains"],
+
+            "title": CHAR_KWARGS,
+            "content": ("contains", "icontains"),
+
+            "correspondent__name": CHAR_KWARGS,
+            "correspondent__slug": CHAR_KWARGS,
+
+            "tags__name": CHAR_KWARGS,
+            "tags__slug": CHAR_KWARGS,
+
+            "document_type__name": CHAR_KWARGS,
+            "document_type__slug": CHAR_KWARGS,
+
         }

src/documents/management/commands/document_exporter.py

@@ -55,7 +55,12 @@ class Command(Renderable, BaseCommand):
         documents = Document.objects.all()
         document_map = {d.pk: d for d in documents}
         manifest = json.loads(serializers.serialize("json", documents))
-        for document_dict in manifest:
+
+        for index, document_dict in enumerate(manifest):
+
+            # Force output to unencrypted as that will be the current state.
+            # The importer will make the decision to encrypt or not.
+            manifest[index]["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED  # NOQA: E501
 
             document = document_map[document_dict["pk"]]
 

src/documents/management/commands/document_importer.py

@@ -94,7 +94,7 @@ class Command(Renderable, BaseCommand):
             document_path = os.path.join(self.source, doc_file)
             thumbnail_path = os.path.join(self.source, thumb_file)
 
-            if document.storage_type == Document.STORAGE_TYPE_GPG:
+            if settings.PASSPHRASE:
 
                 with open(document_path, "rb") as unencrypted:
                     with open(document.source_path, "wb") as encrypted:
@@ -112,3 +112,15 @@ class Command(Renderable, BaseCommand):
 
                 shutil.copy(document_path, document.source_path)
                 shutil.copy(thumbnail_path, document.thumbnail_path)
+
+        # Reset the storage type to whatever we've used while importing
+
+        storage_type = Document.STORAGE_TYPE_UNENCRYPTED
+        if settings.PASSPHRASE:
+            storage_type = Document.STORAGE_TYPE_GPG
+
+        Document.objects.filter(
+            pk__in=[r["pk"] for r in self.manifest]
+        ).update(
+            storage_type=storage_type
+        )

src/documents/migrations/0014_document_checksum.py

@@ -158,9 +158,4 @@ class Migration(migrations.Migration):
             name='modified',
             field=models.DateTimeField(auto_now=True, db_index=True),
         ),
-        migrations.AlterField(
-            model_name='document',
-            name='checksum',
-            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
-        ),
     ]

src/documents/migrations/0015_add_insensitive_to_match.py

@@ -12,6 +12,11 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        migrations.AlterField(
+            model_name='document',
+            name='checksum',
+            field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted).  We use this to prevent duplicate document imports.', max_length=32, unique=True),
+        ),
         migrations.AddField(
             model_name='correspondent',
             name='is_insensitive',

src/documents/templates/admin/documents/document/select_object.html

@@ -1,46 +1,50 @@
 {% extends "admin/base_site.html" %}
+
+
 {% load i18n l10n admin_urls static %}
 {% load staticfiles %}
 
-{% block extrahead %}
-{{ block.super }}
-{{ media }}
-<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 
+{% block extrahead %}
+	{{ block.super }}
+	{{ media }}
+	<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
 {% endblock %}
 
+
 {% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
 
+
 {% block breadcrumbs %}
-<div class="breadcrumbs">
-    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-    &rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
-    &rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
-    &rsaquo; {{title}}
-</div>
+	<div class="breadcrumbs">
+		<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+		&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+		&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+		&rsaquo; {{ title }}
+	</div>
 {% endblock %}
 
 {% block content %}
-<p>Please select the {{itemname}}.</p>
-<form method="post">{% csrf_token %}
-    <div>
-        {% for obj in queryset %}
-        <input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
-        {% endfor %}
-        <p>
-            <select name="obj_id">
-                {% for obj in objects %}
-                <option value="{{obj.id}}">{{obj.name}}</option>
-                {% endfor %}
-            </select>
-        </p>
+	<p>Please select the {{itemname}}.</p>
+	<form method="post">{% csrf_token %}
+		<div>
+			{% for obj in queryset %}
+			<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
+			{% endfor %}
+			<p>
+				<select name="obj_id">
+					{% for obj in objects %}
+					<option value="{{ obj.id }}">{{ obj.name }}</option>
+					{% endfor %}
+				</select>
+			</p>
 
-        <input type="hidden" name="action" value="{{action}}"/>
-        <input type="hidden" name="post" value="yes"/>
-        <p>
-            <input type="submit" value="{% trans "Confirm" %}" />
-            <a href="#" class="button cancel-link">{% trans "Go back" %}</a>
-        </p>
-    </div>
-</form>
+			<input type="hidden" name="action" value="{{ action }}"/>
+			<input type="hidden" name="post" value="yes" />
+			<p>
+				<input type="submit" value="{% trans 'Confirm' %}" />
+				<a href="#" class="button cancel-link">{% trans "Go back" %}</a>
+			</p>
+		</div>
+	</form>
 {% endblock %}

src/paperless/settings.py

@@ -149,8 +149,9 @@ if os.getenv("PAPERLESS_DBENGINE"):
         "ENGINE": os.getenv("PAPERLESS_DBENGINE"),
         "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
         "USER": os.getenv("PAPERLESS_DBUSER"),
-        "PASSWORD": os.getenv("PAPERLESS_DBPASS")
     }
+    if os.getenv("PAPERLESS_DBPASS"):
+        DATABASES["default"]["PASSWORD"] = os.getenv("PAPERLESS_DBPASS")
 
 
 # Password validation

src/paperless_tesseract/parsers.py

@@ -172,8 +172,8 @@ class RasterisedDocumentParser(DocumentParser):
                 raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
                 return raw_text
             raise OCRError(
-                "The guessed language is not available in this instance of "
-                "Tesseract."
+                "The guessed language ({}) is not available in this instance "
+                "of Tesseract.".format(guessed_language)
             )
 
     def _ocr(self, imgs, lang):

src/tox.ini

@@ -5,7 +5,7 @@
 
 [tox]
 skipsdist = True
-envlist = py34, py35, py36, pycodestyle, doc
+envlist = py34, py35, py36, py37, pycodestyle, doc
 
 [testenv]
 commands = pytest