diff --git a/.gitignore b/.gitignore index a16f958a3..439d9df4b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,5 @@ docker-compose.env scripts/import-for-development scripts/nuke +# Static files collected by the collectstatic command +static/ diff --git a/paperless.conf.example b/paperless.conf.example index 15498a26a..05cf81724 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -59,6 +59,11 @@ PAPERLESS_EMAIL_SECRET="" #### Security #### ############################################################################### +# Controls whether django's debug mode is enabled. Disable this on production +# systems. Debug mode is enabled by default. +PAPERLESS_DEBUG="false" + + # Paperless can be instructed to attempt to encrypt your PDF files with GPG # using the PAPERLESS_PASSPHRASE specified below. If however you're not # concerned about encrypting these files (for example if you have disk @@ -203,3 +208,8 @@ PAPERLESS_EMAIL_SECRET="" # positive integer, but if you don't define one in paperless.conf, a default of # 100 will be used. #PAPERLESS_LIST_PER_PAGE=100 + + +# The number of years for which a correspondent will be included in the recent +# correspondents filter. +#PAPERLESS_RECENT_CORRESPONDENT_YEARS=1 diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index 0476efef1..b8c26a579 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -i https://pypi.python.org/simple -apipkg==1.5; python_version != '3.1.*' -atomicwrites==1.2.1; python_version != '3.1.*' +apipkg==1.5; python_version != '3.3.*' +atomicwrites==1.2.1; python_version != '3.3.*' attrs==18.2.0 certifi==2018.8.24 chardet==3.0.4 -coverage==4.5.1; python_version != '3.1.*' +coverage==4.5.1; python_version < '4' coveralls==1.5.0 dateparser==0.7.0 django-cors-headers==2.4.0 @@ -14,9 +14,9 @@ django-filter==2.0.0 django==2.0.8 djangorestframework==3.8.2 docopt==0.6.2 -execnet==1.5.0; python_version != '3.1.*' +execnet==1.5.0; python_version != '3.3.*' factory-boy==2.11.1 -faker==0.9.0 +faker==0.9.0; python_version >= '2.7' filemagic==1.6 fuzzywuzzy==0.15.0 gunicorn==19.9.0 @@ -26,17 +26,17 @@ langdetect==1.0.7 more-itertools==4.3.0 pdftotext==2.1.0 pillow==5.2.0 -pluggy==0.7.1; python_version != '3.1.*' -py==1.6.0; python_version != '3.1.*' +pluggy==0.7.1; python_version != '3.3.*' +py==1.6.0; python_version != '3.3.*' pycodestyle==2.4.0 pyocr==0.5.3 -pytest-cov==2.5.1 +pytest-cov==2.6.0 pytest-django==3.4.2 pytest-env==0.6.2 -pytest-forked==0.2 +pytest-forked==0.2; python_version != '3.3.*' pytest-sugar==0.9.1 pytest-xdist==1.23.0 -pytest==3.7.4 +pytest==3.8.0 python-dateutil==2.7.3 python-dotenv==0.9.1 python-gnupg==0.4.3 @@ -48,4 +48,4 @@ six==1.11.0 termcolor==1.1.0 text-unidecode==1.2 tzlocal==1.5.1 -urllib3==1.23; python_version != '3.0.*' +urllib3==1.23; python_version != '3.3.*' diff --git a/src/documents/actions.py b/src/documents/actions.py new file mode 100644 index 000000000..cd2698a2c --- /dev/null +++ b/src/documents/actions.py @@ -0,0 +1,146 @@ +from django.contrib import messages +from django.contrib.admin import helpers +from django.contrib.admin.utils import model_ngettext +from django.core.exceptions import PermissionDenied +from django.template.response import TemplateResponse + +from documents.models import Correspondent, Tag + + +def select_action( + modeladmin, request, queryset, title, action, modelclass, + success_message="", document_action=None, queryset_action=None): + + opts = modeladmin.model._meta + app_label = opts.app_label + + if not modeladmin.has_change_permission(request): + raise PermissionDenied + + if request.POST.get('post'): + n = queryset.count() + selected_object = modelclass.objects.get(id=request.POST.get('obj_id')) + if n: + for document in queryset: + if document_action: + document_action(document, selected_object) + document_display = str(document) + modeladmin.log_change(request, document, document_display) + if queryset_action: + queryset_action(queryset, selected_object) + + modeladmin.message_user(request, success_message % { + "selected_object": selected_object.name, + "count": n, + "items": model_ngettext(modeladmin.opts, n) + }, messages.SUCCESS) + + # Return None to display the change list page again. + return None + + context = dict( + modeladmin.admin_site.each_context(request), + title=title, + queryset=queryset, + opts=opts, + action_checkbox_name=helpers.ACTION_CHECKBOX_NAME, + media=modeladmin.media, + action=action, + objects=modelclass.objects.all(), + itemname=model_ngettext(modelclass, 1) + ) + + request.current_app = modeladmin.admin_site.name + + return TemplateResponse( + request, + "admin/{}/{}/select_object.html".format(app_label, opts.model_name), + context + ) + + +def simple_action( + modeladmin, request, queryset, success_message="", + document_action=None, queryset_action=None): + + if not modeladmin.has_change_permission(request): + raise PermissionDenied + + n = queryset.count() + if n: + for document in queryset: + if document_action: + document_action(document) + document_display = str(document) + modeladmin.log_change(request, document, document_display) + if queryset_action: + queryset_action(queryset) + modeladmin.message_user(request, success_message % { + "count": n, "items": model_ngettext(modeladmin.opts, n) + }, messages.SUCCESS) + + # Return None to display the change list page again. + return None + + +def add_tag_to_selected(modeladmin, request, queryset): + return select_action( + modeladmin=modeladmin, + request=request, + queryset=queryset, + title="Add tag to multiple documents", + action="add_tag_to_selected", + modelclass=Tag, + success_message="Successfully added tag %(selected_object)s to " + "%(count)d %(items)s.", + document_action=lambda doc, tag: doc.tags.add(tag) + ) + + +def remove_tag_from_selected(modeladmin, request, queryset): + return select_action( + modeladmin=modeladmin, + request=request, + queryset=queryset, + title="Remove tag from multiple documents", + action="remove_tag_from_selected", + modelclass=Tag, + success_message="Successfully removed tag %(selected_object)s from " + "%(count)d %(items)s.", + document_action=lambda doc, tag: doc.tags.remove(tag) + ) + + +def set_correspondent_on_selected(modeladmin, request, queryset): + + return select_action( + modeladmin=modeladmin, + request=request, + queryset=queryset, + title="Set correspondent on multiple documents", + action="set_correspondent_on_selected", + modelclass=Correspondent, + success_message="Successfully set correspondent %(selected_object)s " + "on %(count)d %(items)s.", + queryset_action=lambda qs, corr: qs.update(correspondent=corr) + ) + + +def remove_correspondent_from_selected(modeladmin, request, queryset): + return simple_action( + modeladmin=modeladmin, + request=request, + queryset=queryset, + success_message="Successfully removed correspondent from %(count)d " + "%(items)s.", + queryset_action=lambda qs: qs.update(correspondent=None) + ) + + +add_tag_to_selected.short_description = "Add tag to selected documents" +remove_tag_from_selected.short_description = \ + "Remove tag from selected documents" +set_correspondent_on_selected.short_description = \ + "Set correspondent on selected documents" +remove_correspondent_from_selected.short_description = \ + "Remove correspondent from selected documents" diff --git a/src/documents/admin.py b/src/documents/admin.py index 36154b6ba..365a99c1a 100644 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -1,42 +1,25 @@ -from datetime import datetime +from datetime import datetime, timedelta from django.conf import settings -from django.contrib import admin -from django.contrib.auth.models import User, Group -try: - from django.core.urlresolvers import reverse -except ImportError: - from django.urls import reverse +from django.contrib import admin, messages +from django.contrib.admin.templatetags.admin_urls import add_preserved_filters +from django.contrib.auth.models import Group, User +from django.db import models +from django.http import HttpResponseRedirect from django.templatetags.static import static -from django.utils.safestring import mark_safe +from django.urls import reverse from django.utils.html import format_html, format_html_join +from django.utils.http import urlquote +from django.utils.safestring import mark_safe -from .models import Correspondent, Tag, Document, Log +from documents.actions import ( + add_tag_to_selected, + remove_correspondent_from_selected, + remove_tag_from_selected, + set_correspondent_on_selected +) - -class MonthListFilter(admin.SimpleListFilter): - - title = "Month" - - # Parameter for the filter that will be used in the URL query. - parameter_name = "month" - - def lookups(self, request, model_admin): - r = [] - for document in Document.objects.all(): - r.append(( - document.created.strftime("%Y-%m"), - document.created.strftime("%B %Y") - )) - return sorted(set(r), key=lambda x: x[0], reverse=True) - - def queryset(self, request, queryset): - - if not self.value(): - return None - - year, month = self.value().split("-") - return queryset.filter(created__year=year, created__month=month) +from .models import Correspondent, Document, Log, Tag class FinancialYearFilter(admin.SimpleListFilter): @@ -104,18 +87,59 @@ class FinancialYearFilter(admin.SimpleListFilter): created__lte=self._fy_end(end)) +class RecentCorrespondentFilter(admin.RelatedFieldListFilter): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.title = "correspondent (recent)" + + def field_choices(self, field, request, model_admin): + + years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS + days = 365 * years + + lookups = [] + if years and years > 0: + correspondents = Correspondent.objects.filter( + documents__created__gte=datetime.now() - timedelta(days=days) + ).distinct() + for c in correspondents: + lookups.append((c.id, c.name)) + + return lookups + + class CommonAdmin(admin.ModelAdmin): list_per_page = settings.PAPERLESS_LIST_PER_PAGE class CorrespondentAdmin(CommonAdmin): - list_display = ("name", "match", "matching_algorithm", "document_count") + list_display = ( + "name", + "match", + "matching_algorithm", + "document_count", + "last_correspondence" + ) list_filter = ("matching_algorithm",) list_editable = ("match", "matching_algorithm") + def get_queryset(self, request): + qs = super(CorrespondentAdmin, self).get_queryset(request) + qs = qs.annotate( + document_count=models.Count("documents"), + last_correspondence=models.Max("documents__created") + ) + return qs + def document_count(self, obj): - return obj.documents.count() + return obj.document_count + document_count.admin_order_field = "document_count" + + def last_correspondence(self, obj): + return obj.last_correspondence + last_correspondence.admin_order_field = "last_correspondence" class TagAdmin(CommonAdmin): @@ -125,8 +149,14 @@ class TagAdmin(CommonAdmin): list_filter = ("colour", "matching_algorithm") list_editable = ("colour", "match", "matching_algorithm") + def get_queryset(self, request): + qs = super(TagAdmin, self).get_queryset(request) + qs = qs.annotate(document_count=models.Count("documents")) + return qs + def document_count(self, obj): - return obj.documents.count() + return obj.document_count + document_count.admin_order_field = "document_count" class DocumentAdmin(CommonAdmin): @@ -140,12 +170,30 @@ class DocumentAdmin(CommonAdmin): readonly_fields = ("added",) list_display = ("title", "created", "added", "thumbnail", "correspondent", "tags_") - list_filter = ("tags", "correspondent", FinancialYearFilter, - MonthListFilter) + list_filter = ( + "tags", + ("correspondent", RecentCorrespondentFilter), + "correspondent", + FinancialYearFilter + ) + filter_horizontal = ("tags",) ordering = ["-created", "correspondent"] + actions = [ + add_tag_to_selected, + remove_tag_from_selected, + set_correspondent_on_selected, + remove_correspondent_from_selected + ] + + date_hierarchy = "created" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.document_queue = [] + def has_add_permission(self, request): return False @@ -153,6 +201,79 @@ class DocumentAdmin(CommonAdmin): return obj.created.date().strftime("%Y-%m-%d") created_.short_description = "Created" + def changelist_view(self, request, extra_context=None): + + response = super().changelist_view( + request, + extra_context=extra_context + ) + + if request.method == "GET": + cl = self.get_changelist_instance(request) + self.document_queue = [doc.id for doc in cl.queryset] + + return response + + def change_view(self, request, object_id=None, form_url='', + extra_context=None): + + extra_context = extra_context or {} + + if self.document_queue and object_id: + if int(object_id) in self.document_queue: + # There is a queue of documents + current_index = self.document_queue.index(int(object_id)) + if current_index < len(self.document_queue) - 1: + # ... and there are still documents in the queue + extra_context["next_object"] = self.document_queue[ + current_index + 1 + ] + + return super(DocumentAdmin, self).change_view( + request, + object_id, + form_url, + extra_context=extra_context, + ) + + def response_change(self, request, obj): + + # This is mostly copied from ModelAdmin.response_change() + opts = self.model._meta + preserved_filters = self.get_preserved_filters(request) + + msg_dict = { + "name": opts.verbose_name, + "obj": format_html( + '{}', + urlquote(request.path), + obj + ), + } + if "_saveandeditnext" in request.POST: + msg = format_html( + 'The {name} "{obj}" was changed successfully. ' + 'Editing next object.', + **msg_dict + ) + self.message_user(request, msg, messages.SUCCESS) + redirect_url = reverse( + "admin:{}_{}_change".format(opts.app_label, opts.model_name), + args=(request.POST["_next_object"],), + current_app=self.admin_site.name + ) + redirect_url = add_preserved_filters( + { + "preserved_filters": preserved_filters, + "opts": opts + }, + redirect_url + ) + return HttpResponseRedirect(redirect_url) + + return super().response_change(request, obj) + + @mark_safe def thumbnail(self, obj): return self._html_tag( "a", @@ -165,8 +286,8 @@ class DocumentAdmin(CommonAdmin): ), href=obj.download_url ) - thumbnail.allow_tags = True + @mark_safe def tags_(self, obj): r = "" for tag in obj.tags.all(): @@ -183,10 +304,11 @@ class DocumentAdmin(CommonAdmin): ) } ) - return mark_safe(r) - tags_.allow_tags = True + return r + @mark_safe def document(self, obj): + # TODO: is this method even used anymore? return self._html_tag( "a", self._html_tag( @@ -199,7 +321,6 @@ class DocumentAdmin(CommonAdmin): ), href=obj.download_url ) - document.allow_tags = True @staticmethod def _html_tag(kind, inside=None, **kwargs): diff --git a/src/documents/templates/admin/documents/document/change_form.html b/src/documents/templates/admin/documents/document/change_form.html index 7bd0e483f..88fae955d 100644 --- a/src/documents/templates/admin/documents/document/change_form.html +++ b/src/documents/templates/admin/documents/document/change_form.html @@ -1,5 +1,21 @@ {% extends 'admin/change_form.html' %} +{% block content %} + +{{ block.super }} + +{% if next_object %} + +{% endif %} + +{% endblock content %} {% block footer %} @@ -10,4 +26,4 @@ django.jQuery(".field-created input").first().attr("type", "date") -{% endblock footer %} \ No newline at end of file +{% endblock footer %} diff --git a/src/documents/templates/admin/documents/document/select_object.html b/src/documents/templates/admin/documents/document/select_object.html new file mode 100644 index 000000000..775d57b12 --- /dev/null +++ b/src/documents/templates/admin/documents/document/select_object.html @@ -0,0 +1,50 @@ +{% extends "admin/base_site.html" %} + + +{% load i18n l10n admin_urls static %} +{% load staticfiles %} + + +{% block extrahead %} + {{ block.super }} + {{ media }} + +{% endblock %} + + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + + +{% block breadcrumbs %} + +{% endblock %} + +{% block content %} +

Please select the {{itemname}}.

+
{% csrf_token %} +
+ {% for obj in queryset %} + + {% endfor %} +

+ +

+ + + +

+ + {% trans "Go back" %} +

+
+
+{% endblock %} diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 956b90a7f..433eabe88 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -22,12 +22,12 @@ elif os.path.exists("/usr/local/etc/paperless.conf"): load_dotenv("/usr/local/etc/paperless.conf") -def __get_boolean(key): +def __get_boolean(key, default="NO"): """ Return a boolean value based on whatever the user has supplied in the environment based on whether the value "looks like" it's True or not. """ - return bool(os.getenv(key, "NO").lower() in ("yes", "y", "1", "t", "true")) + return bool(os.getenv(key, default).lower() in ("yes", "y", "1", "t", "true")) # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -47,7 +47,7 @@ SECRET_KEY = os.getenv( # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = __get_boolean("PAPERLESS_DEBUG", "YES") LOGIN_URL = "admin:login" @@ -292,3 +292,9 @@ FY_END = os.getenv("PAPERLESS_FINANCIAL_YEAR_END") # Specify the default date order (for autodetected dates) DATE_ORDER = os.getenv("PAPERLESS_DATE_ORDER", "DMY") + +# Specify for how many years a correspondent is considered recent. Recent +# correspondents will be shown in a separate "Recent correspondents" filter as +# well. Set to 0 to disable this filter. +PAPERLESS_RECENT_CORRESPONDENT_YEARS = int(os.getenv( + "PAPERLESS_RECENT_CORRESPONDENT_YEARS", 0)) diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index e3c2ed361..f54461161 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -172,8 +172,8 @@ class RasterisedDocumentParser(DocumentParser): raw_text = self._assemble_ocr_sections(imgs, middle, raw_text) return raw_text raise OCRError( - "The guessed language is not available in this instance of " - "Tesseract." + "The guessed language ({}) is not available in this instance " + "of Tesseract.".format(guessed_language) ) def _ocr(self, imgs, lang):