paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-10-30 03:56:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
This commit is contained in:
Jonas Winkler
2018-09-25 14:47:12 +02:00
parent a298cbd4ce a511d34d69
commit 94ede7389d
17 changed files with 486 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
src/documents/actions.py Normal file → Executable file
View File

@@ -5,10 +5,13 @@ from django.core.exceptions import PermissionDenied
from django.template.response import TemplateResponse
from documents.classifier import DocumentClassifier
from documents.models import Tag, Correspondent, DocumentType
from documents.models import Correspondent, DocumentType, Tag
def select_action(modeladmin, request, queryset, title, action, modelclass, success_message="", document_action=None, queryset_action=None):
def select_action(
modeladmin, request, queryset, title, action, modelclass,
success_message="", document_action=None, queryset_action=None):
opts = modeladmin.model._meta
app_label = opts.app_label
@@ -28,7 +31,9 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
queryset_action(queryset, selected_object)
modeladmin.message_user(request, success_message % {
"selected_object": selected_object.name, "count": n, "items": model_ngettext(modeladmin.opts, n)
"selected_object": selected_object.name,
"count": n,
"items": model_ngettext(modeladmin.opts, n)
}, messages.SUCCESS)
# Return None to display the change list page again.
@@ -48,10 +53,17 @@ def select_action(modeladmin, request, queryset, title, action, modelclass, succ
request.current_app = modeladmin.admin_site.name
return TemplateResponse(request, "admin/%s/%s/select_object.html" % (app_label, opts.model_name), context)
return TemplateResponse(
request,
"admin/{}/{}/select_object.html".format(app_label, opts.model_name),
context
)
def simple_action(modeladmin, request, queryset, success_message="", document_action=None, queryset_action=None):
def simple_action(
modeladmin, request, queryset, success_message="",
document_action=None, queryset_action=None):
if not modeladmin.has_change_permission(request):
raise PermissionDenied
@@ -73,40 +85,57 @@ def simple_action(modeladmin, request, queryset, success_message="", document_ac
def add_tag_to_selected(modeladmin, request, queryset):
return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
title="Add tag to multiple documents",
action="add_tag_to_selected",
modelclass=Tag,
success_message="Successfully added tag %(selected_object)s to %(count)d %(items)s.",
document_action=lambda doc, tag: doc.tags.add(tag))
add_tag_to_selected.short_description = "Add tag to selected documents"
return select_action(
modeladmin=modeladmin,
request=request,
queryset=queryset,
title="Add tag to multiple documents",
action="add_tag_to_selected",
modelclass=Tag,
success_message="Successfully added tag %(selected_object)s to "
"%(count)d %(items)s.",
document_action=lambda doc, tag: doc.tags.add(tag)
)
def remove_tag_from_selected(modeladmin, request, queryset):
return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
title="Remove tag from multiple documents",
action="remove_tag_from_selected",
modelclass=Tag,
success_message="Successfully removed tag %(selected_object)s from %(count)d %(items)s.",
document_action=lambda doc, tag: doc.tags.remove(tag))
remove_tag_from_selected.short_description = "Remove tag from selected documents"
return select_action(
modeladmin=modeladmin,
request=request,
queryset=queryset,
title="Remove tag from multiple documents",
action="remove_tag_from_selected",
modelclass=Tag,
success_message="Successfully removed tag %(selected_object)s from "
"%(count)d %(items)s.",
document_action=lambda doc, tag: doc.tags.remove(tag)
)
def set_correspondent_on_selected(modeladmin, request, queryset):
return select_action(modeladmin=modeladmin, request=request, queryset=queryset,
title="Set correspondent on multiple documents",
action="set_correspondent_on_selected",
modelclass=Correspondent,
success_message="Successfully set correspondent %(selected_object)s on %(count)d %(items)s.",
queryset_action=lambda qs, correspondent: qs.update(correspondent=correspondent))
set_correspondent_on_selected.short_description = "Set correspondent on selected documents"
return select_action(
modeladmin=modeladmin,
request=request,
queryset=queryset,
title="Set correspondent on multiple documents",
action="set_correspondent_on_selected",
modelclass=Correspondent,
success_message="Successfully set correspondent %(selected_object)s "
"on %(count)d %(items)s.",
queryset_action=lambda qs, corr: qs.update(correspondent=corr)
)
def remove_correspondent_from_selected(modeladmin, request, queryset):
return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
success_message="Successfully removed correspondent from %(count)d %(items)s.",
queryset_action=lambda qs: qs.update(correspondent=None))
remove_correspondent_from_selected.short_description = "Remove correspondent from selected documents"
return simple_action(
modeladmin=modeladmin,
request=request,
queryset=queryset,
success_message="Successfully removed correspondent from %(count)d "
"%(items)s.",
queryset_action=lambda qs: qs.update(correspondent=None)
)
def set_document_type_on_selected(modeladmin, request, queryset):
@@ -116,14 +145,12 @@ def set_document_type_on_selected(modeladmin, request, queryset):
modelclass=DocumentType,
success_message="Successfully set document type %(selected_object)s on %(count)d %(items)s.",
queryset_action=lambda qs, document_type: qs.update(document_type=document_type))
set_document_type_on_selected.short_description = "Set document type on selected documents"
def remove_document_type_from_selected(modeladmin, request, queryset):
return simple_action(modeladmin=modeladmin, request=request, queryset=queryset,
success_message="Successfully removed document type from %(count)d %(items)s.",
queryset_action=lambda qs: qs.update(document_type=None))
remove_document_type_from_selected.short_description = "Remove document type from selected documents"
def run_document_classifier_on_selected(modeladmin, request, queryset):
@@ -135,4 +162,16 @@ def run_document_classifier_on_selected(modeladmin, request, queryset):
except FileNotFoundError:
modeladmin.message_user(request, "Classifier model file not found.", messages.ERROR)
return None
add_tag_to_selected.short_description = "Add tag to selected documents"
remove_tag_from_selected.short_description = \
"Remove tag from selected documents"
set_correspondent_on_selected.short_description = \
"Set correspondent on selected documents"
remove_correspondent_from_selected.short_description = \
"Remove correspondent from selected documents"
set_document_type_on_selected.short_description = "Set document type on selected documents"
remove_document_type_from_selected.short_description = "Remove document type from selected documents"
run_document_classifier_on_selected.short_description = "Run document classifier on selected"

142
src/documents/admin.py Normal file → Executable file
View File

@@ -3,22 +3,26 @@ from datetime import datetime, timedelta
from django.conf import settings
from django.contrib import admin, messages
from django.contrib.admin.templatetags.admin_urls import add_preserved_filters
from django.contrib.auth.models import User, Group
from django.contrib.auth.models import Group, User
from django.db import models
from django.http import HttpResponseRedirect
try:
from django.core.urlresolvers import reverse
except ImportError:
from django.urls import reverse
from django.templatetags.static import static
from django.urls import reverse
from django.utils.html import format_html, format_html_join
from django.utils.http import urlquote
from django.utils.safestring import mark_safe
from django.db import models
from documents.actions import add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, \
remove_correspondent_from_selected, set_document_type_on_selected, remove_document_type_from_selected, \
from documents.actions import (
add_tag_to_selected,
remove_correspondent_from_selected,
remove_tag_from_selected,
set_correspondent_on_selected,
set_document_type_on_selected,
remove_document_type_from_selected,
run_document_classifier_on_selected
from .models import Correspondent, Tag, Document, Log, DocumentType
)
from .models import Correspondent, Document, DocumentType, Log, Tag
class FinancialYearFilter(admin.SimpleListFilter):
@@ -93,11 +97,18 @@ class RecentCorrespondentFilter(admin.RelatedFieldListFilter):
self.title = "correspondent (recent)"
def field_choices(self, field, request, model_admin):
years = settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS
days = 365 * years
lookups = []
if settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS and settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS > 0:
date_limit = datetime.now() - timedelta(days=365*settings.PAPERLESS_RECENT_CORRESPONDENT_YEARS)
for c in Correspondent.objects.filter(documents__created__gte=date_limit).distinct():
if years and years > 0:
correspondents = Correspondent.objects.filter(
documents__created__gte=datetime.now() - timedelta(days=days)
).distinct()
for c in correspondents:
lookups.append((c.id, c.name))
return lookups
@@ -107,12 +118,20 @@ class CommonAdmin(admin.ModelAdmin):
class CorrespondentAdmin(CommonAdmin):
list_display = ("name", "automatic_classification", "document_count", "last_correspondence")
list_editable = ("automatic_classification",)
list_display = (
"name",
"automatic_classification",
"document_count",
"last_correspondence"
)
list_editable = ("automatic_classification")
def get_queryset(self, request):
qs = super(CorrespondentAdmin, self).get_queryset(request)
qs = qs.annotate(document_count=models.Count("documents"), last_correspondence=models.Max("documents__created"))
qs = qs.annotate(
document_count=models.Count("documents"),
last_correspondence=models.Max("documents__created")
)
return qs
def document_count(self, obj):
@@ -160,24 +179,39 @@ class DocumentAdmin(CommonAdmin):
class Media:
css = {
"all": ("paperless.css",)
}
search_fields = ("correspondent__name", "title", "content", "tags__name")
readonly_fields = ("added",)
list_display = ("title", "created", "added", "thumbnail", "correspondent",
"tags_", "archive_serial_number", "document_type")
list_filter = ("document_type", "tags", ('correspondent', RecentCorrespondentFilter), "correspondent", FinancialYearFilter)
list_filter = (
"document_type",
"tags",
("correspondent", RecentCorrespondentFilter),
"correspondent",
FinancialYearFilter
)
filter_horizontal = ("tags",)
ordering = ["-created", "correspondent"]
actions = [add_tag_to_selected, remove_tag_from_selected, set_correspondent_on_selected, remove_correspondent_from_selected, set_document_type_on_selected, remove_document_type_from_selected, run_document_classifier_on_selected]
actions = [
add_tag_to_selected,
remove_tag_from_selected,
set_correspondent_on_selected,
remove_correspondent_from_selected,
set_document_type_on_selected,
remove_document_type_from_selected,
run_document_classifier_on_selected
]
date_hierarchy = 'created'
date_hierarchy = "created"
document_queue = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.document_queue = []
def has_add_permission(self, request):
return False
@@ -187,27 +221,41 @@ class DocumentAdmin(CommonAdmin):
created_.short_description = "Created"
def changelist_view(self, request, extra_context=None):
response = super().changelist_view(request, extra_context)
if request.method == 'GET':
response = super().changelist_view(
request,
extra_context=extra_context
)
if request.method == "GET":
cl = self.get_changelist_instance(request)
self.document_queue = [doc.id for doc in cl.queryset]
return response
def change_view(self, request, object_id=None, form_url='', extra_context=None):
def change_view(self, request, object_id=None, form_url='',
extra_context=None):
extra_context = extra_context or {}
doc = Document.objects.get(id=object_id)
extra_context['download_url'] = doc.download_url
extra_context['file_type'] = doc.file_type
if self.document_queue and object_id and int(object_id) in self.document_queue:
# There is a queue of documents
current_index = self.document_queue.index(int(object_id))
if current_index < len(self.document_queue) - 1:
# ... and there are still documents in the queue
extra_context['next_object'] = self.document_queue[current_index + 1]
if self.document_queue and object_id:
if int(object_id) in self.document_queue:
# There is a queue of documents
current_index = self.document_queue.index(int(object_id))
if current_index < len(self.document_queue) - 1:
# ... and there are still documents in the queue
extra_context["next_object"] = self.document_queue[
current_index + 1
]
return super(DocumentAdmin, self).change_view(
request, object_id, form_url, extra_context=extra_context,
request,
object_id,
form_url,
extra_context=extra_context,
)
def response_change(self, request, obj):
@@ -217,25 +265,35 @@ class DocumentAdmin(CommonAdmin):
preserved_filters = self.get_preserved_filters(request)
msg_dict = {
'name': opts.verbose_name,
'obj': format_html('<a href="{}">{}</a>', urlquote(request.path), obj),
"name": opts.verbose_name,
"obj": format_html(
'<a href="{}">{}</a>',
urlquote(request.path),
obj
),
}
if "_saveandeditnext" in request.POST:
msg = format_html(
'The {name} "{obj}" was changed successfully. Editing next object.',
'The {name} "{obj}" was changed successfully. '
'Editing next object.',
**msg_dict
)
self.message_user(request, msg, messages.SUCCESS)
redirect_url = reverse('admin:%s_%s_change' %
(opts.app_label, opts.model_name),
args=(request.POST['_next_object'],),
current_app=self.admin_site.name)
redirect_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, redirect_url)
response = HttpResponseRedirect(redirect_url)
else:
response = super().response_change(request, obj)
redirect_url = reverse(
"admin:{}_{}_change".format(opts.app_label, opts.model_name),
args=(request.POST["_next_object"],),
current_app=self.admin_site.name
)
redirect_url = add_preserved_filters(
{
"preserved_filters": preserved_filters,
"opts": opts
},
redirect_url
)
return HttpResponseRedirect(redirect_url)
return response
return super().response_change(request, obj)
@mark_safe
def thumbnail(self, obj):

57
src/documents/filters.py Normal file → Executable file
View File

@@ -1,8 +1,14 @@
from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter
from django_filters.rest_framework import CharFilter, FilterSet, BooleanFilter, ModelChoiceFilter
from .models import Correspondent, Document, Tag, DocumentType
CHAR_KWARGS = (
"startswith", "endswith", "contains",
"istartswith", "iendswith", "icontains"
)
class CorrespondentFilterSet(FilterSet):
class Meta:
@@ -44,38 +50,27 @@ class DocumentTypeFilterSet(FilterSet):
class DocumentFilterSet(FilterSet):
CHAR_KWARGS = {
"lookup_expr": (
"startswith",
"endswith",
"contains",
"istartswith",
"iendswith",
"icontains"
)
}
correspondent__name = CharFilter(
field_name="correspondent__name", **CHAR_KWARGS)
correspondent__slug = CharFilter(
field_name="correspondent__slug", **CHAR_KWARGS)
tags__name = CharFilter(
field_name="tags__name", **CHAR_KWARGS)
tags__slug = CharFilter(
field_name="tags__slug", **CHAR_KWARGS)
tags__empty = BooleanFilter(
field_name="tags", lookup_expr="isnull", distinct=True)
document_type__name = CharFilter(
name="document_type__name", **CHAR_KWARGS)
document_type__slug = CharFilter(
name="document_type__slug", **CHAR_KWARGS)
tags_empty = BooleanFilter(
label="Is tagged",
field_name="tags",
lookup_expr="isnull",
exclude=True
)
class Meta:
model = Document
fields = {
"title": [
"startswith", "endswith", "contains",
"istartswith", "iendswith", "icontains"
],
"content": ["contains", "icontains"],
"title": CHAR_KWARGS,
"content": ("contains", "icontains"),
"correspondent__name": CHAR_KWARGS,
"correspondent__slug": CHAR_KWARGS,
"tags__name": CHAR_KWARGS,
"tags__slug": CHAR_KWARGS,
"document_type__name": CHAR_KWARGS,
"document_type__slug": CHAR_KWARGS,
}

7
src/documents/management/commands/document_exporter.py
View File

@@ -55,7 +55,12 @@ class Command(Renderable, BaseCommand):
documents = Document.objects.all()
document_map = {d.pk: d for d in documents}
manifest = json.loads(serializers.serialize("json", documents))
for document_dict in manifest:
for index, document_dict in enumerate(manifest):
# Force output to unencrypted as that will be the current state.
# The importer will make the decision to encrypt or not.
manifest[index]["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED # NOQA: E501
document = document_map[document_dict["pk"]]

14
src/documents/management/commands/document_importer.py
View File

@@ -94,7 +94,7 @@ class Command(Renderable, BaseCommand):
document_path = os.path.join(self.source, doc_file)
thumbnail_path = os.path.join(self.source, thumb_file)
if document.storage_type == Document.STORAGE_TYPE_GPG:
if settings.PASSPHRASE:
with open(document_path, "rb") as unencrypted:
with open(document.source_path, "wb") as encrypted:
@@ -112,3 +112,15 @@ class Command(Renderable, BaseCommand):
shutil.copy(document_path, document.source_path)
shutil.copy(thumbnail_path, document.thumbnail_path)
# Reset the storage type to whatever we've used while importing
storage_type = Document.STORAGE_TYPE_UNENCRYPTED
if settings.PASSPHRASE:
storage_type = Document.STORAGE_TYPE_GPG
Document.objects.filter(
pk__in=[r["pk"] for r in self.manifest]
).update(
storage_type=storage_type
)

5
src/documents/migrations/0014_document_checksum.py
View File

@@ -158,9 +158,4 @@ class Migration(migrations.Migration):
name='modified',
field=models.DateTimeField(auto_now=True, db_index=True),
),
migrations.AlterField(
model_name='document',
name='checksum',
field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted). We use this to prevent duplicate document imports.', max_length=32, unique=True),
),
]

5
src/documents/migrations/0015_add_insensitive_to_match.py
View File

@@ -12,6 +12,11 @@ class Migration(migrations.Migration):
]
operations = [
migrations.AlterField(
model_name='document',
name='checksum',
field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted). We use this to prevent duplicate document imports.', max_length=32, unique=True),
),
migrations.AddField(
model_name='correspondent',
name='is_insensitive',

66
src/documents/templates/admin/documents/document/select_object.html Normal file → Executable file
View File

@@ -1,46 +1,50 @@
{% extends "admin/base_site.html" %}
{% load i18n l10n admin_urls static %}
{% load staticfiles %}
{% block extrahead %}
{{ block.super }}
{{ media }}
<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
{% block extrahead %}
{{ block.super }}
{{ media }}
<script type="text/javascript" src="{% static 'admin/js/cancel.js' %}"></script>
{% endblock %}
{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
{% block breadcrumbs %}
<div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
&rsaquo; {{title}}
</div>
<div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
&rsaquo; {{ title }}
</div>
{% endblock %}
{% block content %}
<p>Please select the {{itemname}}.</p>
<form method="post">{% csrf_token %}
<div>
{% for obj in queryset %}
<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
{% endfor %}
<p>
<select name="obj_id">
{% for obj in objects %}
<option value="{{obj.id}}">{{obj.name}}</option>
{% endfor %}
</select>
</p>
<p>Please select the {{itemname}}.</p>
<form method="post">{% csrf_token %}
<div>
{% for obj in queryset %}
<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}"/>
{% endfor %}
<p>
<select name="obj_id">
{% for obj in objects %}
<option value="{{ obj.id }}">{{ obj.name }}</option>
{% endfor %}
</select>
</p>
<input type="hidden" name="action" value="{{action}}"/>
<input type="hidden" name="post" value="yes"/>
<p>
<input type="submit" value="{% trans "Confirm" %}" />
<a href="#" class="button cancel-link">{% trans "Go back" %}</a>
</p>
</div>
</form>
<input type="hidden" name="action" value="{{ action }}"/>
<input type="hidden" name="post" value="yes" />
<p>
<input type="submit" value="{% trans 'Confirm' %}" />
<a href="#" class="button cancel-link">{% trans "Go back" %}</a>
</p>
</div>
</form>
{% endblock %}

3
src/paperless/settings.py Normal file → Executable file
View File

@@ -149,8 +149,9 @@ if os.getenv("PAPERLESS_DBENGINE"):
"ENGINE": os.getenv("PAPERLESS_DBENGINE"),
"NAME": os.getenv("PAPERLESS_DBNAME", "paperless"),
"USER": os.getenv("PAPERLESS_DBUSER"),
"PASSWORD": os.getenv("PAPERLESS_DBPASS")
}
if os.getenv("PAPERLESS_DBPASS"):
DATABASES["default"]["PASSWORD"] = os.getenv("PAPERLESS_DBPASS")
# Password validation

4
src/paperless_tesseract/parsers.py
View File

@@ -172,8 +172,8 @@ class RasterisedDocumentParser(DocumentParser):
raw_text = self._assemble_ocr_sections(imgs, middle, raw_text)
return raw_text
raise OCRError(
"The guessed language is not available in this instance of "
"Tesseract."
"The guessed language ({}) is not available in this instance "
"of Tesseract.".format(guessed_language)
)
def _ocr(self, imgs, lang):

2
src/tox.ini
View File

@@ -5,7 +5,7 @@
[tox]
skipsdist = True
envlist = py34, py35, py36, pycodestyle, doc
envlist = py34, py35, py36, py37, pycodestyle, doc
[testenv]
commands = pytest