paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-10-30 03:56:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

i18n, better error handling

Browse Source
This commit is contained in:
jonaswinkler
2021-01-26 15:02:46 +01:00
parent 2e112fe7b1
commit c1136ac83c
3 changed files with 194 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
src/documents/consumer.py
View File

@@ -25,6 +25,8 @@ from .signals import (
document_consumption_started document_consumption_started
) )
from django.utils.translation import gettext as _
class ConsumerError(Exception): class ConsumerError(Exception):
pass pass
@@ -32,10 +34,10 @@ class ConsumerError(Exception):
class Consumer(LoggingMixin): class Consumer(LoggingMixin):
def _send_progress(self, filename, current_progress, max_progress, status, def _send_progress(self, current_progress, max_progress, status,
message, document_id=None): message, document_id=None):
payload = { payload = {
'filename': os.path.basename(filename), 'filename': os.path.basename(self.filename),
'task_id': self.task_id, 'task_id': self.task_id,
'current_progress': current_progress, 'current_progress': current_progress,
'max_progress': max_progress, 'max_progress': max_progress,
@@ -47,10 +49,10 @@ class Consumer(LoggingMixin):
{'type': 'status_update', {'type': 'status_update',
'data': payload}) 'data': payload})
def _fail(self, message): def _fail(self, message, log_message=None):
self._send_progress(self.filename, 100, 100, 'FAILED', self._send_progress(100, 100, 'FAILED', message)
message) self.log("error", log_message or message)
raise ConsumerError(f"{self.filename}: {message}") raise ConsumerError(f"{self.filename}: {log_message or message}")
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -66,11 +68,10 @@ class Consumer(LoggingMixin):
def pre_check_file_exists(self): def pre_check_file_exists(self):
if not os.path.isfile(self.path): if not os.path.isfile(self.path):
self.log( self._fail(
"error", _("File not found"),
"Cannot consume {}: It is not a file.".format(self.path) f"Cannot consume {self.path}: It is not a file."
) )
self._fail("File not found")
def pre_check_duplicate(self): def pre_check_duplicate(self):
with open(self.path, "rb") as f: with open(self.path, "rb") as f:
@@ -78,11 +79,10 @@ class Consumer(LoggingMixin):
if Document.objects.filter(Q(checksum=checksum) | Q(archive_checksum=checksum)).exists(): # NOQA: E501 if Document.objects.filter(Q(checksum=checksum) | Q(archive_checksum=checksum)).exists(): # NOQA: E501
if settings.CONSUMER_DELETE_DUPLICATES: if settings.CONSUMER_DELETE_DUPLICATES:
os.unlink(self.path) os.unlink(self.path)
self.log( self._fail(
"error", _("Document is a duplicate"),
"Not consuming {}: It is a duplicate.".format(self.filename) f"Not consuming {self.filename}: It is a duplicate."
) )
self._fail("Document is a duplicate")
def pre_check_directories(self): def pre_check_directories(self):
os.makedirs(settings.SCRATCH_DIR, exist_ok=True) os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
@@ -95,14 +95,16 @@ class Consumer(LoggingMixin):
return return
if not os.path.isfile(settings.PRE_CONSUME_SCRIPT): if not os.path.isfile(settings.PRE_CONSUME_SCRIPT):
raise ConsumerError( self._fail(
_("Pre-consume script does not exist."),
f"Configured pre-consume script " f"Configured pre-consume script "
f"{settings.PRE_CONSUME_SCRIPT} does not exist.") f"{settings.PRE_CONSUME_SCRIPT} does not exist.")
try: try:
Popen((settings.PRE_CONSUME_SCRIPT, self.path)).wait() Popen((settings.PRE_CONSUME_SCRIPT, self.path)).wait()
except Exception as e: except Exception as e:
raise ConsumerError( self._fail(
_("Error while executing pre-consume script"),
f"Error while executing pre-consume script: {e}" f"Error while executing pre-consume script: {e}"
) )
@@ -111,9 +113,11 @@ class Consumer(LoggingMixin):
return return
if not os.path.isfile(settings.POST_CONSUME_SCRIPT): if not os.path.isfile(settings.POST_CONSUME_SCRIPT):
raise ConsumerError( self._fail(
_("Post-consume script does not exist."),
f"Configured post-consume script " f"Configured post-consume script "
f"{settings.POST_CONSUME_SCRIPT} does not exist.") f"{settings.POST_CONSUME_SCRIPT} does not exist."
)
try: try:
Popen(( Popen((
@@ -129,8 +133,9 @@ class Consumer(LoggingMixin):
"name", flat=True))) "name", flat=True)))
)).wait() )).wait()
except Exception as e: except Exception as e:
raise ConsumerError( self._fail(
f"Error while executing pre-consume script: {e}" _("Error while executing post-consume script"),
f"Error while executing post-consume script: {e}"
) )
def try_consume_file(self, def try_consume_file(self,
@@ -153,8 +158,7 @@ class Consumer(LoggingMixin):
self.override_tag_ids = override_tag_ids self.override_tag_ids = override_tag_ids
self.task_id = task_id or str(uuid.uuid4()) self.task_id = task_id or str(uuid.uuid4())
self._send_progress(self.filename, 0, 100, 'WORKING', self._send_progress(0, 100, 'WORKING', _('Received new file'))
'Received new file.')
# this is for grouping logging entries for this particular file # this is for grouping logging entries for this particular file
# together. # together.
@@ -177,10 +181,13 @@ class Consumer(LoggingMixin):
parser_class = get_parser_class_for_mime_type(mime_type) parser_class = get_parser_class_for_mime_type(mime_type)
if not parser_class: if not parser_class:
self._fail(f"Unsupported mime type {mime_type}") self._fail(
_("File type %(type)s not supported") %
{'type': mime_type},
f"Unsupported mime type {mime_type}"
)
else: else:
self.log("debug", self.log("debug", f"Parser: {parser_class.__name__}")
f"Parser: {parser_class.__name__}")
# Notify all listeners that we're going to do some work. # Notify all listeners that we're going to do some work.
@@ -195,7 +202,7 @@ class Consumer(LoggingMixin):
def progress_callback(current_progress, max_progress, message): def progress_callback(current_progress, max_progress, message):
# recalculate progress to be within 20 and 80 # recalculate progress to be within 20 and 80
p = int((current_progress / max_progress) * 50 + 20) p = int((current_progress / max_progress) * 50 + 20)
self._send_progress(self.filename, p, 100, "WORKING", message) self._send_progress(p, 100, "WORKING", message)
# This doesn't parse the document yet, but gives us a parser. # This doesn't parse the document yet, but gives us a parser.
@@ -206,32 +213,36 @@ class Consumer(LoggingMixin):
# Parse the document. This may take some time. # Parse the document. This may take some time.
text = None
date = None
thumbnail = None
archive_path = None
try: try:
self._send_progress(self.filename, 20, 100, 'WORKING', self._send_progress(20, 100, 'WORKING', _('Parsing document...'))
'Parsing document...')
self.log("debug", "Parsing {}...".format(self.filename)) self.log("debug", "Parsing {}...".format(self.filename))
document_parser.parse(self.path, mime_type, self.filename) document_parser.parse(self.path, mime_type, self.filename)
self.log("debug", f"Generating thumbnail for {self.filename}...") self.log("debug", f"Generating thumbnail for {self.filename}...")
self._send_progress(self.filename, 70, 100, 'WORKING', self._send_progress(70, 100, 'WORKING',
'Generating thumbnail...') _('Generating thumbnail...'))
thumbnail = document_parser.get_optimised_thumbnail( thumbnail = document_parser.get_optimised_thumbnail(
self.path, mime_type) self.path, mime_type)
text = document_parser.get_text() text = document_parser.get_text()
date = document_parser.get_date() date = document_parser.get_date()
if not date: if not date:
self._send_progress(self.filename, 90, 100, 'WORKING', self._send_progress(90, 100, 'WORKING',
'Getting date from document...') _('Getting date from document...'))
date = parse_date(self.filename, text) date = parse_date(self.filename, text)
archive_path = document_parser.get_archive_path() archive_path = document_parser.get_archive_path()
except ParseError as e: except ParseError as e:
document_parser.cleanup() document_parser.cleanup()
self.log( self._fail(
"error", str(e),
f"Error while consuming document {self.filename}: {e}") f"Error while consuming document {self.filename}: {e}"
self._fail(str(e)) )
# Prepare the document classifier. # Prepare the document classifier.
@@ -247,8 +258,7 @@ class Consumer(LoggingMixin):
"warning", "warning",
f"Cannot classify documents: {e}.") f"Cannot classify documents: {e}.")
classifier = None classifier = None
self._send_progress(self.filename, 95, 100, 'WORKING', self._send_progress(95, 100, 'WORKING', _('Saving document...'))
'Storing the document...')
# now that everything is done, we can start to store the document # now that everything is done, we can start to store the document
# in the system. This will be a transaction and reasonably fast. # in the system. This will be a transaction and reasonably fast.
try: try:
@@ -302,12 +312,11 @@ class Consumer(LoggingMixin):
os.unlink(self.path) os.unlink(self.path)
except Exception as e: except Exception as e:
self.log( self._fail(
"error", str(e),
f"The following error occured while consuming " f"The following error occured while consuming "
f"{self.filename}: {e}" f"{self.filename}: {e}"
) )
self._fail(str(e))
finally: finally:
document_parser.cleanup() document_parser.cleanup()
@@ -318,8 +327,7 @@ class Consumer(LoggingMixin):
"Document {} consumption finished".format(document) "Document {} consumption finished".format(document)
) )
self._send_progress(self.filename, 100, 100, 'SUCCESS', self._send_progress(100, 100, 'SUCCESS', _('Finished.'), document.id)
'Finished.', document.id)
return document return document

6
src/documents/serialisers.py
View File

@@ -8,6 +8,8 @@ from .models import Correspondent, Tag, Document, Log, DocumentType, \
SavedView, SavedViewFilterRule SavedView, SavedViewFilterRule
from .parsers import is_mime_type_supported from .parsers import is_mime_type_supported
from django.utils.translation import gettext as _
# https://www.django-rest-framework.org/api-guide/serializers/#example # https://www.django-rest-framework.org/api-guide/serializers/#example
class DynamicFieldsModelSerializer(serializers.ModelSerializer): class DynamicFieldsModelSerializer(serializers.ModelSerializer):
@@ -378,7 +380,9 @@ class PostDocumentSerializer(serializers.Serializer):
if not is_mime_type_supported(mime_type): if not is_mime_type_supported(mime_type):
raise serializers.ValidationError( raise serializers.ValidationError(
"This file type is not supported.") _("File type %(type)s not supported") %
{'type': mime_type}
)
return document.name, document_data return document.name, document_data

222
src/locale/en-us/LC_MESSAGES/django.po
View File

@@ -8,7 +8,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: PACKAGE VERSION\n" "Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2021-01-10 21:41+0000\n" "POT-Creation-Date: 2021-01-26 14:59+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n" "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -21,320 +21,374 @@ msgstr ""
msgid "Documents" msgid "Documents"
msgstr "" msgstr ""
#: documents/models.py:32 #: documents/consumer.py:72
msgid "Any word" msgid "File not found"
msgstr "Datei nicht gefunden"
#: documents/consumer.py:83
msgid "Document is a duplicate"
msgstr "Dokument existiert bereits"
#: documents/consumer.py:99
msgid "Pre-consume script does not exist."
msgstr ""
#: documents/consumer.py:107
msgid "Error while executing pre-consume script"
msgstr ""
#: documents/consumer.py:117
msgid "Post-consume script does not exist."
msgstr ""
#: documents/consumer.py:137
msgid "Error while executing post-consume script"
msgstr ""
#: documents/consumer.py:161
msgid "Received new file"
msgstr ""
#: documents/consumer.py:185 documents/serialisers.py:383
#, fuzzy, python-format
#| msgid "File type {type} not supported."
msgid "File type %(type)s not supported"
msgstr "Dateityp {type} wird nicht unterstützt"
#: documents/consumer.py:222
msgid "Parsing document..."
msgstr ""
#: documents/consumer.py:228
msgid "Generating thumbnail..."
msgstr ""
#: documents/consumer.py:236
msgid "Getting date from document..."
msgstr ""
#: documents/consumer.py:261
msgid "Saving document..."
msgstr ""
#: documents/consumer.py:330
msgid "Finished."
msgstr "" msgstr ""
#: documents/models.py:33 #: documents/models.py:33
msgid "All words" msgid "Any word"
msgstr "" msgstr ""
#: documents/models.py:34 #: documents/models.py:34
msgid "Exact match" msgid "All words"
msgstr "" msgstr ""
#: documents/models.py:35 #: documents/models.py:35
msgid "Regular expression" msgid "Exact match"
msgstr "" msgstr ""
#: documents/models.py:36 #: documents/models.py:36
msgid "Fuzzy word" msgid "Regular expression"
msgstr "" msgstr ""
#: documents/models.py:37 #: documents/models.py:37
msgid "Fuzzy word"
msgstr ""
#: documents/models.py:38
msgid "Automatic" msgid "Automatic"
msgstr "" msgstr ""
#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25 #: documents/models.py:42 documents/models.py:352 paperless_mail/models.py:25
#: paperless_mail/models.py:109 #: paperless_mail/models.py:109
msgid "name" msgid "name"
msgstr "" msgstr ""
#: documents/models.py:45 #: documents/models.py:46
msgid "match" msgid "match"
msgstr "" msgstr ""
#: documents/models.py:49 #: documents/models.py:50
msgid "matching algorithm" msgid "matching algorithm"
msgstr "" msgstr ""
#: documents/models.py:55 #: documents/models.py:56
msgid "is insensitive" msgid "is insensitive"
msgstr "" msgstr ""
#: documents/models.py:80 documents/models.py:140 #: documents/models.py:75 documents/models.py:135
msgid "correspondent" msgid "correspondent"
msgstr "" msgstr ""
#: documents/models.py:81 #: documents/models.py:76
msgid "correspondents" msgid "correspondents"
msgstr "" msgstr ""
#: documents/models.py:103 #: documents/models.py:98
msgid "color" msgid "color"
msgstr "" msgstr ""
#: documents/models.py:107 #: documents/models.py:102
msgid "is inbox tag" msgid "is inbox tag"
msgstr "" msgstr ""
#: documents/models.py:109 #: documents/models.py:104
msgid "" msgid ""
"Marks this tag as an inbox tag: All newly consumed documents will be tagged " "Marks this tag as an inbox tag: All newly consumed documents will be tagged "
"with inbox tags." "with inbox tags."
msgstr "" msgstr ""
#: documents/models.py:114 #: documents/models.py:109
msgid "tag" msgid "tag"
msgstr "" msgstr ""
#: documents/models.py:115 documents/models.py:171 #: documents/models.py:110 documents/models.py:166
msgid "tags" msgid "tags"
msgstr "" msgstr ""
#: documents/models.py:121 documents/models.py:153 #: documents/models.py:116 documents/models.py:148
msgid "document type" msgid "document type"
msgstr "" msgstr ""
#: documents/models.py:122 #: documents/models.py:117
msgid "document types" msgid "document types"
msgstr "" msgstr ""
#: documents/models.py:130 #: documents/models.py:125
msgid "Unencrypted" msgid "Unencrypted"
msgstr "" msgstr ""
#: documents/models.py:131 #: documents/models.py:126
msgid "Encrypted with GNU Privacy Guard" msgid "Encrypted with GNU Privacy Guard"
msgstr "" msgstr ""
#: documents/models.py:144 #: documents/models.py:139
msgid "title" msgid "title"
msgstr "" msgstr ""
#: documents/models.py:157 #: documents/models.py:152
msgid "content" msgid "content"
msgstr "" msgstr ""
#: documents/models.py:159 #: documents/models.py:154
msgid "" msgid ""
"The raw, text-only data of the document. This field is primarily used for " "The raw, text-only data of the document. This field is primarily used for "
"searching." "searching."
msgstr "" msgstr ""
#: documents/models.py:164 #: documents/models.py:159
msgid "mime type" msgid "mime type"
msgstr "" msgstr ""
#: documents/models.py:175 #: documents/models.py:170
msgid "checksum" msgid "checksum"
msgstr "" msgstr ""
#: documents/models.py:179 #: documents/models.py:174
msgid "The checksum of the original document." msgid "The checksum of the original document."
msgstr "" msgstr ""
#: documents/models.py:183 #: documents/models.py:178
msgid "archive checksum" msgid "archive checksum"
msgstr "" msgstr ""
#: documents/models.py:188 #: documents/models.py:183
msgid "The checksum of the archived document." msgid "The checksum of the archived document."
msgstr "" msgstr ""
#: documents/models.py:192 documents/models.py:332 #: documents/models.py:187 documents/models.py:330
msgid "created" msgid "created"
msgstr "" msgstr ""
#: documents/models.py:196 #: documents/models.py:191
msgid "modified" msgid "modified"
msgstr "" msgstr ""
#: documents/models.py:200 #: documents/models.py:195
msgid "storage type" msgid "storage type"
msgstr "" msgstr ""
#: documents/models.py:208 #: documents/models.py:203
msgid "added" msgid "added"
msgstr "" msgstr ""
#: documents/models.py:212 #: documents/models.py:207
msgid "filename" msgid "filename"
msgstr "" msgstr ""
#: documents/models.py:217 #: documents/models.py:212
msgid "Current filename in storage" msgid "Current filename in storage"
msgstr "" msgstr ""
#: documents/models.py:221 #: documents/models.py:216
msgid "archive serial number" msgid "archive serial number"
msgstr "" msgstr ""
#: documents/models.py:226 #: documents/models.py:221
msgid "The position of this document in your physical document archive." msgid "The position of this document in your physical document archive."
msgstr "" msgstr ""
#: documents/models.py:232 #: documents/models.py:227
msgid "document" msgid "document"
msgstr "" msgstr ""
#: documents/models.py:233 #: documents/models.py:228
msgid "documents" msgid "documents"
msgstr "" msgstr ""
#: documents/models.py:315 #: documents/models.py:313
msgid "debug" msgid "debug"
msgstr "" msgstr ""
#: documents/models.py:316 #: documents/models.py:314
msgid "information" msgid "information"
msgstr "" msgstr ""
#: documents/models.py:317 #: documents/models.py:315
msgid "warning" msgid "warning"
msgstr "" msgstr ""
#: documents/models.py:318 #: documents/models.py:316
msgid "error" msgid "error"
msgstr "" msgstr ""
#: documents/models.py:319 #: documents/models.py:317
msgid "critical" msgid "critical"
msgstr "" msgstr ""
#: documents/models.py:323 #: documents/models.py:321
msgid "group" msgid "group"
msgstr "" msgstr ""
#: documents/models.py:326 #: documents/models.py:324
msgid "message" msgid "message"
msgstr "" msgstr ""
#: documents/models.py:329 #: documents/models.py:327
msgid "level" msgid "level"
msgstr "" msgstr ""
#: documents/models.py:336 #: documents/models.py:334
msgid "log" msgid "log"
msgstr "" msgstr ""
#: documents/models.py:337 #: documents/models.py:335
msgid "logs" msgid "logs"
msgstr "" msgstr ""
#: documents/models.py:348 documents/models.py:398 #: documents/models.py:346 documents/models.py:396
msgid "saved view" msgid "saved view"
msgstr "" msgstr ""
#: documents/models.py:349 #: documents/models.py:347
msgid "saved views" msgid "saved views"
msgstr "" msgstr ""
#: documents/models.py:352 #: documents/models.py:350
msgid "user" msgid "user"
msgstr "" msgstr ""
#: documents/models.py:358 #: documents/models.py:356
msgid "show on dashboard" msgid "show on dashboard"
msgstr "" msgstr ""
#: documents/models.py:361 #: documents/models.py:359
msgid "show in sidebar" msgid "show in sidebar"
msgstr "" msgstr ""
#: documents/models.py:365 #: documents/models.py:363
msgid "sort field" msgid "sort field"
msgstr "" msgstr ""
#: documents/models.py:368 #: documents/models.py:366
msgid "sort reverse" msgid "sort reverse"
msgstr "" msgstr ""
#: documents/models.py:374 #: documents/models.py:372
msgid "title contains" msgid "title contains"
msgstr "" msgstr ""
#: documents/models.py:375 #: documents/models.py:373
msgid "content contains" msgid "content contains"
msgstr "" msgstr ""
#: documents/models.py:376 #: documents/models.py:374
msgid "ASN is" msgid "ASN is"
msgstr "" msgstr ""
#: documents/models.py:377 #: documents/models.py:375
msgid "correspondent is" msgid "correspondent is"
msgstr "" msgstr ""
#: documents/models.py:378 #: documents/models.py:376
msgid "document type is" msgid "document type is"
msgstr "" msgstr ""
#: documents/models.py:379 #: documents/models.py:377
msgid "is in inbox" msgid "is in inbox"
msgstr "" msgstr ""
#: documents/models.py:380 #: documents/models.py:378
msgid "has tag" msgid "has tag"
msgstr "" msgstr ""
#: documents/models.py:381 #: documents/models.py:379
msgid "has any tag" msgid "has any tag"
msgstr "" msgstr ""
#: documents/models.py:382 #: documents/models.py:380
msgid "created before" msgid "created before"
msgstr "" msgstr ""
#: documents/models.py:383 #: documents/models.py:381
msgid "created after" msgid "created after"
msgstr "" msgstr ""
#: documents/models.py:384 #: documents/models.py:382
msgid "created year is" msgid "created year is"
msgstr "" msgstr ""
#: documents/models.py:385 #: documents/models.py:383
msgid "created month is" msgid "created month is"
msgstr "" msgstr ""
#: documents/models.py:386 #: documents/models.py:384
msgid "created day is" msgid "created day is"
msgstr "" msgstr ""
#: documents/models.py:387 #: documents/models.py:385
msgid "added before" msgid "added before"
msgstr "" msgstr ""
#: documents/models.py:388 #: documents/models.py:386
msgid "added after" msgid "added after"
msgstr "" msgstr ""
#: documents/models.py:389 #: documents/models.py:387
msgid "modified before" msgid "modified before"
msgstr "" msgstr ""
#: documents/models.py:390 #: documents/models.py:388
msgid "modified after" msgid "modified after"
msgstr "" msgstr ""
#: documents/models.py:391 #: documents/models.py:389
msgid "does not have tag" msgid "does not have tag"
msgstr "" msgstr ""
#: documents/models.py:402 #: documents/models.py:400
msgid "rule type" msgid "rule type"
msgstr "" msgstr ""
#: documents/models.py:406 #: documents/models.py:404
msgid "value" msgid "value"
msgstr "" msgstr ""
#: documents/models.py:412 #: documents/models.py:410
msgid "filter rule" msgid "filter rule"
msgstr "" msgstr ""
#: documents/models.py:413 #: documents/models.py:411
msgid "filter rules" msgid "filter rules"
msgstr "" msgstr ""
@@ -378,23 +432,23 @@ msgstr ""
msgid "Sign in" msgid "Sign in"
msgstr "" msgstr ""
#: paperless/settings.py:268 #: paperless/settings.py:286
msgid "English" msgid "English"
msgstr "" msgstr ""
#: paperless/settings.py:269 #: paperless/settings.py:287
msgid "German" msgid "German"
msgstr "" msgstr ""
#: paperless/settings.py:270 #: paperless/settings.py:288
msgid "Dutch" msgid "Dutch"
msgstr "" msgstr ""
#: paperless/settings.py:271 #: paperless/settings.py:289
msgid "French" msgid "French"
msgstr "" msgstr ""
#: paperless/urls.py:108 #: paperless/urls.py:114
msgid "Paperless-ng administration" msgid "Paperless-ng administration"
msgstr "" msgstr ""