From 85721f1d441ff9db85818c9b491092b58f3ac09e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 17 Nov 2020 18:38:52 +0100 Subject: [PATCH 01/52] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 416f0ca06..c1f5b14f8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ [![Build Status](https://travis-ci.org/jonaswinkler/paperless-ng.svg?branch=master)](https://travis-ci.org/jonaswinkler/paperless-ng) [![Documentation Status](https://readthedocs.org/projects/paperless-ng/badge/?version=latest)](https://paperless-ng.readthedocs.io/en/latest/?badge=latest) [![Docker Hub Pulls](https://img.shields.io/docker/pulls/jonaswinkler/paperless-ng.svg)](https://hub.docker.com/r/jonaswinkler/paperless-ng) +[![Coverage Status](https://coveralls.io/repos/github/jonaswinkler/paperless-ng/badge.svg?branch=master)](https://coveralls.io/github/jonaswinkler/paperless-ng?branch=master) # Paperless-ng From 2c9555015b7b4bcc58d93a7e5bf36ee88d596150 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Fri, 20 Nov 2020 11:21:09 +0100 Subject: [PATCH 02/52] make the index dir if it does not exist. --- src/documents/index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/documents/index.py b/src/documents/index.py index d46ccedaf..ad3a50010 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -1,6 +1,8 @@ import logging +import os from contextlib import contextmanager +from django.conf import settings from whoosh import highlight from whoosh.fields import Schema, TEXT, NUMERIC from whoosh.highlight import Formatter, get_text @@ -8,7 +10,6 @@ from whoosh.index import create_in, exists_in, open_dir from whoosh.qparser import MultifieldParser from whoosh.writing import AsyncWriter -from paperless import settings logger = logging.getLogger(__name__) @@ -69,6 +70,8 @@ def open_index(recreate=False): # TODO: this is not thread safe. If 2 instances try to create the index # at the same time, this fails. This currently prevents parallel # tests. + if not os.path.isdir(settings.INDEX_DIR): + os.makedirs(settings.INDEX_DIR, exist_ok=True) return create_in(settings.INDEX_DIR, get_schema()) From 1255ecf86e2d83787a70f0d546cedf5fcd132f63 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Fri, 20 Nov 2020 11:28:19 +0100 Subject: [PATCH 03/52] update dependencies. --- Pipfile | 48 ++++++++++++++++++++++++------------------------ Pipfile.lock | 36 ++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Pipfile b/Pipfile index 66d60845b..ad60e0905 100644 --- a/Pipfile +++ b/Pipfile @@ -9,43 +9,43 @@ verify_ssl = true name = "piwheels" [packages] -django = "~=3.1" -pillow = "*" -dateparser = "~=0.7" +dateparser = "~=0.7.6" +django = "~=3.1.3" django-cors-headers = "*" -djangorestframework = "~=3.12" -python-gnupg = "*" -python-dotenv = "*" -filemagic = "*" -pyocr = "~=0.7" +django-extensions = "*" +django-filter = "~=2.4.0" +django-q = "~=1.3.4" +djangorestframework = "~=3.12.2" +fuzzywuzzy = "*" +gunicorn = "*" +imap-tools = "*" langdetect = "*" pdftotext = "*" -django-filter = "~=2.4" -python-dateutil = "*" -psycopg2-binary = "*" -scikit-learn="~=0.23" -whoosh="~=2.7" -gunicorn = "*" -whitenoise = "~=5.2" -fuzzywuzzy = "*" -python-Levenshtein = "*" -django-extensions = "*" -watchdog = "*" pathvalidate = "*" -django-q = "*" +pillow = "*" +pyocr = "~=0.7.2" +python-gnupg = "*" +python-dotenv = "*" +python-dateutil = "*" +python-Levenshtein = "*" +python-magic = "*" +psycopg2-binary = "*" redis = "*" -imap-tools = "*" +scikit-learn="~=0.23.2" +whitenoise = "~=5.2.0" +watchdog = "*" +whoosh="~=2.7.4" [dev-packages] coveralls = "*" factory-boy = "*" -sphinx = "~=3.3" -tox = "*" pycodestyle = "*" pytest = "*" pytest-cov = "*" pytest-django = "*" -pytest-sugar = "*" pytest-env = "*" +pytest-sugar = "*" pytest-xdist = "*" +sphinx = "~=3.3" sphinx_rtd_theme = "*" +tox = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 15a30e1c0..6ecca3c34 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "abc7e5f5a8d075d4b013ceafd06ca07f57e597f053d670f73449ba210511b114" + "sha256": "ae2643b9cf0cf5741ae149fb6bc0c480de41329ce48e773eb4b5d760bc5e2244" }, "pipfile-spec": 6, "requires": {}, @@ -105,14 +105,6 @@ "index": "pypi", "version": "==3.12.2" }, - "filemagic": { - "hashes": [ - "sha256:b2fd77411975510e28673220c4b8868ed81b5eb5906339b6f4c233b32122d7d3", - "sha256:e684359ef40820fe406f0ebc5bf8a78f89717bdb7fed688af68082d991d6dbf3" - ], - "index": "pypi", - "version": "==1.6" - }, "fuzzywuzzy": { "hashes": [ "sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8", @@ -131,11 +123,11 @@ }, "imap-tools": { "hashes": [ - "sha256:070929b8ec429c0aad94588a37a2962eed656a119ab61dcf91489f20fe983f5d", - "sha256:6232cd43748741496446871e889eb137351fc7a7e7f4c7888cd8c0fa28e20cda" + "sha256:96e9a4ff6483462635737730a1df28e739faa71967b12a84f4363fb386542246", + "sha256:a3ee1827dc4ff185b259b33d0238b091a87d489f63ee59959fcc81716456c602" ], "index": "pypi", - "version": "==0.31.0" + "version": "==0.32.0" }, "joblib": { "hashes": [ @@ -337,6 +329,14 @@ "index": "pypi", "version": "==0.12.0" }, + "python-magic": { + "hashes": [ + "sha256:356efa93c8899047d1eb7d3eb91e871ba2f5b1376edbaf4cc305e3c872207355", + "sha256:b757db2a5289ea3f1ced9e60f072965243ea43a2221430048fd8cacab17be0ce" + ], + "index": "pypi", + "version": "==0.4.18" + }, "pytz": { "hashes": [ "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268", @@ -617,11 +617,11 @@ }, "coveralls": { "hashes": [ - "sha256:4430b862baabb3cf090d36d84d331966615e4288d8a8c5957e0fd456d0dd8bd6", - "sha256:b3b60c17b03a0dee61952a91aed6f131e0b2ac8bd5da909389c53137811409e1" + "sha256:2301a19500b06649d2ec4f2858f9c69638d7699a4c63027c5d53daba666147cc", + "sha256:b990ba1f7bc4288e63340be0433698c1efe8217f78c689d254c2540af3d38617" ], "index": "pypi", - "version": "==2.1.2" + "version": "==2.2.0" }, "distlib": { "hashes": [ @@ -663,11 +663,11 @@ }, "faker": { "hashes": [ - "sha256:4d038ba51ae5e0a956d79cadd684d856e5750bfd608b61dad1807f8f08b1da49", - "sha256:f260f0375a44cd1e1a735c9b8c9b914304f607b5eef431d20e098c7c2f5b50a6" + "sha256:3f5d379e4b5ce92a8afe3c2ce59d7c43886370dd3bf9495a936b91888debfc81", + "sha256:8c0e8a06acef4b9312902e2ce18becabe62badd3a6632180bd0680c6ee111473" ], "markers": "python_version >= '3.5'", - "version": "==4.16.0" + "version": "==4.17.0" }, "filelock": { "hashes": [ From 28ea67f252289fc05479d04a0711735f21422ff4 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Fri, 20 Nov 2020 11:28:30 +0100 Subject: [PATCH 04/52] removed some empty folders. --- data/.keep | 0 data/index/.keep | 0 media/documents/.keep | 0 media/documents/originals/.keep | 0 media/documents/thumbnails/.keep | 0 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 data/.keep delete mode 100644 data/index/.keep delete mode 100644 media/documents/.keep delete mode 100644 media/documents/originals/.keep delete mode 100644 media/documents/thumbnails/.keep diff --git a/data/.keep b/data/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/data/index/.keep b/data/index/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/media/documents/.keep b/media/documents/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/media/documents/originals/.keep b/media/documents/originals/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/media/documents/thumbnails/.keep b/media/documents/thumbnails/.keep deleted file mode 100644 index e69de29bb..000000000 From 8681cad77c412aef956e1f0b5883587213a2fc0a Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Fri, 20 Nov 2020 11:29:34 +0100 Subject: [PATCH 05/52] add required packages to travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 10f2a4d73..2db24da87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: before_install: - sudo apt-get update -qq - - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript install: - pip install --upgrade pipenv From bd45a804a7dc1e1ec14de45f24134680b7545621 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Fri, 20 Nov 2020 13:28:30 +0100 Subject: [PATCH 06/52] docs --- docs/setup.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/setup.rst b/docs/setup.rst index 71acfba42..0f5db1ae5 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -10,10 +10,10 @@ Go to the project page on GitHub and download the `latest release `_. There are multiple options available. -* Download the docker-compose files if you want to pull paperless from +* Download the dockerfiles archive if you want to pull paperless from Docker Hub. -* Download the archive and extract it if you want to build the docker image +* Download the dist archive and extract it if you want to build the docker image yourself or want to install paperless without docker. .. hint:: @@ -22,6 +22,15 @@ There are multiple options available. is not to pull the entire git repository. Paperless-ng includes artifacts that need to be compiled, and that's already done for you in the release. +.. admonition:: Want to try out paperless-ng before migrating? + + The release contains a file ``.env`` which sets the docker-compose project + name to "paperless", which is the same as before and instructs docker-compose + to reuse and upgrade your paperless volumes. + + Just rename the project name in that file to anything else and docker-compose + will create fresh volumes for you! + Overview of Paperless-ng ######################## From 41650f20f458482bd855c12288cb22c0385e5bdc Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Fri, 20 Nov 2020 13:31:03 +0100 Subject: [PATCH 07/52] mime type handling --- src/documents/admin.py | 5 +- src/documents/consumer.py | 24 ++++----- src/documents/file_handling.py | 4 +- .../management/commands/document_exporter.py | 4 +- src/documents/migrations/1003_mime_types.py | 50 +++++++++++++++++++ src/documents/models.py | 28 ++++------- src/documents/parsers.py | 24 ++++++--- src/documents/serialisers.py | 2 +- src/documents/tests/test_api.py | 12 ++--- src/documents/tests/test_consumer.py | 16 +++++- src/documents/tests/test_document_model.py | 3 ++ src/documents/tests/test_file_handling.py | 34 ++++++------- src/documents/tests/test_matchables.py | 2 +- src/documents/tests/test_parsers.py | 18 +++++-- src/documents/views.py | 14 +----- src/paperless_mail/mail.py | 4 +- src/paperless_tesseract/signals.py | 15 ++---- src/paperless_tesseract/tests/test_signals.py | 36 ------------- src/paperless_text/signals.py | 14 ++---- 19 files changed, 163 insertions(+), 146 deletions(-) create mode 100644 src/documents/migrations/1003_mime_types.py delete mode 100644 src/paperless_tesseract/tests/test_signals.py diff --git a/src/documents/admin.py b/src/documents/admin.py index 209ddff35..5b3975fda 100755 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -50,7 +50,7 @@ class DocumentTypeAdmin(admin.ModelAdmin): class DocumentAdmin(admin.ModelAdmin): search_fields = ("correspondent__name", "title", "content", "tags__name") - readonly_fields = ("added", "file_type", "storage_type", "filename") + readonly_fields = ("added", "mime_type", "storage_type", "filename") list_display = ( "title", "created", @@ -58,8 +58,7 @@ class DocumentAdmin(admin.ModelAdmin): "correspondent", "tags_", "archive_serial_number", - "document_type", - "filename" + "document_type" ) list_filter = ( "document_type", diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 3cd57796e..b8eb8cfca 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -2,8 +2,8 @@ import datetime import hashlib import logging import os -import re +import magic from django.conf import settings from django.db import transaction from django.utils import timezone @@ -13,7 +13,7 @@ from .classifier import DocumentClassifier, IncompatibleClassifierVersionError from .file_handling import generate_filename, create_source_path_directory from .loggers import LoggingMixin from .models import Document, FileInfo, Correspondent, DocumentType, Tag -from .parsers import ParseError, get_parser_class +from .parsers import ParseError, get_parser_class_for_mime_type from .signals import ( document_consumption_finished, document_consumption_started @@ -51,12 +51,6 @@ class Consumer(LoggingMixin): "Consumption directory {} does not exist".format( settings.CONSUMPTION_DIR)) - def pre_check_regex(self): - if not re.match(FileInfo.REGEXES["title"], self.filename): - raise ConsumerError( - "Filename {} does not seem to be safe to " - "consume".format(self.filename)) - def pre_check_duplicate(self): with open(self.path, "rb") as f: checksum = hashlib.md5(f.read()).hexdigest() @@ -100,18 +94,19 @@ class Consumer(LoggingMixin): self.pre_check_file_exists() self.pre_check_consumption_dir() self.pre_check_directories() - self.pre_check_regex() self.pre_check_duplicate() self.log("info", "Consuming {}".format(self.filename)) # Determine the parser class. - parser_class = get_parser_class(self.filename) + mime_type = magic.from_file(self.path, mime=True) + + parser_class = get_parser_class_for_mime_type(mime_type) if not parser_class: raise ConsumerError("No parsers abvailable for {}".format(self.filename)) else: - self.log("debug", "Parser: {}".format(parser_class.__name__)) + self.log("debug", "Parser: {} based on mime type {}".format(parser_class.__name__, mime_type)) # Notify all listeners that we're going to do some work. @@ -162,7 +157,8 @@ class Consumer(LoggingMixin): # store the document. document = self._store( text=text, - date=date + date=date, + mime_type=mime_type ) # If we get here, it was successful. Proceed with post-consume @@ -197,7 +193,7 @@ class Consumer(LoggingMixin): return document - def _store(self, text, date): + def _store(self, text, date, mime_type): # If someone gave us the original filename, use it instead of doc. @@ -220,7 +216,7 @@ class Consumer(LoggingMixin): correspondent=file_info.correspondent, title=file_info.title, content=text, - file_type=file_info.extension, + mime_type=mime_type, checksum=hashlib.md5(f.read()).hexdigest(), created=created, modified=created, diff --git a/src/documents/file_handling.py b/src/documents/file_handling.py index 024003118..06d4d2957 100644 --- a/src/documents/file_handling.py +++ b/src/documents/file_handling.py @@ -91,9 +91,9 @@ def generate_filename(document): # Always append the primary key to guarantee uniqueness of filename if len(path) > 0: - filename = "%s-%07i.%s" % (path, document.pk, document.file_type) + filename = "%s-%07i%s" % (path, document.pk, document.file_type) else: - filename = "%07i.%s" % (document.pk, document.file_type) + filename = "%07i%s" % (document.pk, document.file_type) # Append .gpg for encrypted files if document.storage_type == document.STORAGE_TYPE_GPG: diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index 971e6a829..441f1c475 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -127,8 +127,8 @@ class Command(Renderable, BaseCommand): tags = ",".join([t.slug for t in doc.tags.all()]) if tags: - return "{} - {} - {} - {}.{}".format( + return "{} - {} - {} - {}{}".format( created, doc.correspondent, doc.title, tags, doc.file_type) - return "{} - {} - {}.{}".format( + return "{} - {} - {}{}".format( created, doc.correspondent, doc.title, doc.file_type) diff --git a/src/documents/migrations/1003_mime_types.py b/src/documents/migrations/1003_mime_types.py new file mode 100644 index 000000000..4c73a4235 --- /dev/null +++ b/src/documents/migrations/1003_mime_types.py @@ -0,0 +1,50 @@ +# Generated by Django 3.1.3 on 2020-11-20 11:21 +import os + +import magic +from django.conf import settings +from django.db import migrations, models + + +def source_path(self): + if self.filename: + fname = str(self.filename) + else: + fname = "{:07}.{}".format(self.pk, self.file_type) + if self.storage_type == self.STORAGE_TYPE_GPG: + fname += ".gpg" + + return os.path.join( + settings.ORIGINALS_DIR, + fname + ) + + +def add_mime_types(apps, schema_editor): + Document = apps.get_model("documents", "Document") + documents = Document.objects.all() + + for d in documents: + d.mime_type = magic.from_file(source_path(d), mime=True) + d.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '1002_auto_20201111_1105'), + ] + + operations = [ + migrations.AddField( + model_name='document', + name='mime_type', + field=models.CharField(default="-", editable=False, max_length=256), + preserve_default=False, + ), + migrations.RunPython(add_mime_types), + migrations.RemoveField( + model_name='document', + name='file_type', + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 4badd2d56..559c395e0 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -1,6 +1,7 @@ # coding=utf-8 import logging +import mimetypes import os import re from collections import OrderedDict @@ -113,18 +114,6 @@ class DocumentType(MatchingModel): class Document(models.Model): - # TODO: why do we need an explicit list - TYPE_PDF = "pdf" - TYPE_PNG = "png" - TYPE_JPG = "jpg" - TYPE_GIF = "gif" - TYPE_TIF = "tiff" - TYPE_TXT = "txt" - TYPE_CSV = "csv" - TYPE_MD = "md" - TYPES = (TYPE_PDF, TYPE_PNG, TYPE_JPG, TYPE_GIF, TYPE_TIF, - TYPE_TXT, TYPE_CSV, TYPE_MD) - STORAGE_TYPE_UNENCRYPTED = "unencrypted" STORAGE_TYPE_GPG = "gpg" STORAGE_TYPES = ( @@ -156,10 +145,9 @@ class Document(models.Model): "primarily used for searching." ) - file_type = models.CharField( - max_length=4, - editable=False, - choices=tuple([(t, t.upper()) for t in TYPES]) + mime_type = models.CharField( + max_length=256, + editable=False ) tags = models.ManyToManyField( @@ -223,7 +211,7 @@ class Document(models.Model): if self.filename: fname = str(self.filename) else: - fname = "{:07}.{}".format(self.pk, self.file_type) + fname = "{:07}{}".format(self.pk, self.file_type) if self.storage_type == self.STORAGE_TYPE_GPG: fname += ".gpg" @@ -238,7 +226,11 @@ class Document(models.Model): @property def file_name(self): - return slugify(str(self)) + "." + self.file_type + return slugify(str(self)) + self.file_type + + @property + def file_type(self): + return mimetypes.guess_extension(str(self.mime_type)) @property def thumbnail_path(self): diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 496efa188..98f4c5b12 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -6,6 +6,7 @@ import subprocess import tempfile import dateparser +import magic from django.conf import settings from django.utils import timezone @@ -37,10 +38,11 @@ DATE_REGEX = re.compile( logger = logging.getLogger(__name__) -def get_parser_class(doc): - """ - Determine the appropriate parser class based on the file - """ +def is_mime_type_supported(mime_type): + return get_parser_class_for_mime_type(mime_type) is not None + + +def get_parser_class_for_mime_type(mime_type): options = [] @@ -48,9 +50,9 @@ def get_parser_class(doc): for response in document_consumer_declaration.send(None): parser_declaration = response[1] - parser_test = parser_declaration["test"] + supported_mime_types = parser_declaration["mime_types"] - if parser_test(doc): + if mime_type in supported_mime_types: options.append(parser_declaration) if not options: @@ -61,6 +63,16 @@ def get_parser_class(doc): options, key=lambda _: _["weight"], reverse=True)[0]["parser"] +def get_parser_class(path): + """ + Determine the appropriate parser class based on the file + """ + + mime_type = magic.from_file(path, mime=True) + + return get_parser_class_for_mime_type(mime_type) + + def run_convert(input_file, output_file, density=None, scale=None, alpha=None, strip=False, trim=False, type=None, depth=None, extra=None, logging_group=None): environment = os.environ.copy() if settings.CONVERT_MEMORY_LIMIT: diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index e42e26881..cf48e8bd7 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -91,7 +91,7 @@ class DocumentSerializer(serializers.ModelSerializer): "document_type_id", "title", "content", - "file_type", + "mime_type", "tags", "tags_id", "checksum", diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index a049fb825..b0318d2b3 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -45,7 +45,7 @@ class DocumentApiTest(APITestCase): dt = DocumentType.objects.create(name="dt", pk=63) tag = Tag.objects.create(name="t", pk=85) - doc = Document.objects.create(title="WOW", content="the content", correspondent=c, document_type=dt, checksum="123") + doc = Document.objects.create(title="WOW", content="the content", correspondent=c, document_type=dt, checksum="123", mime_type="application/pdf") doc.tags.add(tag) @@ -95,7 +95,7 @@ class DocumentApiTest(APITestCase): with open(filename, "wb") as f: f.write(content) - doc = Document.objects.create(title="none", filename=os.path.basename(filename), file_type="pdf") + doc = Document.objects.create(title="none", filename=os.path.basename(filename), mime_type="application/pdf") with open(os.path.join(self.thumbnail_dir, "{:07d}.png".format(doc.pk)), "wb") as f: f.write(content_thumbnail) @@ -117,7 +117,7 @@ class DocumentApiTest(APITestCase): def test_document_actions_not_existing_file(self): - doc = Document.objects.create(title="none", filename=os.path.basename("asd"), file_type="pdf") + doc = Document.objects.create(title="none", filename=os.path.basename("asd"), mime_type="application/pdf") response = self.client.get('/api/documents/{}/download/'.format(doc.pk)) self.assertEqual(response.status_code, 404) @@ -130,9 +130,9 @@ class DocumentApiTest(APITestCase): def test_document_filters(self): - doc1 = Document.objects.create(title="none1", checksum="A") - doc2 = Document.objects.create(title="none2", checksum="B") - doc3 = Document.objects.create(title="none3", checksum="C") + doc1 = Document.objects.create(title="none1", checksum="A", mime_type="application/pdf") + doc2 = Document.objects.create(title="none2", checksum="B", mime_type="application/pdf") + doc3 = Document.objects.create(title="none3", checksum="C", mime_type="application/pdf") tag_inbox = Tag.objects.create(name="t1", is_inbox_tag=True) tag_2 = Tag.objects.create(name="t2") diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index f61fd5718..a89bd75ae 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -437,6 +437,18 @@ class FaultyParser(DocumentParser): raise ParseError("Does not compute.") +def fake_magic_from_file(file, mime=False): + + if mime: + if os.path.splitext(file)[1] == ".pdf": + return "application/pdf" + else: + return "unknown" + else: + return "A verbose string that describes the contents of the file" + + +@mock.patch("documents.consumer.magic.from_file", fake_magic_from_file) class TestConsumer(TestCase): def make_dummy_parser(self, path, logging_group): @@ -462,7 +474,7 @@ class TestConsumer(TestCase): m = patcher.start() m.return_value = [(None, { "parser": self.make_dummy_parser, - "test": lambda _: True, + "mime_types": ["application/pdf"], "weight": 0 })] @@ -592,7 +604,7 @@ class TestConsumer(TestCase): def testFaultyParser(self, m): m.return_value = [(None, { "parser": self.make_faulty_parser, - "test": lambda _: True, + "mime_types": ["application/pdf"], "weight": 0 })] diff --git a/src/documents/tests/test_document_model.py b/src/documents/tests/test_document_model.py index 2da674527..5b27e2643 100644 --- a/src/documents/tests/test_document_model.py +++ b/src/documents/tests/test_document_model.py @@ -13,9 +13,12 @@ class TestDocument(TestCase): title="Title", content="content", checksum="checksum", + mime_type="application/pdf" ) + file_path = document.source_path thumb_path = document.thumbnail_path + with mock.patch("documents.signals.handlers.os.unlink") as mock_unlink: document.delete() mock_unlink.assert_any_call(file_path) diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index d44e5056a..5ffd35f61 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -31,7 +31,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="") def test_generate_source_filename(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -44,7 +44,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}") def test_file_renaming(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -81,7 +81,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}") def test_file_renaming_missing_permissions(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -111,10 +111,10 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}") def test_file_renaming_database_error(self): - document1 = Document.objects.create(file_type="pdf", storage_type=Document.STORAGE_TYPE_UNENCRYPTED, checksum="AAAAA") + document1 = Document.objects.create(mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_UNENCRYPTED, checksum="AAAAA") document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.checksum = "BBBBB" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -149,7 +149,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}") def test_document_delete(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -170,7 +170,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}") def test_document_delete_nofile(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -179,7 +179,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}") def test_directory_not_empty(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -206,7 +206,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}") def test_tags_with_underscore(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -222,7 +222,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}") def test_tags_with_dash(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -238,7 +238,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}") def test_tags_malformed(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -254,7 +254,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[0]}") def test_tags_all(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -269,7 +269,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[1]}") def test_tags_out_of_bounds(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -284,7 +284,7 @@ class TestDate(TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{correspondent}/{correspondent}") def test_nested_directory_cleanup(self): document = Document() - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED document.save() @@ -309,7 +309,7 @@ class TestDate(TestCase): def test_format_none(self): document = Document() document.pk = 1 - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED self.assertEqual(generate_filename(document), "0000001.pdf") @@ -335,7 +335,7 @@ class TestDate(TestCase): def test_invalid_format(self): document = Document() document.pk = 1 - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED self.assertEqual(generate_filename(document), "0000001.pdf") @@ -344,7 +344,7 @@ class TestDate(TestCase): def test_invalid_format_key(self): document = Document() document.pk = 1 - document.file_type = "pdf" + document.mime_type = "application/pdf" document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED self.assertEqual(generate_filename(document), "0000001.pdf") diff --git a/src/documents/tests/test_matchables.py b/src/documents/tests/test_matchables.py index 93601b9d2..24e285ae7 100644 --- a/src/documents/tests/test_matchables.py +++ b/src/documents/tests/test_matchables.py @@ -213,7 +213,7 @@ class TestDocumentConsumptionFinishedSignal(TestCase): TestCase.setUp(self) User.objects.create_user(username='test_consumer', password='12345') self.doc_contains = Document.objects.create( - content="I contain the keyword.", file_type="pdf") + content="I contain the keyword.", mime_type="application/pdf") def test_tag_applied_any(self): t1 = Tag.objects.create( diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index 5896f3ba3..e99bb8dc6 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -1,3 +1,4 @@ +import os from tempfile import TemporaryDirectory from unittest import mock @@ -5,7 +6,18 @@ from django.test import TestCase from documents.parsers import get_parser_class +def fake_magic_from_file(file, mime=False): + if mime: + if os.path.splitext(file)[1] == ".pdf": + return "application/pdf" + else: + return "unknown" + else: + return "A verbose string that describes the contents of the file" + + +@mock.patch("documents.parsers.magic.from_file", fake_magic_from_file) class TestParserDiscovery(TestCase): @mock.patch("documents.parsers.document_consumer_declaration.send") @@ -14,7 +26,7 @@ class TestParserDiscovery(TestCase): pass m.return_value = ( - (None, {"weight": 0, "parser": DummyParser, "test": lambda _: True}), + (None, {"weight": 0, "parser": DummyParser, "mime_types": ["application/pdf"]}), ) self.assertEqual( @@ -32,8 +44,8 @@ class TestParserDiscovery(TestCase): pass m.return_value = ( - (None, {"weight": 0, "parser": DummyParser1, "test": lambda _: True}), - (None, {"weight": 1, "parser": DummyParser2, "test": lambda _: True}), + (None, {"weight": 0, "parser": DummyParser1, "mime_types": ["application/pdf"]}), + (None, {"weight": 1, "parser": DummyParser2, "mime_types": ["application/pdf"]}), ) self.assertEqual( diff --git a/src/documents/views.py b/src/documents/views.py index f4c5d0797..89d03a4df 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -104,18 +104,6 @@ class DocumentViewSet(RetrieveModelMixin, return super(DocumentViewSet, self).destroy(request, *args, **kwargs) def file_response(self, pk, disposition): - # TODO: this should not be necessary here. - content_types = { - Document.TYPE_PDF: "application/pdf", - Document.TYPE_PNG: "image/png", - Document.TYPE_JPG: "image/jpeg", - Document.TYPE_GIF: "image/gif", - Document.TYPE_TIF: "image/tiff", - Document.TYPE_CSV: "text/csv", - Document.TYPE_MD: "text/markdown", - Document.TYPE_TXT: "text/plain" - } - doc = Document.objects.get(id=pk) if doc.storage_type == Document.STORAGE_TYPE_UNENCRYPTED: @@ -123,7 +111,7 @@ class DocumentViewSet(RetrieveModelMixin, else: file_handle = GnuPG.decrypted(doc.source_file) - response = HttpResponse(file_handle, content_type=content_types[doc.file_type]) + response = HttpResponse(file_handle, content_type=doc.mime_type) response["Content-Disposition"] = '{}; filename="{}"'.format( disposition, doc.file_name) return response diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index b942e420a..1aea65d90 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -10,6 +10,7 @@ from imap_tools import MailBox, MailBoxUnencrypted, AND, MailMessageFlags, \ from documents.loggers import LoggingMixin from documents.models import Correspondent +from documents.parsers import is_mime_type_supported from paperless_mail.models import MailAccount, MailRule @@ -249,8 +250,7 @@ class MailAccountHandler(LoggingMixin): title = get_title(message, att, rule) - # TODO: check with parsers what files types are supported - if att.content_type == 'application/pdf': + if is_mime_type_supported(att.content_type): os.makedirs(settings.SCRATCH_DIR, exist_ok=True) _, temp_filename = tempfile.mkstemp(prefix="paperless-mail-", dir=settings.SCRATCH_DIR) diff --git a/src/paperless_tesseract/signals.py b/src/paperless_tesseract/signals.py index 3fc6c2a11..712034038 100644 --- a/src/paperless_tesseract/signals.py +++ b/src/paperless_tesseract/signals.py @@ -1,5 +1,3 @@ -import re - from .parsers import RasterisedDocumentParser @@ -7,12 +5,9 @@ def tesseract_consumer_declaration(sender, **kwargs): return { "parser": RasterisedDocumentParser, "weight": 0, - "test": tesseract_consumer_test + "mime_types": [ + "application/pdf", + "image/jpeg", + "image/png" + ] } - - -MATCHING_FILES = re.compile(r"^.*\.(pdf|jpe?g|gif|png|tiff?|pnm|bmp)$") - - -def tesseract_consumer_test(doc): - return MATCHING_FILES.match(doc.lower()) diff --git a/src/paperless_tesseract/tests/test_signals.py b/src/paperless_tesseract/tests/test_signals.py deleted file mode 100644 index 354557732..000000000 --- a/src/paperless_tesseract/tests/test_signals.py +++ /dev/null @@ -1,36 +0,0 @@ -from django.test import TestCase - -from paperless_tesseract.signals import tesseract_consumer_test - - -class SignalsTestCase(TestCase): - - def test_test_handles_various_file_names_true(self): - - prefixes = ( - "doc", "My Document", "Μυ Γρεεκ Δοψθμεντ", "Doc -with - tags", - "A document with a . in it", "Doc with -- in it" - ) - suffixes = ( - "pdf", "jpg", "jpeg", "gif", "png", "tiff", "tif", "pnm", "bmp", - "PDF", "JPG", "JPEG", "GIF", "PNG", "TIFF", "TIF", "PNM", "BMP", - "pDf", "jPg", "jpEg", "gIf", "pNg", "tIff", "tIf", "pNm", "bMp", - ) - - for prefix in prefixes: - for suffix in suffixes: - name = "{}.{}".format(prefix, suffix) - self.assertTrue(tesseract_consumer_test(name)) - - def test_test_handles_various_file_names_false(self): - - prefixes = ("doc",) - suffixes = ("txt", "markdown", "",) - - for prefix in prefixes: - for suffix in suffixes: - name = "{}.{}".format(prefix, suffix) - self.assertFalse(tesseract_consumer_test(name)) - - self.assertFalse(tesseract_consumer_test("")) - self.assertFalse(tesseract_consumer_test("doc")) diff --git a/src/paperless_text/signals.py b/src/paperless_text/signals.py index 784bfd45d..f9ac9ad23 100644 --- a/src/paperless_text/signals.py +++ b/src/paperless_text/signals.py @@ -1,5 +1,3 @@ -import re - from .parsers import TextDocumentParser @@ -7,12 +5,8 @@ def text_consumer_declaration(sender, **kwargs): return { "parser": TextDocumentParser, "weight": 10, - "test": text_consumer_test + "mime_types": [ + "text/plain", + "text/comma-separated-values" + ] } - - -MATCHING_FILES = re.compile(r"^.*\.(te?xt|md|csv)$") - - -def text_consumer_test(doc): - return MATCHING_FILES.match(doc.lower()) From 3d5b66c2b77f8653758d87d432e6d379f69a5399 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Fri, 20 Nov 2020 16:18:59 +0100 Subject: [PATCH 08/52] FileType does not care about the extension anymore. --- src/documents/consumer.py | 2 +- src/documents/forms.py | 3 +- src/documents/models.py | 64 ++++---- src/documents/tests/test_consumer.py | 213 +++++++++++---------------- 4 files changed, 118 insertions(+), 164 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index b8eb8cfca..175f6710f 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -197,7 +197,7 @@ class Consumer(LoggingMixin): # If someone gave us the original filename, use it instead of doc. - file_info = FileInfo.from_path(self.filename) + file_info = FileInfo.from_filename(self.filename) stats = os.stat(self.path) diff --git a/src/documents/forms.py b/src/documents/forms.py index 38a95a068..c3efc774f 100644 --- a/src/documents/forms.py +++ b/src/documents/forms.py @@ -34,8 +34,7 @@ class UploadForm(forms.Form): os.makedirs(settings.SCRATCH_DIR, exist_ok=True) - # TODO: dont just append pdf. This is here for taht weird regex check at the start of the consumer. - with tempfile.NamedTemporaryFile(prefix="paperless-upload-", suffix=".pdf", dir=settings.SCRATCH_DIR, delete=False) as f: + with tempfile.NamedTemporaryFile(prefix="paperless-upload-", dir=settings.SCRATCH_DIR, delete=False) as f: f.write(document) os.utime(f.name, times=(t, t)) diff --git a/src/documents/models.py b/src/documents/models.py index 559c395e0..6288980c5 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -269,7 +269,7 @@ class Log(models.Model): def __str__(self): return self.message - +# TODO: why is this in the models file? class FileInfo: # This epic regex *almost* worked for our needs, so I'm keeping it here for @@ -284,53 +284,44 @@ class FileInfo: non_separated_word=r"([\w,. ]|([^\s]-))" ) ) - # TODO: what is this used for - formats = "pdf|jpe?g|png|gif|tiff?|te?xt|md|csv" REGEXES = OrderedDict([ ("created-correspondent-title-tags", re.compile( r"^(?P\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " r"(?P.*) - " r"(?P.*) - " - r"(?P<tags>[a-z0-9\-,]*)" - r"\.(?P<extension>{})$".format(formats), + r"(?P<tags>[a-z0-9\-,]*)$", flags=re.IGNORECASE )), ("created-title-tags", re.compile( r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)" - r"\.(?P<extension>{})$".format(formats), + r"(?P<tags>[a-z0-9\-,]*)$", flags=re.IGNORECASE )), ("created-correspondent-title", re.compile( r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " r"(?P<correspondent>.*) - " - r"(?P<title>.*)" - r"\.(?P<extension>{})$".format(formats), + r"(?P<title>.*)$", flags=re.IGNORECASE )), ("created-title", re.compile( r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " - r"(?P<title>.*)" - r"\.(?P<extension>{})$".format(formats), + r"(?P<title>.*)$", flags=re.IGNORECASE )), ("correspondent-title-tags", re.compile( r"(?P<correspondent>.*) - " r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)" - r"\.(?P<extension>{})$".format(formats), + r"(?P<tags>[a-z0-9\-,]*)$", flags=re.IGNORECASE )), ("correspondent-title", re.compile( r"(?P<correspondent>.*) - " - r"(?P<title>.*)?" - r"\.(?P<extension>{})$".format(formats), + r"(?P<title>.*)?$", flags=re.IGNORECASE )), ("title", re.compile( - r"(?P<title>.*)" - r"\.(?P<extension>{})$".format(formats), + r"(?P<title>.*)$", flags=re.IGNORECASE )) ]) @@ -373,15 +364,6 @@ class FileInfo: )[0]) return tuple(r) - @classmethod - def _get_extension(cls, extension): - r = extension.lower() - if r == "jpeg": - return "jpg" - if r == "tif": - return "tiff" - return r - @classmethod def _mangle_property(cls, properties, name): if name in properties: @@ -390,18 +372,16 @@ class FileInfo: ) @classmethod - def from_path(cls, path): + def from_filename(cls, filename): """ We use a crude naming convention to make handling the correspondent, title, and tags easier: - "<date> - <correspondent> - <title> - <tags>.<suffix>" - "<correspondent> - <title> - <tags>.<suffix>" - "<correspondent> - <title>.<suffix>" - "<title>.<suffix>" + "<date> - <correspondent> - <title> - <tags>" + "<correspondent> - <title> - <tags>" + "<correspondent> - <title>" + "<title>" """ - filename = os.path.basename(path) - # Mutate filename in-place before parsing its components # by applying at most one of the configured transformations. for (pattern, repl) in settings.FILENAME_PARSE_TRANSFORMS: @@ -409,6 +389,23 @@ class FileInfo: if count: break + # do this after the transforms so that the transforms can do whatever + # with the file extension. + filename_no_ext = os.path.splitext(filename)[0] + + if filename_no_ext == filename and filename.startswith("."): + # This is a very special case where there is no text before the + # file type. + # TODO: this should be handled better. The ext is not removed + # because usually, files like '.pdf' are just hidden files + # with the name pdf, but in our case, its more likely that + # there's just no name to begin with. + filename = "" + # This isn't too bad either, since we'll just not match anything + # and return an empty title. TODO: actually, this is kinda bad. + else: + filename = filename_no_ext + # Parse filename components. for regex in cls.REGEXES.values(): m = regex.match(filename) @@ -418,5 +415,4 @@ class FileInfo: cls._mangle_property(properties, "correspondent") cls._mangle_property(properties, "title") cls._mangle_property(properties, "tags") - cls._mangle_property(properties, "extension") return cls(**properties) diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index a89bd75ae..6dab98d02 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -15,57 +15,42 @@ from ..parsers import DocumentParser, ParseError class TestAttributes(TestCase): TAGS = ("tag1", "tag2", "tag3") - EXTENSIONS = ( - "pdf", "png", "jpg", "jpeg", "gif", "tiff", "tif", - "PDF", "PNG", "JPG", "JPEG", "GIF", "TIFF", "TIF", - "PdF", "PnG", "JpG", "JPeG", "GiF", "TiFf", "TiF", - ) - def _test_guess_attributes_from_name(self, path, sender, title, tags): + def _test_guess_attributes_from_name(self, filename, sender, title, tags): + file_info = FileInfo.from_filename(filename) - for extension in self.EXTENSIONS: + if sender: + self.assertEqual(file_info.correspondent.name, sender, filename) + else: + self.assertIsNone(file_info.correspondent, filename) - f = path.format(extension) - file_info = FileInfo.from_path(f) + self.assertEqual(file_info.title, title, filename) - if sender: - self.assertEqual(file_info.correspondent.name, sender, f) - else: - self.assertIsNone(file_info.correspondent, f) - - self.assertEqual(file_info.title, title, f) - - self.assertEqual(tuple([t.slug for t in file_info.tags]), tags, f) - if extension.lower() == "jpeg": - self.assertEqual(file_info.extension, "jpg", f) - elif extension.lower() == "tif": - self.assertEqual(file_info.extension, "tiff", f) - else: - self.assertEqual(file_info.extension, extension.lower(), f) + self.assertEqual(tuple([t.slug for t in file_info.tags]), tags, filename) def test_guess_attributes_from_name0(self): self._test_guess_attributes_from_name( - "/path/to/Sender - Title.{}", "Sender", "Title", ()) + "Sender - Title.pdf", "Sender", "Title", ()) def test_guess_attributes_from_name1(self): self._test_guess_attributes_from_name( - "/path/to/Spaced Sender - Title.{}", "Spaced Sender", "Title", ()) + "Spaced Sender - Title.pdf", "Spaced Sender", "Title", ()) def test_guess_attributes_from_name2(self): self._test_guess_attributes_from_name( - "/path/to/Sender - Spaced Title.{}", "Sender", "Spaced Title", ()) + "Sender - Spaced Title.pdf", "Sender", "Spaced Title", ()) def test_guess_attributes_from_name3(self): self._test_guess_attributes_from_name( - "/path/to/Dashed-Sender - Title.{}", "Dashed-Sender", "Title", ()) + "Dashed-Sender - Title.pdf", "Dashed-Sender", "Title", ()) def test_guess_attributes_from_name4(self): self._test_guess_attributes_from_name( - "/path/to/Sender - Dashed-Title.{}", "Sender", "Dashed-Title", ()) + "Sender - Dashed-Title.pdf", "Sender", "Dashed-Title", ()) def test_guess_attributes_from_name5(self): self._test_guess_attributes_from_name( - "/path/to/Sender - Title - tag1,tag2,tag3.{}", + "Sender - Title - tag1,tag2,tag3.pdf", "Sender", "Title", self.TAGS @@ -73,7 +58,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name6(self): self._test_guess_attributes_from_name( - "/path/to/Spaced Sender - Title - tag1,tag2,tag3.{}", + "Spaced Sender - Title - tag1,tag2,tag3.pdf", "Spaced Sender", "Title", self.TAGS @@ -81,7 +66,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name7(self): self._test_guess_attributes_from_name( - "/path/to/Sender - Spaced Title - tag1,tag2,tag3.{}", + "Sender - Spaced Title - tag1,tag2,tag3.pdf", "Sender", "Spaced Title", self.TAGS @@ -89,7 +74,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name8(self): self._test_guess_attributes_from_name( - "/path/to/Dashed-Sender - Title - tag1,tag2,tag3.{}", + "Dashed-Sender - Title - tag1,tag2,tag3.pdf", "Dashed-Sender", "Title", self.TAGS @@ -97,7 +82,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name9(self): self._test_guess_attributes_from_name( - "/path/to/Sender - Dashed-Title - tag1,tag2,tag3.{}", + "Sender - Dashed-Title - tag1,tag2,tag3.pdf", "Sender", "Dashed-Title", self.TAGS @@ -105,7 +90,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name10(self): self._test_guess_attributes_from_name( - "/path/to/Σενδερ - Τιτλε - tag1,tag2,tag3.{}", + "Σενδερ - Τιτλε - tag1,tag2,tag3.pdf", "Σενδερ", "Τιτλε", self.TAGS @@ -113,7 +98,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name_when_correspondent_empty(self): self._test_guess_attributes_from_name( - '/path/to/ - weird empty correspondent but should not break.{}', + ' - weird empty correspondent but should not break.pdf', None, 'weird empty correspondent but should not break', () @@ -121,7 +106,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name_when_title_starts_with_dash(self): self._test_guess_attributes_from_name( - '/path/to/- weird but should not break.{}', + '- weird but should not break.pdf', None, '- weird but should not break', () @@ -129,7 +114,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name_when_title_ends_with_dash(self): self._test_guess_attributes_from_name( - '/path/to/weird but should not break -.{}', + 'weird but should not break -.pdf', None, 'weird but should not break -', () @@ -137,7 +122,7 @@ class TestAttributes(TestCase): def test_guess_attributes_from_name_when_title_is_empty(self): self._test_guess_attributes_from_name( - '/path/to/weird correspondent but should not break - .{}', + 'weird correspondent but should not break - .pdf', 'weird correspondent but should not break', '', () @@ -149,11 +134,11 @@ class TestAttributes(TestCase): :return: """ - path = "Title - Correspondent - tAg1,TAG2.pdf" - self.assertEqual(len(FileInfo.from_path(path).tags), 2) + filename = "Title - Correspondent - tAg1,TAG2.pdf" + self.assertEqual(len(FileInfo.from_filename(filename).tags), 2) path = "Title - Correspondent - tag1,tag2.pdf" - self.assertEqual(len(FileInfo.from_path(path).tags), 2) + self.assertEqual(len(FileInfo.from_filename(filename).tags), 2) self.assertEqual(Tag.objects.all().count(), 2) @@ -173,13 +158,12 @@ class TestFieldPermutations(TestCase): ] valid_titles = ["title", "Title w Spaces", "Title a-dash", "Τίτλος", ""] valid_tags = ["tag", "tig,tag", "tag1,tag2,tag-3"] - valid_extensions = ["pdf", "png", "jpg", "jpeg", "gif"] def _test_guessed_attributes(self, filename, created=None, correspondent=None, title=None, - extension=None, tags=None): + tags=None): - info = FileInfo.from_path(filename) + info = FileInfo.from_filename(filename) # Created if created is None: @@ -207,68 +191,56 @@ class TestFieldPermutations(TestCase): filename ) - # Extension - if extension == 'jpeg': - extension = 'jpg' - self.assertEqual(info.extension, extension, filename) - def test_just_title(self): - template = '/path/to/{title}.{extension}' + template = '{title}.pdf' for title in self.valid_titles: - for extension in self.valid_extensions: - spec = dict(title=title, extension=extension) + spec = dict(title=title) + filename = template.format(**spec) + self._test_guessed_attributes(filename, **spec) + + def test_title_and_correspondent(self): + template = '{correspondent} - {title}.pdf' + for correspondent in self.valid_correspondents: + for title in self.valid_titles: + spec = dict(correspondent=correspondent, title=title) filename = template.format(**spec) self._test_guessed_attributes(filename, **spec) - def test_title_and_correspondent(self): - template = '/path/to/{correspondent} - {title}.{extension}' - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - for extension in self.valid_extensions: - spec = dict(correspondent=correspondent, title=title, - extension=extension) - filename = template.format(**spec) - self._test_guessed_attributes(filename, **spec) - def test_title_and_correspondent_and_tags(self): - template = '/path/to/{correspondent} - {title} - {tags}.{extension}' + template = '{correspondent} - {title} - {tags}.pdf' for correspondent in self.valid_correspondents: for title in self.valid_titles: for tags in self.valid_tags: - for extension in self.valid_extensions: - spec = dict(correspondent=correspondent, title=title, - tags=tags, extension=extension) - filename = template.format(**spec) - self._test_guessed_attributes(filename, **spec) + spec = dict(correspondent=correspondent, title=title, + tags=tags) + filename = template.format(**spec) + self._test_guessed_attributes(filename, **spec) def test_created_and_correspondent_and_title_and_tags(self): template = ( - "/path/to/{created} - " + "{created} - " "{correspondent} - " "{title} - " - "{tags}" - ".{extension}" + "{tags}.pdf" ) for created in self.valid_dates: for correspondent in self.valid_correspondents: for title in self.valid_titles: for tags in self.valid_tags: - for extension in self.valid_extensions: - spec = { - "created": created, - "correspondent": correspondent, - "title": title, - "tags": tags, - "extension": extension - } - self._test_guessed_attributes( - template.format(**spec), **spec) + spec = { + "created": created, + "correspondent": correspondent, + "title": title, + "tags": tags, + } + self._test_guessed_attributes( + template.format(**spec), **spec) def test_created_and_correspondent_and_title(self): - template = "/path/to/{created} - {correspondent} - {title}.{extension}" + template = "{created} - {correspondent} - {title}.pdf" for created in self.valid_dates: for correspondent in self.valid_correspondents: @@ -279,56 +251,50 @@ class TestFieldPermutations(TestCase): if title.lower() == title: continue - for extension in self.valid_extensions: - spec = { - "created": created, - "correspondent": correspondent, - "title": title, - "extension": extension - } - self._test_guessed_attributes( - template.format(**spec), **spec) - - def test_created_and_title(self): - - template = "/path/to/{created} - {title}.{extension}" - - for created in self.valid_dates: - for title in self.valid_titles: - for extension in self.valid_extensions: spec = { "created": created, - "title": title, - "extension": extension + "correspondent": correspondent, + "title": title } self._test_guessed_attributes( template.format(**spec), **spec) + def test_created_and_title(self): + + template = "{created} - {title}.pdf" + + for created in self.valid_dates: + for title in self.valid_titles: + spec = { + "created": created, + "title": title + } + self._test_guessed_attributes( + template.format(**spec), **spec) + def test_created_and_title_and_tags(self): - template = "/path/to/{created} - {title} - {tags}.{extension}" + template = "{created} - {title} - {tags}.pdf" for created in self.valid_dates: for title in self.valid_titles: for tags in self.valid_tags: - for extension in self.valid_extensions: - spec = { - "created": created, - "title": title, - "tags": tags, - "extension": extension - } - self._test_guessed_attributes( - template.format(**spec), **spec) + spec = { + "created": created, + "title": title, + "tags": tags + } + self._test_guessed_attributes( + template.format(**spec), **spec) def test_invalid_date_format(self): - info = FileInfo.from_path("/path/to/06112017Z - title.pdf") + info = FileInfo.from_filename("06112017Z - title.pdf") self.assertEqual(info.title, "title") self.assertIsNone(info.created) def test_filename_parse_transforms(self): - path = "/some/path/to/tag1,tag2_20190908_180610_0001.pdf" + filename = "tag1,tag2_20190908_180610_0001.pdf" all_patt = re.compile("^.*$") none_patt = re.compile("$a") exact_patt = re.compile("^([a-z0-9,]+)_(\\d{8})_(\\d{6})_([0-9]+)\\.") @@ -336,50 +302,44 @@ class TestFieldPermutations(TestCase): repl2 = "\\2Z - " + repl1 # creation date + repl1 # No transformations configured (= default) - info = FileInfo.from_path(path) + info = FileInfo.from_filename(filename) self.assertEqual(info.title, "tag1,tag2_20190908_180610_0001") - self.assertEqual(info.extension, "pdf") self.assertEqual(info.tags, ()) self.assertIsNone(info.created) # Pattern doesn't match (filename unaltered) with self.settings( FILENAME_PARSE_TRANSFORMS=[(none_patt, "none.gif")]): - info = FileInfo.from_path(path) + info = FileInfo.from_filename(filename) self.assertEqual(info.title, "tag1,tag2_20190908_180610_0001") - self.assertEqual(info.extension, "pdf") # Simple transformation (match all) with self.settings( FILENAME_PARSE_TRANSFORMS=[(all_patt, "all.gif")]): - info = FileInfo.from_path(path) + info = FileInfo.from_filename(filename) self.assertEqual(info.title, "all") - self.assertEqual(info.extension, "gif") # Multiple transformations configured (first pattern matches) with self.settings( FILENAME_PARSE_TRANSFORMS=[ (all_patt, "all.gif"), (all_patt, "anotherall.gif")]): - info = FileInfo.from_path(path) + info = FileInfo.from_filename(filename) self.assertEqual(info.title, "all") - self.assertEqual(info.extension, "gif") # Multiple transformations configured (second pattern matches) with self.settings( FILENAME_PARSE_TRANSFORMS=[ (none_patt, "none.gif"), (all_patt, "anotherall.gif")]): - info = FileInfo.from_path(path) + info = FileInfo.from_filename(filename) self.assertEqual(info.title, "anotherall") - self.assertEqual(info.extension, "gif") # Complex transformation without date in replacement string with self.settings( FILENAME_PARSE_TRANSFORMS=[(exact_patt, repl1)]): - info = FileInfo.from_path(path) + info = FileInfo.from_filename(filename) self.assertEqual(info.title, "0001") - self.assertEqual(info.extension, "pdf") self.assertEqual(len(info.tags), 2) self.assertEqual(info.tags[0].slug, "tag1") self.assertEqual(info.tags[1].slug, "tag2") @@ -392,9 +352,8 @@ class TestFieldPermutations(TestCase): (exact_patt, repl2), # <-- matches (exact_patt, repl1), (all_patt, "all.gif")]): - info = FileInfo.from_path(path) + info = FileInfo.from_filename(filename) self.assertEqual(info.title, "0001") - self.assertEqual(info.extension, "pdf") self.assertEqual(len(info.tags), 2) self.assertEqual(info.tags[0].slug, "tag1") self.assertEqual(info.tags[1].slug, "tag2") From 09acb134b77fd15cf95c6d7416013811314ed8da Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Fri, 20 Nov 2020 18:14:42 +0100 Subject: [PATCH 09/52] updated mail: now uses mime type detection --- src/paperless_mail/mail.py | 22 +++++- src/paperless_mail/tests/test_mail.py | 96 ++++++++++++++++++--------- 2 files changed, 85 insertions(+), 33 deletions(-) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 1aea65d90..6db5e9070 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -2,6 +2,7 @@ import os import tempfile from datetime import timedelta, date +import magic from django.conf import settings from django.utils.text import slugify from django_q.tasks import async_task @@ -248,9 +249,21 @@ class MailAccountHandler(LoggingMixin): for att in message.attachments: + if not att.content_disposition == "attachment": + self.log( + 'debug', + f"Rule {rule.account}.{rule}: " + f"Skipping attachment {att.filename} " + f"with content disposition inline") + continue + title = get_title(message, att, rule) - if is_mime_type_supported(att.content_type): + # don't trust the content type of the attachment. Could be + # generic application/octet-stream. + mime_type = magic.from_buffer(att.payload, mime=True) + + if is_mime_type_supported(mime_type): os.makedirs(settings.SCRATCH_DIR, exist_ok=True) _, temp_filename = tempfile.mkstemp(prefix="paperless-mail-", dir=settings.SCRATCH_DIR) @@ -275,5 +288,12 @@ class MailAccountHandler(LoggingMixin): ) processed_attachments += 1 + else: + self.log( + 'debug', + f"Rule {rule.account}.{rule}: " + f"Skipping attachment {att.filename} " + f"since guessed mime type {mime_type} is not supported " + f"by paperless") return processed_attachments diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index a3404b774..17d7119a0 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -99,11 +99,7 @@ def create_message(num_attachments=1, body="", subject="the suject", from_="noon message.from_ = from_ message.body = body for i in range(num_attachments): - attachment = namedtuple('Attachment', []) - attachment.filename = 'some_file.pdf' - attachment.content_type = 'application/pdf' - attachment.payload = b'content of the attachment' - message.attachments.append(attachment) + message.attachments.append(create_attachment(filename=f"file_{i}.pdf")) message.seen = seen message.flagged = flagged @@ -111,6 +107,26 @@ def create_message(num_attachments=1, body="", subject="the suject", from_="noon return message +def create_attachment(filename="the_file.pdf", content_disposition="attachment", payload=b"a PDF document"): + attachment = namedtuple('Attachment', []) + attachment.filename = filename + attachment.content_disposition = content_disposition + attachment.payload = payload + return attachment + + +def fake_magic_from_buffer(buffer, mime=False): + + if mime: + if 'PDF' in str(buffer): + return 'application/pdf' + else: + return 'unknown/type' + else: + return 'Some verbose file description' + + +@mock.patch('paperless_mail.mail.magic.from_buffer', fake_magic_from_buffer) class TestMail(TestCase): def setUp(self): @@ -182,26 +198,7 @@ class TestMail(TestCase): self.assertEqual(get_title(message, att, rule), "the message title") def test_handle_message(self): - message = namedtuple('MailMessage', []) - message.subject = "the message title" - message.from_ = "Myself" - - att = namedtuple('Attachment', []) - att.filename = "test1.pdf" - att.content_type = 'application/pdf' - att.payload = b"attachment contents" - - att2 = namedtuple('Attachment', []) - att2.filename = "test2.pdf" - att2.content_type = 'application/pdf' - att2.payload = b"attachment contents" - - att3 = namedtuple('Attachment', []) - att3.filename = "test3.pdf" - att3.content_type = 'application/invalid' - att3.payload = b"attachment contents" - - message.attachments = [att, att2, att3] + message = create_message(subject="the message title", from_="Myself", num_attachments=2) account = MailAccount() rule = MailRule(assign_title_from=MailRule.TITLE_FROM_FILENAME, account=account) @@ -215,14 +212,13 @@ class TestMail(TestCase): args1, kwargs1 = self.async_task.call_args_list[0] args2, kwargs2 = self.async_task.call_args_list[1] - self.assertEqual(kwargs1['override_title'], "test1") - self.assertEqual(kwargs1['override_filename'], "test1.pdf") + self.assertEqual(kwargs1['override_title'], "file_0") + self.assertEqual(kwargs1['override_filename'], "file_0.pdf") - self.assertEqual(kwargs2['override_title'], "test2") - self.assertEqual(kwargs2['override_filename'], "test2.pdf") + self.assertEqual(kwargs2['override_title'], "file_1") + self.assertEqual(kwargs2['override_filename'], "file_1.pdf") - @mock.patch("paperless_mail.mail.async_task") - def test_handle_empty_message(self, m): + def test_handle_empty_message(self): message = namedtuple('MailMessage', []) message.attachments = [] @@ -230,9 +226,45 @@ class TestMail(TestCase): result = self.mail_account_handler.handle_message(message, rule) - self.assertFalse(m.called) + self.assertFalse(self.async_task.called) self.assertEqual(result, 0) + def test_handle_unknown_mime_type(self): + message = create_message() + message.attachments = [ + create_attachment(filename="f1.pdf"), + create_attachment(filename="f2.json", payload=b"{'much': 'payload.', 'so': 'json', 'wow': true}") + ] + + account = MailAccount() + rule = MailRule(assign_title_from=MailRule.TITLE_FROM_FILENAME, account=account) + + result = self.mail_account_handler.handle_message(message, rule) + + self.assertEqual(result, 1) + self.assertEqual(self.async_task.call_count, 1) + + args, kwargs = self.async_task.call_args + self.assertEqual(kwargs['override_filename'], "f1.pdf") + + def test_handle_disposition(self): + message = create_message() + message.attachments = [ + create_attachment(filename="f1.pdf", content_disposition='inline'), + create_attachment(filename="f2.pdf", content_disposition='attachment') + ] + + account = MailAccount() + rule = MailRule(assign_title_from=MailRule.TITLE_FROM_FILENAME, account=account) + + result = self.mail_account_handler.handle_message(message, rule) + + self.assertEqual(result, 1) + self.assertEqual(self.async_task.call_count, 1) + + args, kwargs = self.async_task.call_args + self.assertEqual(kwargs['override_filename'], "f2.pdf") + def test_handle_mail_account_mark_read(self): account = MailAccount.objects.create(name="test", imap_server="", username="admin", password="secret") From 321adb5df25161ed5d258d1c5aafef0fb26d4520 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Fri, 20 Nov 2020 18:45:37 +0100 Subject: [PATCH 10/52] making the migration reversible --- src/documents/migrations/1003_mime_types.py | 29 ++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/documents/migrations/1003_mime_types.py b/src/documents/migrations/1003_mime_types.py index 4c73a4235..1038d57b3 100644 --- a/src/documents/migrations/1003_mime_types.py +++ b/src/documents/migrations/1003_mime_types.py @@ -1,4 +1,5 @@ # Generated by Django 3.1.3 on 2020-11-20 11:21 +import mimetypes import os import magic @@ -29,6 +30,15 @@ def add_mime_types(apps, schema_editor): d.save() +def add_file_extensions(apps, schema_editor): + Document = apps.get_model("documents", "Document") + documents = Document.objects.all() + + for d in documents: + d.file_type = os.path.splitext(d.filename)[1].strip('.') + d.save() + + class Migration(migrations.Migration): dependencies = [ @@ -42,7 +52,24 @@ class Migration(migrations.Migration): field=models.CharField(default="-", editable=False, max_length=256), preserve_default=False, ), - migrations.RunPython(add_mime_types), + migrations.RunPython(add_mime_types, migrations.RunPython.noop), + + # This operation is here so that we can revert the entire migration: + # By allowing this field to be blank and null, we can revert the + # remove operation further down and the database won't complain about + # NOT NULL violations. + migrations.AlterField( + model_name='document', + name='file_type', + field=models.CharField( + choices=[('pdf', 'PDF'), ('png', 'PNG'), ('jpg', 'JPG'), ('gif', 'GIF'), ('tiff', 'TIFF'), ('txt', 'TXT'), ('csv', 'CSV'), ('md', 'MD')], + editable=False, + max_length=4, + null=True, + blank=True + ), + ), + migrations.RunPython(migrations.RunPython.noop, add_file_extensions), migrations.RemoveField( model_name='document', name='file_type', From 77559332bc543216d0f1275c496304cab172f8a2 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Fri, 20 Nov 2020 18:45:44 +0100 Subject: [PATCH 11/52] docs --- docs/faq.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 747ffaf53..6cfa4d36f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -21,6 +21,17 @@ is files around manually. This folder is meant to be entirely managed by docker and paperless. +**Q:** *What file types does paperless-ng support?* + +**A:** Currently, the following files are supported: + +* PDF documents, PNG images and JPEG images are processed with OCR. +* Plain text documents are supported as well and are added verbatim + to paperless. + +Paperless determines the type of a file by inspecting its content. The +file extensions do not matter. + **Q:** *Will paperless-ng run on Raspberry Pi?* **A:** The short answer is yes. I've tested it on a Raspberry Pi 3 B. From b7fec4d3551cb95b84eb7de5b555f4ae8370c022 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 01:42:55 +0100 Subject: [PATCH 12/52] using mime type checking during upload --- src/documents/forms.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/documents/forms.py b/src/documents/forms.py index c3efc774f..f44090164 100644 --- a/src/documents/forms.py +++ b/src/documents/forms.py @@ -3,22 +3,35 @@ import tempfile from datetime import datetime from time import mktime +import magic from django import forms from django.conf import settings from django_q.tasks import async_task from pathvalidate import validate_filename, ValidationError +from documents.parsers import is_mime_type_supported + class UploadForm(forms.Form): document = forms.FileField() def clean_document(self): + document_name = self.cleaned_data.get("document").name + try: - validate_filename(self.cleaned_data.get("document").name) + validate_filename(document_name) except ValidationError: raise forms.ValidationError("That filename is suspicious.") - return self.cleaned_data.get("document") + + document_data = self.cleaned_data.get("document").read() + + mime_type = magic.from_buffer(document_data, mime=True) + + if not is_mime_type_supported(mime_type): + raise forms.ValidationError("This mime type is not supported.") + + return document_name, document_data def save(self): """ @@ -27,8 +40,7 @@ class UploadForm(forms.Form): form do that as well. Think of it as a poor-man's queue server. """ - document = self.cleaned_data.get("document").read() - original_filename = self.cleaned_data.get("document").name + original_filename, data = self.cleaned_data.get("document") t = int(mktime(datetime.now().timetuple())) @@ -36,7 +48,7 @@ class UploadForm(forms.Form): with tempfile.NamedTemporaryFile(prefix="paperless-upload-", dir=settings.SCRATCH_DIR, delete=False) as f: - f.write(document) + f.write(data) os.utime(f.name, times=(t, t)) async_task("documents.tasks.consume_file", f.name, override_filename=original_filename, task_name=os.path.basename(original_filename)) From 529cc04fd1712105b11796117d821500da57a44d Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 12:12:19 +0100 Subject: [PATCH 13/52] code cleanup --- src/documents/models.py | 1 + src/documents/tests/test_parsers.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/documents/models.py b/src/documents/models.py index 6288980c5..8e0435647 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -269,6 +269,7 @@ class Log(models.Model): def __str__(self): return self.message + # TODO: why is this in the models file? class FileInfo: diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index e99bb8dc6..239203186 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -6,6 +6,7 @@ from django.test import TestCase from documents.parsers import get_parser_class + def fake_magic_from_file(file, mime=False): if mime: From 5a84cc835a1a23f857c7c38b883b44971173f7e8 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 13:05:17 +0100 Subject: [PATCH 14/52] updated release script --- docs/changelog.rst | 13 +++++++++++++ scripts/make-release.sh | 9 ++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0da528b60..86a24df27 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,19 @@ Changelog ********* +next +#### + +* Paperless now uses mime types and libmagic detection to determine + if a file type is supported and which parser to use. Removes all + file type checks that where present in MANY different places in + paperless. + +* Mail consumer now correctly consumes documents even when their + content type was not set correctly. (i.e. PDF documents with + content type ``application/octet-stream``) + + paperless-ng 0.9.1 ################## diff --git a/scripts/make-release.sh b/scripts/make-release.sh index ef3e5769b..06548748b 100755 --- a/scripts/make-release.sh +++ b/scripts/make-release.sh @@ -17,6 +17,7 @@ PAPERLESS_ROOT=$(git rev-parse --show-toplevel) # output directory PAPERLESS_DIST="$PAPERLESS_ROOT/dist" PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" +PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles" if [ -d "$PAPERLESS_DIST" ] then @@ -27,6 +28,7 @@ fi mkdir "$PAPERLESS_DIST" mkdir "$PAPERLESS_DIST_APP" mkdir "$PAPERLESS_DIST_APP/docker" +mkdir "$PAPERLESS_DIST_DOCKERFILES" # setup dependencies. @@ -78,9 +80,9 @@ cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP" cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP" # docker files for pulling from docker hub -cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST" -cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST" +cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES" +cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES" +cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES" # auxiliary files required for the docker image cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/" @@ -99,3 +101,4 @@ docker build . -t "jonaswinkler/paperless-ng:$VERSION" cd "$PAPERLESS_DIST" tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/ +tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/ From b44f8383e447089628673a7b222e0b6c1a9b5c15 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 14:03:45 +0100 Subject: [PATCH 15/52] code cleanup --- src/documents/consumer.py | 14 +++-- src/documents/file_handling.py | 4 +- src/documents/forms.py | 9 ++- src/documents/index.py | 3 +- .../management/commands/document_consumer.py | 15 +++-- .../management/commands/document_exporter.py | 4 +- .../management/commands/document_importer.py | 2 +- .../management/commands/document_retagger.py | 6 +- src/documents/matching.py | 18 ++++-- src/documents/parsers.py | 30 +++++++--- src/documents/serialisers.py | 6 +- src/documents/signals/handlers.py | 60 +++++++++++++------ src/documents/tests/test_checks.py | 8 --- src/documents/views.py | 41 ++++++++++--- src/paperless_mail/mail.py | 14 +++-- src/paperless_mail/models.py | 12 ++-- src/paperless_mail/tasks.py | 3 +- src/paperless_tesseract/parsers.py | 60 ++++++++++++------- 18 files changed, 208 insertions(+), 101 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 175f6710f..65febc937 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -104,9 +104,11 @@ class Consumer(LoggingMixin): parser_class = get_parser_class_for_mime_type(mime_type) if not parser_class: - raise ConsumerError("No parsers abvailable for {}".format(self.filename)) + raise ConsumerError(f"No parsers abvailable for {self.filename}") else: - self.log("debug", "Parser: {} based on mime type {}".format(parser_class.__name__, mime_type)) + self.log("debug", + f"Parser: {parser_class.__name__} " + f"based on mime type {mime_type}") # Notify all listeners that we're going to do some work. @@ -126,7 +128,7 @@ class Consumer(LoggingMixin): # Parse the document. This may take some time. try: - self.log("debug", "Generating thumbnail for {}...".format(self.filename)) + self.log("debug", f"Generating thumbnail for {self.filename}...") thumbnail = document_parser.get_optimised_thumbnail() self.log("debug", "Parsing {}...".format(self.filename)) text = document_parser.get_text() @@ -244,10 +246,12 @@ class Consumer(LoggingMixin): document.title = self.override_title if self.override_correspondent_id: - document.correspondent = Correspondent.objects.get(pk=self.override_correspondent_id) + document.correspondent = Correspondent.objects.get( + pk=self.override_correspondent_id) if self.override_document_type_id: - document.document_type = DocumentType.objects.get(pk=self.override_document_type_id) + document.document_type = DocumentType.objects.get( + pk=self.override_document_type_id) if self.override_tag_ids: for tag_id in self.override_tag_ids: diff --git a/src/documents/file_handling.py b/src/documents/file_handling.py index 06d4d2957..cd47406b6 100644 --- a/src/documents/file_handling.py +++ b/src/documents/file_handling.py @@ -87,7 +87,9 @@ def generate_filename(document): tags=tags, ) except (ValueError, KeyError, IndexError): - logging.getLogger(__name__).warning("Invalid PAPERLESS_FILENAME_FORMAT: {}, falling back to default,".format(settings.PAPERLESS_FILENAME_FORMAT)) + logging.getLogger(__name__).warning( + f"Invalid PAPERLESS_FILENAME_FORMAT: " + f"{settings.PAPERLESS_FILENAME_FORMAT}, falling back to default") # Always append the primary key to guarantee uniqueness of filename if len(path) > 0: diff --git a/src/documents/forms.py b/src/documents/forms.py index f44090164..0471a8312 100644 --- a/src/documents/forms.py +++ b/src/documents/forms.py @@ -46,9 +46,14 @@ class UploadForm(forms.Form): os.makedirs(settings.SCRATCH_DIR, exist_ok=True) - with tempfile.NamedTemporaryFile(prefix="paperless-upload-", dir=settings.SCRATCH_DIR, delete=False) as f: + with tempfile.NamedTemporaryFile(prefix="paperless-upload-", + dir=settings.SCRATCH_DIR, + delete=False) as f: f.write(data) os.utime(f.name, times=(t, t)) - async_task("documents.tasks.consume_file", f.name, override_filename=original_filename, task_name=os.path.basename(original_filename)) + async_task("documents.tasks.consume_file", + f.name, + override_filename=original_filename, + task_name=os.path.basename(original_filename)) diff --git a/src/documents/index.py b/src/documents/index.py index ad3a50010..cf312cbcc 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -120,6 +120,7 @@ def query_page(ix, query, page): def autocomplete(ix, term, limit=10): with ix.reader() as reader: terms = [] - for (score, t) in reader.most_distinctive_terms("content", limit, term.lower()): + for (score, t) in reader.most_distinctive_terms( + "content", number=limit, prefix=term.lower()): terms.append(t) return terms diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index 2b8ac7100..70c36a03c 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -19,10 +19,13 @@ class Handler(FileSystemEventHandler): def _consume(self, file): if os.path.isfile(file): try: - async_task("documents.tasks.consume_file", file, task_name=os.path.basename(file)) + async_task("documents.tasks.consume_file", + file, + task_name=os.path.basename(file)) except Exception as e: # Catch all so that the consumer won't crash. - logging.getLogger(__name__).error("Error while consuming document: {}".format(e)) + logging.getLogger(__name__).error( + "Error while consuming document: {}".format(e)) def on_created(self, event): self._consume(event.src_path) @@ -66,12 +69,14 @@ class Command(BaseCommand): # Consume all files as this is not done initially by the watchdog for entry in os.scandir(directory): if entry.is_file(): - async_task("documents.tasks.consume_file", entry.path, task_name=os.path.basename(entry.path)) + async_task("documents.tasks.consume_file", + entry.path, + task_name=os.path.basename(entry.path)) # Start the watchdog. Woof! if settings.CONSUMER_POLLING > 0: - logging.getLogger(__name__).info('Using polling instead of file' - 'system notifications.') + logging.getLogger(__name__).info( + "Using polling instead of file system notifications.") observer = PollingObserver(timeout=settings.CONSUMER_POLLING) else: observer = Observer() diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index 441f1c475..f86462119 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -63,7 +63,7 @@ class Command(Renderable, BaseCommand): document = document_map[document_dict["pk"]] - unique_filename = "{:07}_{}".format(document.pk, document.file_name) + unique_filename = f"{document.pk:07}_{document.file_name}" file_target = os.path.join(self.target, unique_filename) @@ -73,7 +73,7 @@ class Command(Renderable, BaseCommand): document_dict[EXPORTER_FILE_NAME] = unique_filename document_dict[EXPORTER_THUMBNAIL_NAME] = thumbnail_name - print("Exporting: {}".format(file_target)) + print(f"Exporting: {file_target}") t = int(time.mktime(document.created.timetuple())) if document.storage_type == Document.STORAGE_TYPE_GPG: diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index da9086144..208a0ef37 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -120,7 +120,7 @@ class Command(Renderable, BaseCommand): encrypted.write(GnuPG.encrypted(unencrypted)) else: - print("Moving {} to {}".format(document_path, document.source_path)) + print(f"Moving {document_path} to {document.source_path}") shutil.copy(document_path, document.source_path) shutil.copy(thumbnail_path, document.thumbnail_path) diff --git a/src/documents/management/commands/document_retagger.py b/src/documents/management/commands/document_retagger.py index e48b8802c..cf014dc6f 100755 --- a/src/documents/management/commands/document_retagger.py +++ b/src/documents/management/commands/document_retagger.py @@ -74,13 +74,13 @@ class Command(Renderable, BaseCommand): try: classifier.reload() except (FileNotFoundError, IncompatibleClassifierVersionError) as e: - logging.getLogger(__name__).warning("Cannot classify documents: {}.".format(e)) + logging.getLogger(__name__).warning( + f"Cannot classify documents: {e}.") classifier = None for document in documents: logging.getLogger(__name__).info( - "Processing document {}".format(document.title) - ) + f"Processing document {document.title}") if options['correspondent']: set_correspondent( diff --git a/src/documents/matching.py b/src/documents/matching.py index e5789ab2e..ae1a9a9cf 100644 --- a/src/documents/matching.py +++ b/src/documents/matching.py @@ -6,17 +6,23 @@ from documents.models import MatchingModel, Correspondent, DocumentType, Tag def match_correspondents(document_content, classifier): - correspondents = Correspondent.objects.all() - predicted_correspondent_id = classifier.predict_correspondent(document_content) if classifier else None + if classifier: + pred_id = classifier.predict_correspondent(document_content) + else: + pred_id = None - return [o for o in correspondents if matches(o, document_content) or o.pk == predicted_correspondent_id] + correspondents = Correspondent.objects.all() + return [o for o in correspondents if matches(o, document_content) or o.pk == pred_id] def match_document_types(document_content, classifier): - document_types = DocumentType.objects.all() - predicted_document_type_id = classifier.predict_document_type(document_content) if classifier else None + if classifier: + pred_id = classifier.predict_document_type(document_content) + else: + pred_id = None - return [o for o in document_types if matches(o, document_content) or o.pk == predicted_document_type_id] + document_types = DocumentType.objects.all() + return [o for o in document_types if matches(o, document_content) or o.pk == pred_id] def match_tags(document_content, classifier): diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 98f4c5b12..eb8ccf45e 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -73,7 +73,18 @@ def get_parser_class(path): return get_parser_class_for_mime_type(mime_type) -def run_convert(input_file, output_file, density=None, scale=None, alpha=None, strip=False, trim=False, type=None, depth=None, extra=None, logging_group=None): +def run_convert(input_file, + output_file, + density=None, + scale=None, + alpha=None, + strip=False, + trim=False, + type=None, + depth=None, + extra=None, + logging_group=None): + environment = os.environ.copy() if settings.CONVERT_MEMORY_LIMIT: environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT @@ -102,10 +113,13 @@ def run_unpaper(pnm, logging_group=None): command_args = (settings.UNPAPER_BINARY, "--overwrite", "--quiet", pnm, pnm_out) - logger.debug("Execute: " + " ".join(command_args), extra={'group': logging_group}) + logger.debug(f"Execute: {' '.join(command_args)}", + extra={'group': logging_group}) - if not subprocess.Popen(command_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).wait() == 0: - raise ParseError("Unpaper failed at {}".format(command_args)) + if not subprocess.Popen(command_args, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).wait() == 0: + raise ParseError(f"Unpaper failed at {command_args}") return pnm_out @@ -124,7 +138,8 @@ class DocumentParser(LoggingMixin): super().__init__() self.logging_group = logging_group self.document_path = path - self.tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR) + self.tempdir = tempfile.mkdtemp( + prefix="paperless-", dir=settings.SCRATCH_DIR) def get_thumbnail(self): """ @@ -137,9 +152,10 @@ class DocumentParser(LoggingMixin): if settings.OPTIMIZE_THUMBNAILS: out_path = os.path.join(self.tempdir, "optipng.png") - args = (settings.OPTIPNG_BINARY, "-silent", "-o5", in_path, "-out", out_path) + args = (settings.OPTIPNG_BINARY, + "-silent", "-o5", in_path, "-out", out_path) - self.log('debug', 'Execute: ' + " ".join(args)) + self.log('debug', f"Execute: {' '.join(args)}") if not subprocess.Popen(args).wait() == 0: raise ParseError("Optipng failed at {}".format(args)) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index cf48e8bd7..e0ad73a23 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -76,9 +76,11 @@ class DocumentTypeField(serializers.PrimaryKeyRelatedField): class DocumentSerializer(serializers.ModelSerializer): - correspondent_id = CorrespondentField(allow_null=True, source='correspondent') + correspondent_id = CorrespondentField( + allow_null=True, source='correspondent') tags_id = TagsField(many=True, source='tags') - document_type_id = DocumentTypeField(allow_null=True, source='document_type') + document_type_id = DocumentTypeField( + allow_null=True, source='document_type') class Meta: model = Document diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 671cdb104..f83f88783 100755 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -25,11 +25,18 @@ def add_inbox_tags(sender, document=None, logging_group=None, **kwargs): document.tags.add(*inbox_tags) -def set_correspondent(sender, document=None, logging_group=None, classifier=None, replace=False, use_first=True, **kwargs): +def set_correspondent(sender, + document=None, + logging_group=None, + classifier=None, + replace=False, + use_first=True, + **kwargs): if document.correspondent and not replace: return - potential_correspondents = matching.match_correspondents(document.content, classifier) + potential_correspondents = matching.match_correspondents(document.content, + classifier) potential_count = len(potential_correspondents) if potential_correspondents: @@ -38,22 +45,22 @@ def set_correspondent(sender, document=None, logging_group=None, classifier=None selected = None if potential_count > 1: if use_first: - message = "Detected {} potential correspondents, so we've opted for {}" logger( - message.format(potential_count, selected), + f"Detected {potential_count} potential correspondents, " + f"so we've opted for {selected}", logging_group ) else: - message = "Detected {} potential correspondents, not assigning any correspondent" logger( - message.format(potential_count), + f"Detected {potential_count} potential correspondents, " + f"not assigning any correspondent", logging_group ) return if selected or replace: logger( - 'Assigning correspondent "{}" to "{}" '.format(selected, document), + f"Assigning correspondent {selected} to {document}", logging_group ) @@ -61,11 +68,18 @@ def set_correspondent(sender, document=None, logging_group=None, classifier=None document.save(update_fields=("correspondent",)) -def set_document_type(sender, document=None, logging_group=None, classifier=None, replace=False, use_first=True, **kwargs): +def set_document_type(sender, + document=None, + logging_group=None, + classifier=None, + replace=False, + use_first=True, + **kwargs): if document.document_type and not replace: return - potential_document_type = matching.match_document_types(document.content, classifier) + potential_document_type = matching.match_document_types(document.content, + classifier) potential_count = len(potential_document_type) if potential_document_type: @@ -75,22 +89,22 @@ def set_document_type(sender, document=None, logging_group=None, classifier=None if potential_count > 1: if use_first: - message = "Detected {} potential document types, so we've opted for {}" logger( - message.format(potential_count, selected), + f"Detected {potential_count} potential document types, " + f"so we've opted for {selected}", logging_group ) else: - message = "Detected {} potential document types, not assigning any document type" logger( - message.format(potential_count), + f"Detected {potential_count} potential document types, " + f"not assigning any document type", logging_group ) return if selected or replace: logger( - 'Assigning document type "{}" to "{}" '.format(selected, document), + f"Assigning document type {selected} to {document}", logging_group ) @@ -98,14 +112,21 @@ def set_document_type(sender, document=None, logging_group=None, classifier=None document.save(update_fields=("document_type",)) -def set_tags(sender, document=None, logging_group=None, classifier=None, replace=False, **kwargs): +def set_tags(sender, + document=None, + logging_group=None, + classifier=None, + replace=False, + **kwargs): if replace: document.tags.clear() current_tags = set([]) else: current_tags = set(document.tags.all()) - relevant_tags = set(matching.match_tags(document.content, classifier)) - current_tags + matched_tags = matching.match_tags(document.content, classifier) + + relevant_tags = set(matched_tags) - current_tags if not relevant_tags: return @@ -180,12 +201,15 @@ def update_filename_and_move_files(sender, instance, **kwargs): if not os.path.isfile(old_path): # Can't do anything if the old file does not exist anymore. - logging.getLogger(__name__).fatal('Document {}: File {} has gone.'.format(str(instance), old_path)) + logging.getLogger(__name__).fatal( + f"Document {str(instance)}: File {old_path} has gone.") return if os.path.isfile(new_path): # Can't do anything if the new file already exists. Skip updating file. - logging.getLogger(__name__).warning('Document {}: Cannot rename file since target path {} already exists.'.format(str(instance), new_path)) + logging.getLogger(__name__).warning( + f"Document {str(instance)}: Cannot rename file " + f"since target path {new_path} already exists.") return create_source_path_directory(new_path) diff --git a/src/documents/tests/test_checks.py b/src/documents/tests/test_checks.py index d316f94b5..1027c11a0 100644 --- a/src/documents/tests/test_checks.py +++ b/src/documents/tests/test_checks.py @@ -15,11 +15,3 @@ class ChecksTestCase(TestCase): def test_changed_password_check_no_encryption(self): DocumentFactory.create(storage_type=Document.STORAGE_TYPE_UNENCRYPTED) self.assertEqual(changed_password_check(None), []) - - @unittest.skip("I don't know how to test this") - def test_changed_password_check_gpg_encryption_with_good_password(self): - pass - - @unittest.skip("I don't know how to test this") - def test_changed_password_check_fail(self): - pass diff --git a/src/documents/views.py b/src/documents/views.py index 89d03a4df..14323e933 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -47,18 +47,30 @@ class IndexView(TemplateView): class CorrespondentViewSet(ModelViewSet): model = Correspondent - queryset = Correspondent.objects.annotate(document_count=Count('documents'), last_correspondence=Max('documents__created')).order_by('name') + + queryset = Correspondent.objects.annotate( + document_count=Count('documents'), + last_correspondence=Max('documents__created')).order_by('name') + serializer_class = CorrespondentSerializer pagination_class = StandardPagination permission_classes = (IsAuthenticated,) filter_backends = (DjangoFilterBackend, OrderingFilter) filterset_class = CorrespondentFilterSet - ordering_fields = ("name", "matching_algorithm", "match", "document_count", "last_correspondence") + ordering_fields = ( + "name", + "matching_algorithm", + "match", + "document_count", + "last_correspondence") class TagViewSet(ModelViewSet): model = Tag - queryset = Tag.objects.annotate(document_count=Count('documents')).order_by('name') + + queryset = Tag.objects.annotate( + document_count=Count('documents')).order_by('name') + serializer_class = TagSerializer pagination_class = StandardPagination permission_classes = (IsAuthenticated,) @@ -69,7 +81,10 @@ class TagViewSet(ModelViewSet): class DocumentTypeViewSet(ModelViewSet): model = DocumentType - queryset = DocumentType.objects.annotate(document_count=Count('documents')).order_by('name') + + queryset = DocumentType.objects.annotate( + document_count=Count('documents')).order_by('name') + serializer_class = DocumentTypeSerializer pagination_class = StandardPagination permission_classes = (IsAuthenticated,) @@ -92,10 +107,18 @@ class DocumentViewSet(RetrieveModelMixin, filterset_class = DocumentFilterSet search_fields = ("title", "correspondent__name", "content") ordering_fields = ( - "id", "title", "correspondent__name", "document_type__name", "created", "modified", "added", "archive_serial_number") + "id", + "title", + "correspondent__name", + "document_type__name", + "created", + "modified", + "added", + "archive_serial_number") def update(self, request, *args, **kwargs): - response = super(DocumentViewSet, self).update(request, *args, **kwargs) + response = super(DocumentViewSet, self).update( + request, *args, **kwargs) index.add_or_update_document(self.get_object()) return response @@ -138,7 +161,8 @@ class DocumentViewSet(RetrieveModelMixin, @cache_control(public=False, max_age=315360000) def thumb(self, request, pk=None): try: - return HttpResponse(Document.objects.get(id=pk).thumbnail_file, content_type='image/png') + return HttpResponse(Document.objects.get(id=pk).thumbnail_file, + content_type='image/png') except FileNotFoundError: raise Http404("Document thumbnail does not exist") @@ -230,5 +254,6 @@ class StatisticsView(APIView): def get(self, request, format=None): return Response({ 'documents_total': Document.objects.all().count(), - 'documents_inbox': Document.objects.filter(tags__is_inbox_tag=True).distinct().count() + 'documents_inbox': Document.objects.filter( + tags__is_inbox_tag=True).distinct().count() }) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 6db5e9070..03f915769 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -174,8 +174,8 @@ class MailAccountHandler(LoggingMixin): M.folder.set(rule.folder) except MailboxFolderSelectError: raise MailError( - f"Rule {rule.name}: Folder {rule.folder} does not exist " - f"in account {account.name}") + f"Rule {rule.name}: Folder {rule.folder} " + f"does not exist in account {account.name}") criterias = make_criterias(rule) @@ -185,7 +185,8 @@ class MailAccountHandler(LoggingMixin): f"{str(AND(**criterias))}") try: - messages = M.fetch(criteria=AND(**criterias), mark_seen=False) + messages = M.fetch(criteria=AND(**criterias), + mark_seen=False) except Exception: raise MailError( f"Rule {rule.name}: Error while fetching folder " @@ -226,8 +227,8 @@ class MailAccountHandler(LoggingMixin): except Exception: raise MailError( - f"Rule {rule.name}: Error while processing post-consume " - f"actions for account {account.name}") + f"Rule {rule.name}: Error while processing " + f"post-consume actions for account {account.name}") return total_processed_files @@ -266,7 +267,8 @@ class MailAccountHandler(LoggingMixin): if is_mime_type_supported(mime_type): os.makedirs(settings.SCRATCH_DIR, exist_ok=True) - _, temp_filename = tempfile.mkstemp(prefix="paperless-mail-", dir=settings.SCRATCH_DIR) + _, temp_filename = tempfile.mkstemp(prefix="paperless-mail-", + dir=settings.SCRATCH_DIR) with open(temp_filename, 'wb') as f: f.write(att.payload) diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index e37fbee16..14da202fa 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -66,10 +66,14 @@ class MailRule(models.Model): CORRESPONDENT_FROM_CUSTOM = 4 CORRESPONDENT_SELECTOR = ( - (CORRESPONDENT_FROM_NOTHING, "Do not assign a correspondent"), - (CORRESPONDENT_FROM_EMAIL, "Use mail address"), - (CORRESPONDENT_FROM_NAME, "Use name (or mail address if not available)"), - (CORRESPONDENT_FROM_CUSTOM, "Use correspondent selected below") + (CORRESPONDENT_FROM_NOTHING, + "Do not assign a correspondent"), + (CORRESPONDENT_FROM_EMAIL, + "Use mail address"), + (CORRESPONDENT_FROM_NAME, + "Use name (or mail address if not available)"), + (CORRESPONDENT_FROM_CUSTOM, + "Use correspondent selected below") ) name = models.CharField(max_length=256, unique=True) diff --git a/src/paperless_mail/tasks.py b/src/paperless_mail/tasks.py index 22d512c1e..e75711dce 100644 --- a/src/paperless_mail/tasks.py +++ b/src/paperless_mail/tasks.py @@ -7,7 +7,8 @@ from paperless_mail.models import MailAccount def process_mail_accounts(): total_new_documents = 0 for account in MailAccount.objects.all(): - total_new_documents += MailAccountHandler().handle_mail_account(account) + total_new_documents += MailAccountHandler().handle_mail_account( + account) if total_new_documents > 0: return f"Added {total_new_documents} document(s)." diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index 73b2414d5..d0ce01327 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -50,7 +50,10 @@ class RasterisedDocumentParser(DocumentParser): except ParseError: # if convert fails, fall back to extracting # the first PDF page as a PNG using Ghostscript - self.log('warning', 'Thumbnail generation with ImageMagick failed, falling back to ghostscript. Check your /etc/ImageMagick-x/policy.xml!') + self.log( + 'warning', + "Thumbnail generation with ImageMagick failed, falling back " + "to ghostscript. Check your /etc/ImageMagick-x/policy.xml!") gs_out_path = os.path.join(self.tempdir, "gs_out.png") cmd = [settings.GS_BINARY, "-q", @@ -98,24 +101,38 @@ class RasterisedDocumentParser(DocumentParser): try: sample_page_index = int(len(images) / 2) - self.log("debug", "Attempting language detection on page {} of {}...".format(sample_page_index + 1, len(images))) - sample_page_text = self._ocr([images[sample_page_index]], settings.OCR_LANGUAGE)[0] + self.log( + "debug", + f"Attempting language detection on page " + f"{sample_page_index + 1} of {len(images)}...") + + sample_page_text = self._ocr([images[sample_page_index]], + settings.OCR_LANGUAGE)[0] guessed_language = self._guess_language(sample_page_text) if not guessed_language or guessed_language not in ISO639: self.log("warning", "Language detection failed.") - ocr_pages = self._complete_ocr_default_language(images, sample_page_index, sample_page_text) + ocr_pages = self._complete_ocr_default_language( + images, sample_page_index, sample_page_text) elif ISO639[guessed_language] == settings.OCR_LANGUAGE: - self.log("debug", "Detected language: {} (default language)".format(guessed_language)) - ocr_pages = self._complete_ocr_default_language(images, sample_page_index, sample_page_text) + self.log( + "debug", + f"Detected language: {guessed_language} " + f"(default language)") + ocr_pages = self._complete_ocr_default_language( + images, sample_page_index, sample_page_text) elif not ISO639[guessed_language] in pyocr.get_available_tools()[0].get_available_languages(): - self.log("warning", "Detected language {} is not available on this system.".format(guessed_language)) - ocr_pages = self._complete_ocr_default_language(images, sample_page_index, sample_page_text) + self.log( + "warning", + f"Detected language {guessed_language} is not available " + f"on this system.") + ocr_pages = self._complete_ocr_default_language( + images, sample_page_index, sample_page_text) else: - self.log("debug", "Detected language: {}".format(guessed_language)) + self.log("debug", f"Detected language: {guessed_language}") ocr_pages = self._ocr(images, ISO639[guessed_language]) self.log("debug", "OCR completed.") @@ -130,7 +147,9 @@ class RasterisedDocumentParser(DocumentParser): Greyscale images are easier for Tesseract to OCR """ - self.log("debug", "Converting document {} into greyscale images...".format(self.document_path)) + self.log( + "debug", + f"Converting document {self.document_path} into greyscale images") # Convert PDF to multiple PNMs pnm = os.path.join(self.tempdir, "convert-%04d.pnm") @@ -148,7 +167,7 @@ class RasterisedDocumentParser(DocumentParser): if f.endswith(".pnm"): pnms.append(os.path.join(self.tempdir, f)) - self.log("debug", "Running unpaper on {} pages...".format(len(pnms))) + self.log("debug", f"Running unpaper on {len(pnms)} pages...") # Run unpaper in parallel on converted images with ThreadPool(processes=settings.THREADS_PER_WORKER) as pool: @@ -161,26 +180,25 @@ class RasterisedDocumentParser(DocumentParser): guess = langdetect.detect(text) return guess except Exception as e: - self.log('warning', "Language detection failed with: {}".format(e)) + self.log('warning', f"Language detection failed with: {e}") return None def _ocr(self, imgs, lang): - self.log("debug", "Performing OCR on {} page(s) with language {}".format(len(imgs), lang)) + self.log( + "debug", + f"Performing OCR on {len(imgs)} page(s) with language {lang}") with ThreadPool(processes=settings.THREADS_PER_WORKER) as pool: r = pool.map(image_to_string, itertools.product(imgs, [lang])) return r - def _complete_ocr_default_language(self, images, sample_page_index, sample_page): - """ - Given a `middle` value and the text that middle page represents, we OCR - the remainder of the document and return the whole thing. - """ - # text = self._ocr(imgs[:middle], settings.OCR_LANGUAGE) + text - # text += self._ocr(imgs[middle + 1:], settings.OCR_LANGUAGE) + def _complete_ocr_default_language(self, + images, + sample_page_index, + sample_page): images_copy = list(images) del images_copy[sample_page_index] if images_copy: - self.log('debug', 'Continuing ocr with default language.') + self.log('debug', "Continuing ocr with default language.") ocr_pages = self._ocr(images_copy, settings.OCR_LANGUAGE) ocr_pages.insert(sample_page_index, sample_page) return ocr_pages From 450fb877f6214202240cd7429c2c94c0ed26562b Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 15:34:00 +0100 Subject: [PATCH 16/52] code cleanup --- src/documents/classifier.py | 30 +++++++++++------- src/documents/file_handling.py | 31 +++++++++--------- src/documents/matching.py | 51 ++++++++++++++++++++---------- src/paperless/auth.py | 2 +- src/paperless_mail/mail.py | 4 +-- src/paperless_tesseract/parsers.py | 2 +- 6 files changed, 71 insertions(+), 49 deletions(-) diff --git a/src/documents/classifier.py b/src/documents/classifier.py index 1b70dcd6f..6e0d6f946 100755 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -30,10 +30,12 @@ class DocumentClassifier(object): FORMAT_VERSION = 5 def __init__(self): - # mtime of the model file on disk. used to prevent reloading when nothing has changed. + # mtime of the model file on disk. used to prevent reloading when + # nothing has changed. self.classifier_version = 0 - # hash of the training data. used to prevent re-training when the training data has not changed. + # hash of the training data. used to prevent re-training when the + # training data has not changed. self.data_hash = None self.data_vectorizer = None @@ -48,10 +50,12 @@ class DocumentClassifier(object): schema_version = pickle.load(f) if schema_version != self.FORMAT_VERSION: - raise IncompatibleClassifierVersionError("Cannor load classifier, incompatible versions.") + raise IncompatibleClassifierVersionError( + "Cannor load classifier, incompatible versions.") else: if self.classifier_version > 0: - logger.info("Classifier updated on disk, reloading classifier models") + logger.info("Classifier updated on disk, " + "reloading classifier models") self.data_hash = pickle.load(f) self.data_vectorizer = pickle.load(f) self.tags_binarizer = pickle.load(f) @@ -82,20 +86,22 @@ class DocumentClassifier(object): # Step 1: Extract and preprocess training data from the database. logging.getLogger(__name__).debug("Gathering data from database...") m = hashlib.sha1() - for doc in Document.objects.order_by('pk').exclude(tags__is_inbox_tag=True): + for doc in Document.objects.order_by('pk').exclude(tags__is_inbox_tag=True): # NOQA: E501 preprocessed_content = preprocess_content(doc.content) m.update(preprocessed_content.encode('utf-8')) data.append(preprocessed_content) y = -1 - if doc.document_type and doc.document_type.matching_algorithm == MatchingModel.MATCH_AUTO: - y = doc.document_type.pk + dt = doc.document_type + if dt and dt.matching_algorithm == MatchingModel.MATCH_AUTO: + y = dt.pk m.update(y.to_bytes(4, 'little', signed=True)) labels_document_type.append(y) y = -1 - if doc.correspondent and doc.correspondent.matching_algorithm == MatchingModel.MATCH_AUTO: - y = doc.correspondent.pk + cor = doc.correspondent + if cor and cor.matching_algorithm == MatchingModel.MATCH_AUTO: + y = cor.pk m.update(y.to_bytes(4, 'little', signed=True)) labels_correspondent.append(y) @@ -145,7 +151,7 @@ class DocumentClassifier(object): # Step 3: train the classifiers if num_tags > 0: logging.getLogger(__name__).debug("Training tags classifier...") - self.tags_classifier = MLPClassifier(verbose=True, tol=0.01) + self.tags_classifier = MLPClassifier(tol=0.01) self.tags_classifier.fit(data_vectorized, labels_tags_vectorized) else: self.tags_classifier = None @@ -157,7 +163,7 @@ class DocumentClassifier(object): logging.getLogger(__name__).debug( "Training correspondent classifier..." ) - self.correspondent_classifier = MLPClassifier(verbose=True, tol=0.01) + self.correspondent_classifier = MLPClassifier(tol=0.01) self.correspondent_classifier.fit( data_vectorized, labels_correspondent @@ -173,7 +179,7 @@ class DocumentClassifier(object): logging.getLogger(__name__).debug( "Training document type classifier..." ) - self.document_type_classifier = MLPClassifier(verbose=True, tol=0.01) + self.document_type_classifier = MLPClassifier(tol=0.01) self.document_type_classifier.fit( data_vectorized, labels_document_type diff --git a/src/documents/file_handling.py b/src/documents/file_handling.py index cd47406b6..ee7e9b761 100644 --- a/src/documents/file_handling.py +++ b/src/documents/file_handling.py @@ -65,25 +65,24 @@ def many_to_dictionary(field): return mydictionary -def generate_filename(document): - # Create filename based on configured format +def generate_filename(doc): path = "" try: if settings.PAPERLESS_FILENAME_FORMAT is not None: tags = defaultdict(lambda: slugify(None), - many_to_dictionary(document.tags)) + many_to_dictionary(doc.tags)) path = settings.PAPERLESS_FILENAME_FORMAT.format( - correspondent=slugify(document.correspondent), - title=slugify(document.title), - created=slugify(document.created), - created_year=document.created.year if document.created else "none", - created_month=document.created.month if document.created else "none", - created_day=document.created.day if document.created else "none", - added=slugify(document.added), - added_year=document.added.year if document.added else "none", - added_month=document.added.month if document.added else "none", - added_day=document.added.day if document.added else "none", + correspondent=slugify(doc.correspondent), + title=slugify(doc.title), + created=slugify(doc.created), + created_year=doc.created.year if doc.created else "none", + created_month=doc.created.month if doc.created else "none", + created_day=doc.created.day if doc.created else "none", + added=slugify(doc.added), + added_year=doc.added.year if doc.added else "none", + added_month=doc.added.month if doc.added else "none", + added_day=doc.added.day if doc.added else "none", tags=tags, ) except (ValueError, KeyError, IndexError): @@ -93,12 +92,12 @@ def generate_filename(document): # Always append the primary key to guarantee uniqueness of filename if len(path) > 0: - filename = "%s-%07i%s" % (path, document.pk, document.file_type) + filename = "%s-%07i%s" % (path, doc.pk, doc.file_type) else: - filename = "%07i%s" % (document.pk, document.file_type) + filename = "%07i%s" % (doc.pk, doc.file_type) # Append .gpg for encrypted files - if document.storage_type == document.STORAGE_TYPE_GPG: + if doc.storage_type == doc.STORAGE_TYPE_GPG: filename += ".gpg" return filename diff --git a/src/documents/matching.py b/src/documents/matching.py index ae1a9a9cf..212698ad3 100644 --- a/src/documents/matching.py +++ b/src/documents/matching.py @@ -12,7 +12,10 @@ def match_correspondents(document_content, classifier): pred_id = None correspondents = Correspondent.objects.all() - return [o for o in correspondents if matches(o, document_content) or o.pk == pred_id] + + return list(filter( + lambda o: matches(o, document_content) or o.pk == pred_id, + correspondents)) def match_document_types(document_content, classifier): @@ -22,15 +25,23 @@ def match_document_types(document_content, classifier): pred_id = None document_types = DocumentType.objects.all() - return [o for o in document_types if matches(o, document_content) or o.pk == pred_id] + + return list(filter( + lambda o: matches(o, document_content) or o.pk == pred_id, + document_types)) def match_tags(document_content, classifier): - objects = Tag.objects.all() - predicted_tag_ids = classifier.predict_tags(document_content) if classifier else [] + if classifier: + predicted_tag_ids = classifier.predict_tags(document_content) + else: + predicted_tag_ids = [] - matched_tags = [o for o in objects if matches(o, document_content) or o.pk in predicted_tag_ids] - return matched_tags + tags = Tag.objects.all() + + return list(filter( + lambda o: matches(o, document_content) or o.pk in predicted_tag_ids, + tags)) def matches(matching_model, document_content): @@ -48,39 +59,45 @@ def matches(matching_model, document_content): if matching_model.matching_algorithm == MatchingModel.MATCH_ALL: for word in _split_match(matching_model): search_result = re.search( - r"\b{}\b".format(word), document_content, **search_kwargs) + rf"\b{word}\b", document_content, **search_kwargs) if not search_result: return False return True - if matching_model.matching_algorithm == MatchingModel.MATCH_ANY: + elif matching_model.matching_algorithm == MatchingModel.MATCH_ANY: for word in _split_match(matching_model): - if re.search(r"\b{}\b".format(word), document_content, **search_kwargs): + if re.search(rf"\b{word}\b", document_content, **search_kwargs): return True return False - if matching_model.matching_algorithm == MatchingModel.MATCH_LITERAL: + elif matching_model.matching_algorithm == MatchingModel.MATCH_LITERAL: return bool(re.search( - r"\b{}\b".format(matching_model.match), document_content, **search_kwargs)) + rf"\b{matching_model.match}\b", + document_content, + **search_kwargs + )) - if matching_model.matching_algorithm == MatchingModel.MATCH_REGEX: + elif matching_model.matching_algorithm == MatchingModel.MATCH_REGEX: return bool(re.search( - re.compile(matching_model.match, **search_kwargs), document_content)) + re.compile(matching_model.match, **search_kwargs), + document_content + )) - if matching_model.matching_algorithm == MatchingModel.MATCH_FUZZY: + elif matching_model.matching_algorithm == MatchingModel.MATCH_FUZZY: match = re.sub(r'[^\w\s]', '', matching_model.match) text = re.sub(r'[^\w\s]', '', document_content) if matching_model.is_insensitive: match = match.lower() text = text.lower() - return True if fuzz.partial_ratio(match, text) >= 90 else False + return fuzz.partial_ratio(match, text) >= 90 - if matching_model.matching_algorithm == MatchingModel.MATCH_AUTO: + elif matching_model.matching_algorithm == MatchingModel.MATCH_AUTO: # this is done elsewhere. return False - raise NotImplementedError("Unsupported matching algorithm") + else: + raise NotImplementedError("Unsupported matching algorithm") def _split_match(matching_model): diff --git a/src/paperless/auth.py b/src/paperless/auth.py index ecd697f0e..83279ef36 100644 --- a/src/paperless/auth.py +++ b/src/paperless/auth.py @@ -9,7 +9,7 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication): """ def authenticate(self, request): - if settings.DEBUG and 'Referer' in request.headers and request.headers['Referer'].startswith('http://localhost:4200/'): + if settings.DEBUG and 'Referer' in request.headers and request.headers['Referer'].startswith('http://localhost:4200/'): # NOQA: E501 user = User.objects.filter(is_staff=True).first() print("Auto-Login with user {}".format(user)) return (user, None) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 03f915769..9d0397f24 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -283,8 +283,8 @@ class MailAccountHandler(LoggingMixin): path=temp_filename, override_filename=att.filename, override_title=title, - override_correspondent_id=correspondent.id if correspondent else None, - override_document_type_id=doc_type.id if doc_type else None, + override_correspondent_id=correspondent.id if correspondent else None, # NOQA: E501 + override_document_type_id=doc_type.id if doc_type else None, # NOQA: E501 override_tag_ids=[tag.id] if tag else None, task_name=f"Mail: {att.filename}" ) diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index d0ce01327..c9e77486e 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -123,7 +123,7 @@ class RasterisedDocumentParser(DocumentParser): ocr_pages = self._complete_ocr_default_language( images, sample_page_index, sample_page_text) - elif not ISO639[guessed_language] in pyocr.get_available_tools()[0].get_available_languages(): + elif not ISO639[guessed_language] in pyocr.get_available_tools()[0].get_available_languages(): # NOQA: E501 self.log( "warning", f"Detected language {guessed_language} is not available " From db4519a64433262d8df1c9586a9dc01e1708ff9e Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 15:34:30 +0100 Subject: [PATCH 17/52] url patterns cleanup --- src/paperless/urls.py | 63 ++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 220e6402c..dd5e6a379 100755 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -28,43 +28,56 @@ api_router.register(r"tags", TagViewSet) urlpatterns = [ + re_path(r"^api/", include([ + re_path(r"^auth/", + include(('rest_framework.urls', 'rest_framework'), + namespace="rest_framework")), - # API - re_path(r"^api/auth/", include(('rest_framework.urls', 'rest_framework'), namespace="rest_framework")), - re_path(r"^api/search/autocomplete/", SearchAutoCompleteView.as_view(), name="autocomplete"), - re_path(r"^api/search/", SearchView.as_view(), name="search"), - re_path(r"^api/statistics/", StatisticsView.as_view(), name="statistics"), - re_path(r"^api/", include((api_router.urls, 'drf'), namespace="drf")), + re_path(r"^search/autocomplete/", + SearchAutoCompleteView.as_view(), + name="autocomplete"), + + re_path(r"^search/", + SearchView.as_view(), + name="search"), + + re_path(r"^statistics/", + StatisticsView.as_view(), + name="statistics"), + + ] + api_router.urls)), - # Favicon re_path(r"^favicon.ico$", FaviconView.as_view(), name="favicon"), - # The Django admin re_path(r"admin/", admin.site.urls), - # These redirects are here to support clients that use the old FetchView. - re_path( - r"^fetch/doc/(?P<pk>\d+)$", - RedirectView.as_view(url='/api/documents/%(pk)s/download/'), - ), - re_path( - r"^fetch/thumb/(?P<pk>\d+)$", - RedirectView.as_view(url='/api/documents/%(pk)s/thumb/'), - ), - re_path( - r"^fetch/preview/(?P<pk>\d+)$", - RedirectView.as_view(url='/api/documents/%(pk)s/preview/'), - ), - re_path(r"^push$", csrf_exempt(RedirectView.as_view(url='/api/documents/post_document/'))), + re_path(r"^fetch/", include([ + re_path( + r"^doc/(?P<pk>\d+)$", + RedirectView.as_view(url='/api/documents/%(pk)s/download/'), + ), + re_path( + r"^thumb/(?P<pk>\d+)$", + RedirectView.as_view(url='/api/documents/%(pk)s/thumb/'), + ), + re_path( + r"^preview/(?P<pk>\d+)$", + RedirectView.as_view(url='/api/documents/%(pk)s/preview/'), + ), + ])), - # Frontend assets TODO: this is pretty bad. - path('assets/<path:path>', RedirectView.as_view(url='/static/frontend/assets/%(path)s')), + re_path(r"^push$", csrf_exempt( + RedirectView.as_view(url='/api/documents/post_document/'))), + # Frontend assets TODO: this is pretty bad, but it works. + path('assets/<path:path>', + RedirectView.as_view(url='/static/frontend/assets/%(path)s')), + + # login, logout path('accounts/', include('django.contrib.auth.urls')), # Root of the Frontent re_path(r".*", login_required(IndexView.as_view())), - ] # Text in each page's <h1> (and above login form). From 110c5c392cd1c50113b74f9dfa4ae06733a95974 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 16:07:28 +0100 Subject: [PATCH 18/52] added tests to pycodestyle ignore for now. 79 characters really doesnt work there and i don't really care enough. --- src/setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/setup.cfg b/src/setup.cfg index c0f80d964..4b0a216f5 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -1,6 +1,5 @@ [pycodestyle] -exclude = migrations, paperless/settings.py, .tox -ignore = E501 +exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings From 3afee66aaa1a06db46aea97c29aae4e4f68b8713 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 20:14:48 +0100 Subject: [PATCH 19/52] updated entrypoint script to wait for postgres --- docker/docker-entrypoint.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index c6e0d1cab..dfa7cfc65 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -15,8 +15,42 @@ map_uidgid() { fi } + +wait_for_postgres() { + attempt_num=1 + max_attempts=5 + + echo "Waiting for PostgreSQL to start..." + + host="${PAPERLESS_DBHOST}" + + while !</dev/tcp/$host/5432 ; + do + + if [ $attempt_num -eq $max_attempts ] + then + echo "Unable to connect to database." + exit 1 + else + echo "Attempt $attempt_num failed! Trying again in 5 seconds..." + + fi + + attempt_num=$(expr "$attempt_num" + 1) + sleep 5 + done + + +} + + migrations() { + if [[ -n "${PAPERLESS_DBHOST}" ]] + then + wait_for_postgres + fi + ( # flock is in place to prevent multiple containers from doing migrations # simultaneously. This also ensures that the db is ready when the command From 3600e5a8fbbe41ed2d1e703f8211824eb4b93ee3 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 20:29:30 +0100 Subject: [PATCH 20/52] updated docs --- docs/index.rst | 3 ++ docs/setup.rst | 96 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 756fee3b1..a9142a682 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,9 @@ resources in the documentation: that's fully tested and production ready. * See :ref:`this note <utilities-encyption>` about GnuPG encryption in paperless-ng. +* Paperless is now integrated with a + :ref:`task processing queue <setup-task_processor>` that tells you + at a glance when and why something is not working. * The :ref:`changelog <paperless_changelog>` contains a detailed list of all changes in paperless-ng. diff --git a/docs/setup.rst b/docs/setup.rst index 0f5db1ae5..5520f5594 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -66,6 +66,8 @@ Paperless consists of the following components: $ cd /path/to/paperless/src/ $ pipenv run python3 manage.py document_consumer + .. _setup-task_processor: + * **The task processor:** Paperless relies on `Django Q <https://django-q.readthedocs.io/en/latest/>`_ for doing much of the heavy lifting. This is a task queue that accepts tasks from multiple sources and processes tasks in parallel. It also comes with a scheduler that executes @@ -86,7 +88,8 @@ Paperless consists of the following components: a modern multicore system, consumption with full ocr is blazing fast. The task processor comes with a built-in admin interface that you can use to see whenever any of the - tasks fail and inspect the errors. + tasks fail and inspect the errors (i.e., wrong email credentials, errors during consuming a specific + file, etc). You may start the task processor by executing: @@ -249,15 +252,21 @@ Migration to paperless-ng is then performed in a few simple steps: .. caution:: - Make sure you also download the ``.env`` file. This will set the - project name for docker compose to ``paperless`` and then it will - automatically reuse your existing paperless volumes. + The release include a ``.env`` file. This will set the + project name for docker compose to ``paperless`` so that paperless-ng will + automatically reuse your existing paperless volumes. When you start it, it + will migrate your existing data. After that, your old paperless installation + will be incompatible with the migrated volumes. -4. Adjust ``docker-compose.yml`` and +4. Copy the ``docker-compose.sqlite.yml`` file to ``docker-compose.yml``. + If you want to migrate to PostgreSQL, do that after you migrated your existing + SQLite database. + +5. Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs. - See `docker route`_ for details on which edits are required. + See `docker route`_ for details on which edits are advised. -5. Start paperless-ng. +6. Start paperless-ng. .. code:: bash @@ -273,19 +282,80 @@ Migration to paperless-ng is then performed in a few simple steps: This will run paperless in the background and automatically start it on system boot. -6. Paperless installed a permanent redirect to ``admin/`` in your browser. This +7. Paperless installed a permanent redirect to ``admin/`` in your browser. This redirect is still in place and prevents access to the new UI. Clear - everything related to paperless in your browsers data in order to fix - this issue. + browsing cache in order to fix this. + +8. Optionally, follow the instructions below to migrate your existing data to PostgreSQL. .. _setup-sqlite_to_psql: -Moving data from sqlite to postgresql +Moving data from SQLite to PostgreSQL ===================================== -.. warning:: +Moving your data from SQLite to PostgreSQL is done via executing a series of django +management commands as below. + +.. caution:: + + Make sure that your sqlite database is migrated to the latest version. + Starting paperless will make sure that this is the case. If your try to + load data from an old database schema in SQLite into a newer database + schema in PostgreSQL, you will run into trouble. + +1. Stop paperless, if it is running. +2. Tell paperless to use PostgreSQL: + + a) With docker, copy the provided ``docker-compose.postgres.yml`` file to + ``docker-compose.yml``. Remember to adjust the consumption directory, + if necessary. + b) Without docker, configure the database in your ``paperless.conf`` file. + See :ref:`configuration` for details. + +3. Open a shell and initialize the database: + + a) With docker, run the following command to open a shell within the paperless + container: + + .. code:: shell-session + + $ cd /path/to/paperless + $ docker-compose run --rm webserver /bin/bash + + This will lauch the container and initialize the PostgreSQL database. + + b) Without docker, open a shell in your virtual environment, switch to + the ``src`` directory and create the database schema: + + .. code:: shell-session + + $ cd /path/to/paperless + $ pipenv shell + $ cd src + $ python3 manage.py migrate + + This will not copy any data yet. + +4. Dump your data from SQLite: + + .. code:: shell-session + + $ python3 manage.py dumpdata --database=sqlite --exclude=contenttypes --exclude=auth.Permission > data.json + +5. Load your data into PostgreSQL: + + .. code:: shell-session + + $ python3 manage.py loaddata data.json + +6. Exit the shell. + + .. code:: shell-session + + $ exit + +7. Start paperless. - TBD. .. _redis: https://redis.io/ From d3482a4aef874ba20f9ab87b2870582afe622d29 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 20:44:35 +0100 Subject: [PATCH 21/52] changelog --- docs/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 86a24df27..4c938ba87 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,9 @@ next content type was not set correctly. (i.e. PDF documents with content type ``application/octet-stream``) +* Docker entrypoint script awaits the database server if it is + configured. + paperless-ng 0.9.1 ################## From af3d161f666b98e0640f038a0c37eb3ae876866d Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sat, 21 Nov 2020 23:12:34 +0100 Subject: [PATCH 22/52] updated the admin, ordering for mail rules --- src/documents/admin.py | 21 ++++++++++++++----- src/paperless_mail/admin.py | 10 ++++++++- src/paperless_mail/mail.py | 2 +- .../migrations/0004_mailrule_order.py | 18 ++++++++++++++++ src/paperless_mail/models.py | 2 ++ 5 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 src/paperless_mail/migrations/0004_mailrule_order.py diff --git a/src/documents/admin.py b/src/documents/admin.py index 5b3975fda..8b9f2fce9 100755 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -51,15 +51,16 @@ class DocumentAdmin(admin.ModelAdmin): search_fields = ("correspondent__name", "title", "content", "tags__name") readonly_fields = ("added", "mime_type", "storage_type", "filename") + + list_display_links = ("title",) + list_display = ( - "title", - "created", - "added", "correspondent", + "title", "tags_", - "archive_serial_number", - "document_type" + "created", ) + list_filter = ( "document_type", "tags", @@ -117,9 +118,19 @@ class DocumentAdmin(admin.ModelAdmin): class LogAdmin(admin.ModelAdmin): + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + list_display = ("created", "message", "level",) list_filter = ("level", "created",) + ordering = ('-created',) + + list_display_links = ("created", "message") + admin.site.register(Correspondent, CorrespondentAdmin) admin.site.register(Tag, TagAdmin) diff --git a/src/paperless_mail/admin.py b/src/paperless_mail/admin.py index 8d05c2a42..d8560c418 100644 --- a/src/paperless_mail/admin.py +++ b/src/paperless_mail/admin.py @@ -11,7 +11,15 @@ class MailRuleAdmin(admin.ModelAdmin): list_filter = ("account",) - list_display = ("name", "account", "folder", "action") + list_display = ("order", "name", "account", "folder", "action") + + list_editable = ("order", ) + + list_display_links = ("name", ) + + sortable_by = [] + + ordering = ["order"] admin.site.register(MailAccount, MailAccountAdmin) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 9d0397f24..dfdfa09ce 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -161,7 +161,7 @@ class MailAccountHandler(LoggingMixin): self.log('debug', f"Account {account}: Processing " f"{account.rules.count()} rule(s)") - for rule in account.rules.all(): + for rule in account.rules.order_by('order'): self.log( 'debug', f"Account {account}: Processing rule {rule.name}") diff --git a/src/paperless_mail/migrations/0004_mailrule_order.py b/src/paperless_mail/migrations/0004_mailrule_order.py new file mode 100644 index 000000000..498f280a1 --- /dev/null +++ b/src/paperless_mail/migrations/0004_mailrule_order.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.3 on 2020-11-21 21:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('paperless_mail', '0003_auto_20201118_1940'), + ] + + operations = [ + migrations.AddField( + model_name='mailrule', + name='order', + field=models.IntegerField(default=0), + ), + ] diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index 14da202fa..c8ab09479 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -78,6 +78,8 @@ class MailRule(models.Model): name = models.CharField(max_length=256, unique=True) + order = models.IntegerField(default=0) + account = models.ForeignKey( MailAccount, related_name="rules", From d65a118d8aac6cc30d937b5f1d4843611f4c0999 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 00:35:19 +0100 Subject: [PATCH 23/52] use docker compose for building --- scripts/make-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/make-release.sh b/scripts/make-release.sh index 06548748b..6860b4ae6 100755 --- a/scripts/make-release.sh +++ b/scripts/make-release.sh @@ -94,7 +94,7 @@ cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/" cd "$PAPERLESS_DIST_APP" -docker build . -t "jonaswinkler/paperless-ng:$VERSION" +docker-compose -f docker-compose.postgres.yml build # works. package the app! From 54af13e4b8981e470734a49dc8521251d2cabc30 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 01:39:48 +0100 Subject: [PATCH 24/52] much better mail rule admin --- docs/changelog.rst | 8 +++++++- src/paperless_mail/admin.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c938ba87..2af97b33b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,7 +12,7 @@ next if a file type is supported and which parser to use. Removes all file type checks that where present in MANY different places in paperless. - + * Mail consumer now correctly consumes documents even when their content type was not set correctly. (i.e. PDF documents with content type ``application/octet-stream``) @@ -20,6 +20,12 @@ next * Docker entrypoint script awaits the database server if it is configured. +* Basic sorting of mail rules added. + +* Disabled editing of logs. + +* Much better admin for mail rule editing. + paperless-ng 0.9.1 ################## diff --git a/src/paperless_mail/admin.py b/src/paperless_mail/admin.py index d8560c418..b959171f7 100644 --- a/src/paperless_mail/admin.py +++ b/src/paperless_mail/admin.py @@ -9,6 +9,38 @@ class MailAccountAdmin(admin.ModelAdmin): class MailRuleAdmin(admin.ModelAdmin): + radio_fields = { + "action": admin.VERTICAL, + "assign_title_from": admin.VERTICAL, + "assign_correspondent_from": admin.VERTICAL + } + + fieldsets = ( + (None, { + 'fields': ('name', 'order', 'account', 'folder') + }), + ("Filter", { + 'description': "Paperless will only process mails that match ALL " + "of the filters given below.", + 'fields': ('filter_from', 'filter_subject', 'filter_body', 'maximum_age') + }), + ("Actions", { + 'description': "The action applied to the mail. This action is " + "only performed when documents were consumed from " + "the mail. Mails without attachments will remain " + "entirely untouched.", + 'fields': ('action', 'action_parameter') + }), + ("Metadata", { + 'description': "Assign metadata to documents consumed from this " + "rule automatically. If you do not assign tags, " + "types or correspondents here, paperless will " + "still process all matching rules that you have " + "defined.", + "fields": ('assign_title_from', 'assign_tag', 'assign_document_type', 'assign_correspondent_from', 'assign_correspondent') + }) + ) + list_filter = ("account",) list_display = ("order", "name", "account", "folder", "action") From 532d5c1744b835402019a8e5595dc6ff7d3abae2 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 11:35:04 +0100 Subject: [PATCH 25/52] a couple styling changes, collapsible menu --- .../app-frame/app-frame.component.html | 7 ++++--- .../components/app-frame/app-frame.component.ts | 2 ++ .../document-detail.component.html | 16 ++++++++-------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 519b69bf0..1232ecf12 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -1,7 +1,8 @@ <nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0 shadow"> <span class="navbar-brand col-md-3 col-lg-2 mr-0 px-3" href="#">Paperless-ng</span> <button class="navbar-toggler position-absolute d-md-none collapsed" type="button" data-toggle="collapse" - data-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation"> + data-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation" + (click)="isMenuCollapsed = !isMenuCollapsed"> <span class="navbar-toggler-icon"></span> </button> <form (ngSubmit)="search()" class="w-100"> @@ -22,7 +23,7 @@ <div class="container-fluid"> <div class="row"> - <nav id="sidebarMenu" class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse"> + <nav id="sidebarMenu" class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse" [ngbCollapse]="isMenuCollapsed"> <div class="sidebar-sticky pt-3"> <ul class="nav flex-column"> <li class="nav-item"> @@ -161,4 +162,4 @@ <router-outlet></router-outlet> </main> </div> -</div> \ No newline at end of file +</div> diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index 7e8694f9f..be72ad469 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -25,6 +25,8 @@ export class AppFrameComponent implements OnInit, OnDestroy { ) { } + isMenuCollapsed: boolean = true + searchField = new FormControl('') openDocuments: PaperlessDocument[] = [] diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index 577b61a55..b460e7f97 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -3,19 +3,19 @@ <svg class="buttonicon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#trash" /> </svg> - <span class="d-none d-lg-inline">Delete</span> + <span class="d-none d-lg-inline"> Delete</span> </button> <a [href]="downloadUrl" class="btn btn-sm btn-outline-secondary mr-2"> <svg class="buttonicon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#download" /> </svg> - <span class="d-none d-lg-inline">Download</span> + <span class="d-none d-lg-inline"> Download</span> </a> <button type="button" class="btn btn-sm btn-outline-secondary" (click)="close()"> <svg class="buttonicon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#x" /> </svg> - <span class="d-none d-lg-inline">Close</span> + <span class="d-none d-lg-inline"> Close</span> </button> </app-page-header> @@ -23,17 +23,17 @@ <div class="row"> <div class="col-xl"> <form [formGroup]='documentForm' (ngSubmit)="save()"> - + <app-input-text title="Title" formControlName="title"></app-input-text> - + <div class="form-group"> <label for="archive_serial_number">Archive Serial Number</label> <input type="number" class="form-control" id="archive_serial_number" formControlName='archive_serial_number'> </div> - + <app-input-date-time title="Date created" titleTime="Time created" formControlName="created"></app-input-date-time> - + <div class="form-group"> <label for="content">Content</label> <textarea class="form-control" id="content" rows="5" formControlName='content'></textarea> @@ -58,4 +58,4 @@ </object> </div> -</div> \ No newline at end of file +</div> From 388e5b56de7a24f558126ba043001039d383b822 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 11:41:13 +0100 Subject: [PATCH 26/52] reversible migrations. --- .../migrations/1000_update_paperless_all.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/documents/migrations/1000_update_paperless_all.py b/src/documents/migrations/1000_update_paperless_all.py index 5e5b475a3..dc6313dd8 100644 --- a/src/documents/migrations/1000_update_paperless_all.py +++ b/src/documents/migrations/1000_update_paperless_all.py @@ -1,4 +1,6 @@ # Generated by Django 3.1.3 on 2020-11-07 12:35 +import uuid + from django.db import migrations, models import django.db.models.deletion @@ -20,6 +22,14 @@ def make_index(apps, schema_editor): print(" --> Cannot create document index.") +def logs_set_default_group(apps, schema_editor): + Log = apps.get_model('documents', 'Log') + for log in Log.objects.all(): + if log.group is None: + log.group = uuid.uuid4() + log.save() + + class Migration(migrations.Migration): dependencies = [ @@ -85,6 +95,10 @@ class Migration(migrations.Migration): name='group', field=models.UUIDField(blank=True, null=True), ), + migrations.RunPython( + code=django.db.migrations.operations.special.RunPython.noop, + reverse_code=logs_set_default_group + ), migrations.RunPython( code=make_index, reverse_code=django.db.migrations.operations.special.RunPython.noop, From 172b37239fec9bb7b88da084158c45946826145f Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 11:42:17 +0100 Subject: [PATCH 27/52] changed a few things with the mail rule admin. --- .../migrations/0005_help_texts.py | 23 +++++++++++++++++++ src/paperless_mail/models.py | 14 +++++------ 2 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 src/paperless_mail/migrations/0005_help_texts.py diff --git a/src/paperless_mail/migrations/0005_help_texts.py b/src/paperless_mail/migrations/0005_help_texts.py new file mode 100644 index 000000000..71899c8ef --- /dev/null +++ b/src/paperless_mail/migrations/0005_help_texts.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.3 on 2020-11-22 10:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('paperless_mail', '0004_mailrule_order'), + ] + + operations = [ + migrations.AlterField( + model_name='mailrule', + name='action', + field=models.PositiveIntegerField(choices=[(3, "Mark as read, don't process read mails"), (4, "Flag the mail, don't process flagged mails"), (2, 'Move to specified folder'), (1, 'Delete')], default=3), + ), + migrations.AlterField( + model_name='mailrule', + name='maximum_age', + field=models.PositiveIntegerField(default=30, help_text='Specified in days.'), + ), + ] diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index c8ab09479..fbcfaf980 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -46,10 +46,10 @@ class MailRule(models.Model): ACTION_FLAG = 4 ACTIONS = ( - (ACTION_DELETE, "Delete"), - (ACTION_MOVE, "Move to specified folder"), (ACTION_MARK_READ, "Mark as read, don't process read mails"), - (ACTION_FLAG, "Flag the mail, don't process flagged mails") + (ACTION_FLAG, "Flag the mail, don't process flagged mails"), + (ACTION_MOVE, "Move to specified folder"), + (ACTION_DELETE, "Delete"), ) TITLE_FROM_SUBJECT = 1 @@ -92,15 +92,13 @@ class MailRule(models.Model): filter_subject = models.CharField(max_length=256, null=True, blank=True) filter_body = models.CharField(max_length=256, null=True, blank=True) - maximum_age = models.PositiveIntegerField(default=30) + maximum_age = models.PositiveIntegerField( + default=30, + help_text="Specified in days.") action = models.PositiveIntegerField( choices=ACTIONS, default=ACTION_MARK_READ, - help_text="The action applied to the mail. This action is only " - "performed when documents were consumed from the mail. " - "Mails without attachments will remain entirely " - "untouched." ) action_parameter = models.CharField( From ea089de3b3e17b0e5a3d444db2b86b246a2f6859 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 11:42:30 +0100 Subject: [PATCH 28/52] added a test case for the index --- src/documents/tests/test_index.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/documents/tests/test_index.py diff --git a/src/documents/tests/test_index.py b/src/documents/tests/test_index.py new file mode 100644 index 000000000..830fca0e0 --- /dev/null +++ b/src/documents/tests/test_index.py @@ -0,0 +1,14 @@ +from django.test import TestCase + +from documents.index import JsonFormatter + + +class JsonFormatterTest(TestCase): + + def setUp(self) -> None: + self.formatter = JsonFormatter() + + def test_empty_fragments(self): + self.assertListEqual(self.formatter.format([]), []) + + From fec9e54049d94eea30b83f46a40cd96fdde4c8cd Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 12:54:08 +0100 Subject: [PATCH 29/52] new setting: PAPERLESS_OCR_PAGES --- docs/changelog.rst | 3 +++ docs/configuration.rst | 10 ++++++++++ docs/setup.rst | 26 ++++++++++++++++++++++++++ paperless.conf.example | 1 + src/paperless/settings.py | 2 ++ src/paperless_tesseract/parsers.py | 17 ++++++++++++----- 6 files changed, 54 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2af97b33b..bb119bf1f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,9 @@ next * Much better admin for mail rule editing. +* New setting ``PAPERLESS_OCR_PAGES`` limits the tesseract parser + to the first n pages of scanned documents. + paperless-ng 0.9.1 ################## diff --git a/docs/configuration.rst b/docs/configuration.rst index 1ddd7ca0e..afb0b5f90 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -184,6 +184,16 @@ PAPERLESS_TIME_ZONE=<timezone> +PAPERLESS_OCR_PAGES=<num> + Tells paperless to use only the specified amount of pages for OCR. Documents + with less than the specified amount of pages get OCR'ed completely. + + Specifying 1 here will only use the first page. + + Defaults to 0, which disables this feature and always uses all pages. + + + PAPERLESS_OCR_LANGUAGE=<lang> Customize the default language that tesseract will attempt to use when parsing documents. The default language is used whenever diff --git a/docs/setup.rst b/docs/setup.rst index 5520f5594..dff605889 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -358,4 +358,30 @@ management commands as below. 7. Start paperless. +Considerations for less powerful devices +######################################## + +Paperless runs on Raspberry Pi. However, some things are rather slow on the Pi and +configuring some options in paperless can help improve performance immensely: + +* Consider setting ``PAPERLESS_OCR_PAGES`` to 1, so that paperless will only OCR + the first page of your documents. +* ``PAPERLESS_TASK_WORKERS`` and ``PAPERLESS_THREADS_PER_WORKER`` are configured + to use all cores. The Raspberry Pi models 3 and up have 4 cores, meaning that + paperless will use 2 workers and 2 threads per worker. This may result in + slugish response times during consumption, so you might want to lower these + settings (example: 2 workers and 1 thread to always have some computing power + left for other tasks). +* Keep ``PAPERLESS_OCR_ALWAYS`` at its default value 'false' and consider OCR'ing + your documents before feeding them into paperless. Some scanners are able to + do this! +* Lower ``PAPERLESS_CONVERT_DENSITY`` from its default value 300 to 200. This + will still result in rather accurate OCR, but will decrease consumption time + by quite a bit. +* Set ``PAPERLESS_OPTIMIZE_THUMBNAILS`` to 'false' if you want faster consumption + times. Thumbnails will be about 20% larger. + +For details, refer to :ref:`configuration`. + + .. _redis: https://redis.io/ diff --git a/paperless.conf.example b/paperless.conf.example index e1fd17a77..4749151e7 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -35,6 +35,7 @@ #PAPERLESS_TASK_WORKERS=1 #PAPERLESS_THREADS_PER_WORKER=1 #PAPERLESS_TIME_ZONE=UTC +#PAPERLESS_OCR_PAGES=1 #PAPERLESS_OCR_LANGUAGE=eng #PAPERLESS_OCR_ALWAYS=false #PAPERLESS_CONSUMER_POLLING=10 diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 06895e92f..0d64efa57 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -322,6 +322,8 @@ CONSUMER_DELETE_DUPLICATES = __get_boolean("PAPERLESS_CONSUMER_DELETE_DUPLICATES OPTIMIZE_THUMBNAILS = __get_boolean("PAPERLESS_OPTIMIZE_THUMBNAILS", "true") +OCR_PAGES = int(os.getenv('PAPERLESS_OCR_PAGES', 0)) + # The default language that tesseract will attempt to use when parsing # documents. It should be a 3-letter language code consistent with ISO 639. OCR_LANGUAGE = os.getenv("PAPERLESS_OCR_LANGUAGE", "eng") diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index c9e77486e..b8320a4f0 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -147,18 +147,25 @@ class RasterisedDocumentParser(DocumentParser): Greyscale images are easier for Tesseract to OCR """ + # Convert PDF to multiple PNMs + input_file = self.document_path + + if settings.OCR_PAGES == 1: + input_file += "[0]" + elif settings.OCR_PAGES > 1: + input_file += f"[0-{settings.OCR_PAGES - 1}]" + self.log( "debug", - f"Converting document {self.document_path} into greyscale images") + f"Converting document {input_file} into greyscale images") - # Convert PDF to multiple PNMs - pnm = os.path.join(self.tempdir, "convert-%04d.pnm") + output_files = os.path.join(self.tempdir, "convert-%04d.pnm") run_convert(density=settings.CONVERT_DENSITY, depth="8", type="grayscale", - input_file=self.document_path, - output_file=pnm, + input_file=input_file, + output_file=output_files, logging_group=self.logging_group) # Get a list of converted images From d2df1b0fc941de680032fcee2575ff3cd88b3203 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 13:20:20 +0100 Subject: [PATCH 30/52] updated travis --- .travis.yml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2db24da87..fd5253375 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,37 @@ language: python -python: - - "3.6" - - "3.7" - - "3.8" +matrix: + include: + - name: "Paperless on Python 3.6" + python: "3.6" + + - name: "Paperless on Python 3.7" + python: "3.7" + + - name: "Paperless on Python 3.8" + python: "3.8" + + - name: "Documentation" + before_install: true + install: true + script: + - cd docs/ + - make html + after_success: true + + - name: "Front end" + language: node_js + node_js: + - 6 + before_install: true + install: + - cd src-ui/ + - npm install -g + script: + - cd src-ui/ + - ng build --prod + after_success: true + before_install: - sudo apt-get update -qq From 5a292426c91a32ba9e3e4641aec79cbdb88ac973 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 13:23:46 +0100 Subject: [PATCH 31/52] codestyle --- src/paperless_mail/admin.py | 40 ++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/paperless_mail/admin.py b/src/paperless_mail/admin.py index b959171f7..0440234b7 100644 --- a/src/paperless_mail/admin.py +++ b/src/paperless_mail/admin.py @@ -20,24 +20,36 @@ class MailRuleAdmin(admin.ModelAdmin): 'fields': ('name', 'order', 'account', 'folder') }), ("Filter", { - 'description': "Paperless will only process mails that match ALL " - "of the filters given below.", - 'fields': ('filter_from', 'filter_subject', 'filter_body', 'maximum_age') + 'description': + "Paperless will only process mails that match ALL of the " + "filters given below.", + 'fields': + ('filter_from', + 'filter_subject', + 'filter_body', + 'maximum_age') }), ("Actions", { - 'description': "The action applied to the mail. This action is " - "only performed when documents were consumed from " - "the mail. Mails without attachments will remain " - "entirely untouched.", - 'fields': ('action', 'action_parameter') + 'description': + "The action applied to the mail. This action is only " + "performed when documents were consumed from the mail. Mails " + "without attachments will remain entirely untouched.", + 'fields': ( + 'action', + 'action_parameter') }), ("Metadata", { - 'description': "Assign metadata to documents consumed from this " - "rule automatically. If you do not assign tags, " - "types or correspondents here, paperless will " - "still process all matching rules that you have " - "defined.", - "fields": ('assign_title_from', 'assign_tag', 'assign_document_type', 'assign_correspondent_from', 'assign_correspondent') + 'description': + "Assign metadata to documents consumed from this rule " + "automatically. If you do not assign tags, types or " + "correspondents here, paperless will still process all " + "matching rules that you have defined.", + "fields": ( + 'assign_title_from', + 'assign_tag', + 'assign_document_type', + 'assign_correspondent_from', + 'assign_correspondent') }) ) From 79e6b1f5dd0aaf3c5fad6d448ed3fd2d58617262 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 13:25:59 +0100 Subject: [PATCH 32/52] travis fixes --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd5253375..f0412c937 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ matrix: - name: "Documentation" before_install: true - install: true script: - cd docs/ - make html @@ -22,7 +21,7 @@ matrix: - name: "Front end" language: node_js node_js: - - 6 + - 15 before_install: true install: - cd src-ui/ From 1e0020b56b05b1f23ccef80fa54ce43ab098536c Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 13:31:54 +0100 Subject: [PATCH 33/52] travis fixes --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f0412c937..1eaa9da68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python -matrix: +jobs: include: - name: "Paperless on Python 3.6" python: "3.6" @@ -12,7 +12,6 @@ matrix: python: "3.8" - name: "Documentation" - before_install: true script: - cd docs/ - make html @@ -25,9 +24,8 @@ matrix: before_install: true install: - cd src-ui/ - - npm install -g + - npm install -g @angular/cli script: - - cd src-ui/ - ng build --prod after_success: true From 9e3142973287d01f45359f8ee5286a98d495e0c0 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 13:37:24 +0100 Subject: [PATCH 34/52] travis fixes --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1eaa9da68..248eebb64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,7 @@ jobs: install: - cd src-ui/ - npm install -g @angular/cli + - npm install script: - ng build --prod after_success: true From d8e27600be59f45efb7958d88da440e1cf09a4bc Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 13:53:19 +0100 Subject: [PATCH 35/52] workaround for a bug in django-q: task results with too long names would not show up in the result lists. --- src/documents/forms.py | 2 +- src/documents/management/commands/document_consumer.py | 4 ++-- src/paperless_mail/mail.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/documents/forms.py b/src/documents/forms.py index 0471a8312..63dd307b2 100644 --- a/src/documents/forms.py +++ b/src/documents/forms.py @@ -56,4 +56,4 @@ class UploadForm(forms.Form): async_task("documents.tasks.consume_file", f.name, override_filename=original_filename, - task_name=os.path.basename(original_filename)) + task_name=os.path.basename(original_filename)[:100]) diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index 70c36a03c..05711ebd8 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -21,7 +21,7 @@ class Handler(FileSystemEventHandler): try: async_task("documents.tasks.consume_file", file, - task_name=os.path.basename(file)) + task_name=os.path.basename(file)[:100]) except Exception as e: # Catch all so that the consumer won't crash. logging.getLogger(__name__).error( @@ -71,7 +71,7 @@ class Command(BaseCommand): if entry.is_file(): async_task("documents.tasks.consume_file", entry.path, - task_name=os.path.basename(entry.path)) + task_name=os.path.basename(entry.path)[:100]) # Start the watchdog. Woof! if settings.CONSUMER_POLLING > 0: diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index dfdfa09ce..1ce4fe825 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -286,7 +286,7 @@ class MailAccountHandler(LoggingMixin): override_correspondent_id=correspondent.id if correspondent else None, # NOQA: E501 override_document_type_id=doc_type.id if doc_type else None, # NOQA: E501 override_tag_ids=[tag.id] if tag else None, - task_name=f"Mail: {att.filename}" + task_name=att.filename[:100] ) processed_attachments += 1 From e75f48d148a95eea4c00e0941ad7ec7fdf5c2d6e Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 13:55:31 +0100 Subject: [PATCH 36/52] changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bb119bf1f..38866c3e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,8 @@ next * New setting ``PAPERLESS_OCR_PAGES`` limits the tesseract parser to the first n pages of scanned documents. +* Fixed a bug where tasks with too long task names would not show + up in the admin. paperless-ng 0.9.1 ################## From 25f88b7ae945c48156efa39eaa63aad33cf46786 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 14:43:59 +0100 Subject: [PATCH 37/52] now using SCSS for better theming support --- src-ui/angular.json | 254 +++++++++--------- .../{app.component.css => app.component.scss} | 0 src-ui/src/app/app.component.ts | 2 +- ...component.css => app-frame.component.scss} | 0 .../app-frame/app-frame.component.ts | 2 +- ...onent.css => delete-dialog.component.scss} | 0 .../delete-dialog/delete-dialog.component.ts | 2 +- ...eck.component.css => check.component.scss} | 0 .../common/input/check/check.component.ts | 2 +- ...component.css => date-time.component.scss} | 0 .../input/date-time/date-time.component.ts | 2 +- ...ct.component.css => select.component.scss} | 0 .../common/input/select/select.component.ts | 2 +- ...tags.component.css => tags.component.scss} | 0 .../common/input/tags/tags.component.ts | 2 +- ...text.component.css => text.component.scss} | 0 .../common/input/text/text.component.ts | 2 +- ...mponent.css => page-header.component.scss} | 0 .../page-header/page-header.component.ts | 2 +- .../{tag.component.css => tag.component.scss} | 0 .../components/common/tag/tag.component.ts | 2 +- ...ts.component.css => toasts.component.scss} | 0 .../common/toasts/toasts.component.ts | 2 +- ...component.css => dashboard.component.scss} | 0 .../dashboard/dashboard.component.ts | 2 +- ...ent.css => document-detail.component.scss} | 0 .../document-detail.component.ts | 2 +- ...css => document-card-large.component.scss} | 0 .../document-card-large.component.ts | 2 +- ...css => document-card-small.component.scss} | 0 .../document-card-small.component.ts | 2 +- ...onent.css => document-list.component.scss} | 0 .../document-list/document-list.component.ts | 2 +- ...=> save-view-config-dialog.component.scss} | 0 .../save-view-config-dialog.component.ts | 2 +- ...onent.css => filter-editor.component.scss} | 0 .../filter-editor/filter-editor.component.ts | 2 +- ... correspondent-edit-dialog.component.scss} | 0 .../correspondent-edit-dialog.component.ts | 2 +- ....css => correspondent-list.component.scss} | 0 .../correspondent-list.component.ts | 2 +- ... document-type-edit-dialog.component.scss} | 0 .../document-type-edit-dialog.component.ts | 2 +- ....css => document-type-list.component.scss} | 0 .../document-type-list.component.ts | 2 +- ...logs.component.css => logs.component.scss} | 0 .../components/manage/logs/logs.component.ts | 2 +- ....component.css => settings.component.scss} | 0 .../manage/settings/settings.component.ts | 2 +- ...ent.css => tag-edit-dialog.component.scss} | 0 .../tag-edit-dialog.component.ts | 2 +- ....component.css => tag-list.component.scss} | 0 .../manage/tag-list/tag-list.component.ts | 2 +- ...component.css => not-found.component.scss} | 0 .../not-found/not-found.component.ts | 2 +- ...t.css => result-hightlight.component.scss} | 0 .../result-hightlight.component.ts | 2 +- ...ch.component.css => search.component.scss} | 0 .../app/components/search/search.component.ts | 2 +- src-ui/src/{styles.css => styles.scss} | 2 + 60 files changed, 160 insertions(+), 154 deletions(-) rename src-ui/src/app/{app.component.css => app.component.scss} (100%) rename src-ui/src/app/components/app-frame/{app-frame.component.css => app-frame.component.scss} (100%) rename src-ui/src/app/components/common/delete-dialog/{delete-dialog.component.css => delete-dialog.component.scss} (100%) rename src-ui/src/app/components/common/input/check/{check.component.css => check.component.scss} (100%) rename src-ui/src/app/components/common/input/date-time/{date-time.component.css => date-time.component.scss} (100%) rename src-ui/src/app/components/common/input/select/{select.component.css => select.component.scss} (100%) rename src-ui/src/app/components/common/input/tags/{tags.component.css => tags.component.scss} (100%) rename src-ui/src/app/components/common/input/text/{text.component.css => text.component.scss} (100%) rename src-ui/src/app/components/common/page-header/{page-header.component.css => page-header.component.scss} (100%) rename src-ui/src/app/components/common/tag/{tag.component.css => tag.component.scss} (100%) rename src-ui/src/app/components/common/toasts/{toasts.component.css => toasts.component.scss} (100%) rename src-ui/src/app/components/dashboard/{dashboard.component.css => dashboard.component.scss} (100%) rename src-ui/src/app/components/document-detail/{document-detail.component.css => document-detail.component.scss} (100%) rename src-ui/src/app/components/document-list/document-card-large/{document-card-large.component.css => document-card-large.component.scss} (100%) rename src-ui/src/app/components/document-list/document-card-small/{document-card-small.component.css => document-card-small.component.scss} (100%) rename src-ui/src/app/components/document-list/{document-list.component.css => document-list.component.scss} (100%) rename src-ui/src/app/components/document-list/save-view-config-dialog/{save-view-config-dialog.component.css => save-view-config-dialog.component.scss} (100%) rename src-ui/src/app/components/filter-editor/{filter-editor.component.css => filter-editor.component.scss} (100%) rename src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/{correspondent-edit-dialog.component.css => correspondent-edit-dialog.component.scss} (100%) rename src-ui/src/app/components/manage/correspondent-list/{correspondent-list.component.css => correspondent-list.component.scss} (100%) rename src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/{document-type-edit-dialog.component.css => document-type-edit-dialog.component.scss} (100%) rename src-ui/src/app/components/manage/document-type-list/{document-type-list.component.css => document-type-list.component.scss} (100%) rename src-ui/src/app/components/manage/logs/{logs.component.css => logs.component.scss} (100%) rename src-ui/src/app/components/manage/settings/{settings.component.css => settings.component.scss} (100%) rename src-ui/src/app/components/manage/tag-list/tag-edit-dialog/{tag-edit-dialog.component.css => tag-edit-dialog.component.scss} (100%) rename src-ui/src/app/components/manage/tag-list/{tag-list.component.css => tag-list.component.scss} (100%) rename src-ui/src/app/components/not-found/{not-found.component.css => not-found.component.scss} (100%) rename src-ui/src/app/components/search/result-hightlight/{result-hightlight.component.css => result-hightlight.component.scss} (100%) rename src-ui/src/app/components/search/{search.component.css => search.component.scss} (100%) rename src-ui/src/{styles.css => styles.scss} (99%) diff --git a/src-ui/angular.json b/src-ui/angular.json index aca54b8e0..2ff1bb3b0 100644 --- a/src-ui/angular.json +++ b/src-ui/angular.json @@ -1,126 +1,130 @@ { - "$schema": "./node_modules/@angular/cli/lib/config/schema.json", - "version": 1, - "newProjectRoot": "projects", - "projects": { - "paperless-ui": { - "projectType": "application", - "schematics": {}, - "root": "", - "sourceRoot": "src", - "prefix": "app", - "architect": { - "build": { - "builder": "@angular-devkit/build-angular:browser", - "options": { - "outputPath": "dist/paperless-ui", - "outputHashing": "none", - "index": "src/index.html", - "main": "src/main.ts", - "polyfills": "src/polyfills.ts", - "tsConfig": "tsconfig.app.json", - "aot": true, - "assets": [ - "src/favicon.ico", - "src/assets" - ], - "styles": [ - "node_modules/bootstrap/dist/css/bootstrap.min.css", - "src/styles.css" - ], - "scripts": [] - }, - "configurations": { - "production": { - "fileReplacements": [ - { - "replace": "src/environments/environment.ts", - "with": "src/environments/environment.prod.ts" - } - ], - "optimization": true, - "outputHashing": "none", - "sourceMap": false, - "extractCss": true, - "namedChunks": false, - "extractLicenses": true, - "vendorChunk": false, - "buildOptimizer": true, - "budgets": [ - { - "type": "initial", - "maximumWarning": "2mb", - "maximumError": "5mb" - }, - { - "type": "anyComponentStyle", - "maximumWarning": "6kb", - "maximumError": "10kb" - } - ] - } - } - }, - "serve": { - "builder": "@angular-devkit/build-angular:dev-server", - "options": { - "browserTarget": "paperless-ui:build" - }, - "configurations": { - "production": { - "browserTarget": "paperless-ui:build:production" - } - } - }, - "extract-i18n": { - "builder": "@angular-devkit/build-angular:extract-i18n", - "options": { - "browserTarget": "paperless-ui:build" - } - }, - "test": { - "builder": "@angular-devkit/build-angular:karma", - "options": { - "main": "src/test.ts", - "polyfills": "src/polyfills.ts", - "tsConfig": "tsconfig.spec.json", - "karmaConfig": "karma.conf.js", - "assets": [ - "src/favicon.ico", - "src/assets" - ], - "styles": [ - "src/styles.css" - ], - "scripts": [] - } - }, - "lint": { - "builder": "@angular-devkit/build-angular:tslint", - "options": { - "tsConfig": [ - "tsconfig.app.json", - "tsconfig.spec.json", - "e2e/tsconfig.json" - ], - "exclude": [ - "**/node_modules/**" - ] - } - }, - "e2e": { - "builder": "@angular-devkit/build-angular:protractor", - "options": { - "protractorConfig": "e2e/protractor.conf.js", - "devServerTarget": "paperless-ui:serve" - }, - "configurations": { - "production": { - "devServerTarget": "paperless-ui:serve:production" - } - } - } - } - }}, - "defaultProject": "paperless-ui" -} + "$schema": "./node_modules/@angular/cli/lib/config/schema.json", + "version": 1, + "newProjectRoot": "projects", + "projects": { + "paperless-ui": { + "projectType": "application", + "schematics": { + "@schematics/angular:component": { + "style": "scss" + } + }, + "root": "", + "sourceRoot": "src", + "prefix": "app", + "architect": { + "build": { + "builder": "@angular-devkit/build-angular:browser", + "options": { + "outputPath": "dist/paperless-ui", + "outputHashing": "none", + "index": "src/index.html", + "main": "src/main.ts", + "polyfills": "src/polyfills.ts", + "tsConfig": "tsconfig.app.json", + "aot": true, + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.scss" + ], + "scripts": [] + }, + "configurations": { + "production": { + "fileReplacements": [ + { + "replace": "src/environments/environment.ts", + "with": "src/environments/environment.prod.ts" + } + ], + "optimization": true, + "outputHashing": "none", + "sourceMap": false, + "extractCss": true, + "namedChunks": false, + "extractLicenses": true, + "vendorChunk": false, + "buildOptimizer": true, + "budgets": [ + { + "type": "initial", + "maximumWarning": "2mb", + "maximumError": "5mb" + }, + { + "type": "anyComponentStyle", + "maximumWarning": "6kb", + "maximumError": "10kb" + } + ] + } + } + }, + "serve": { + "builder": "@angular-devkit/build-angular:dev-server", + "options": { + "browserTarget": "paperless-ui:build" + }, + "configurations": { + "production": { + "browserTarget": "paperless-ui:build:production" + } + } + }, + "extract-i18n": { + "builder": "@angular-devkit/build-angular:extract-i18n", + "options": { + "browserTarget": "paperless-ui:build" + } + }, + "test": { + "builder": "@angular-devkit/build-angular:karma", + "options": { + "main": "src/test.ts", + "polyfills": "src/polyfills.ts", + "tsConfig": "tsconfig.spec.json", + "karmaConfig": "karma.conf.js", + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.scss" + ], + "scripts": [] + } + }, + "lint": { + "builder": "@angular-devkit/build-angular:tslint", + "options": { + "tsConfig": [ + "tsconfig.app.json", + "tsconfig.spec.json", + "e2e/tsconfig.json" + ], + "exclude": [ + "**/node_modules/**" + ] + } + }, + "e2e": { + "builder": "@angular-devkit/build-angular:protractor", + "options": { + "protractorConfig": "e2e/protractor.conf.js", + "devServerTarget": "paperless-ui:serve" + }, + "configurations": { + "production": { + "devServerTarget": "paperless-ui:serve:production" + } + } + } + } + } + }, + "defaultProject": "paperless-ui" +} \ No newline at end of file diff --git a/src-ui/src/app/app.component.css b/src-ui/src/app/app.component.scss similarity index 100% rename from src-ui/src/app/app.component.css rename to src-ui/src/app/app.component.scss diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts index a6cd8bebe..84c173a18 100644 --- a/src-ui/src/app/app.component.ts +++ b/src-ui/src/app/app.component.ts @@ -3,7 +3,7 @@ import { Component } from '@angular/core'; @Component({ selector: 'app-root', templateUrl: './app.component.html', - styleUrls: ['./app.component.css'] + styleUrls: ['./app.component.scss'] }) export class AppComponent { diff --git a/src-ui/src/app/components/app-frame/app-frame.component.css b/src-ui/src/app/components/app-frame/app-frame.component.scss similarity index 100% rename from src-ui/src/app/components/app-frame/app-frame.component.css rename to src-ui/src/app/components/app-frame/app-frame.component.scss diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index be72ad469..a7755ffeb 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -12,7 +12,7 @@ import { DocumentDetailComponent } from '../document-detail/document-detail.comp @Component({ selector: 'app-app-frame', templateUrl: './app-frame.component.html', - styleUrls: ['./app-frame.component.css'] + styleUrls: ['./app-frame.component.scss'] }) export class AppFrameComponent implements OnInit, OnDestroy { diff --git a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.css b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.scss similarity index 100% rename from src-ui/src/app/components/common/delete-dialog/delete-dialog.component.css rename to src-ui/src/app/components/common/delete-dialog/delete-dialog.component.scss diff --git a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts index dd00b45f2..20114c78c 100644 --- a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts +++ b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts @@ -4,7 +4,7 @@ import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; @Component({ selector: 'app-delete-dialog', templateUrl: './delete-dialog.component.html', - styleUrls: ['./delete-dialog.component.css'] + styleUrls: ['./delete-dialog.component.scss'] }) export class DeleteDialogComponent implements OnInit { diff --git a/src-ui/src/app/components/common/input/check/check.component.css b/src-ui/src/app/components/common/input/check/check.component.scss similarity index 100% rename from src-ui/src/app/components/common/input/check/check.component.css rename to src-ui/src/app/components/common/input/check/check.component.scss diff --git a/src-ui/src/app/components/common/input/check/check.component.ts b/src-ui/src/app/components/common/input/check/check.component.ts index 28b7ea4db..de0b9a0d1 100644 --- a/src-ui/src/app/components/common/input/check/check.component.ts +++ b/src-ui/src/app/components/common/input/check/check.component.ts @@ -11,7 +11,7 @@ import { AbstractInputComponent } from '../abstract-input'; }], selector: 'app-input-check', templateUrl: './check.component.html', - styleUrls: ['./check.component.css'] + styleUrls: ['./check.component.scss'] }) export class CheckComponent extends AbstractInputComponent<boolean> { diff --git a/src-ui/src/app/components/common/input/date-time/date-time.component.css b/src-ui/src/app/components/common/input/date-time/date-time.component.scss similarity index 100% rename from src-ui/src/app/components/common/input/date-time/date-time.component.css rename to src-ui/src/app/components/common/input/date-time/date-time.component.scss diff --git a/src-ui/src/app/components/common/input/date-time/date-time.component.ts b/src-ui/src/app/components/common/input/date-time/date-time.component.ts index f8b66133a..07238e94f 100644 --- a/src-ui/src/app/components/common/input/date-time/date-time.component.ts +++ b/src-ui/src/app/components/common/input/date-time/date-time.component.ts @@ -11,7 +11,7 @@ import { AbstractInputComponent } from '../abstract-input'; }], selector: 'app-input-date-time', templateUrl: './date-time.component.html', - styleUrls: ['./date-time.component.css'] + styleUrls: ['./date-time.component.scss'] }) export class DateTimeComponent implements OnInit,ControlValueAccessor { diff --git a/src-ui/src/app/components/common/input/select/select.component.css b/src-ui/src/app/components/common/input/select/select.component.scss similarity index 100% rename from src-ui/src/app/components/common/input/select/select.component.css rename to src-ui/src/app/components/common/input/select/select.component.scss diff --git a/src-ui/src/app/components/common/input/select/select.component.ts b/src-ui/src/app/components/common/input/select/select.component.ts index c8e213722..e6e02ac87 100644 --- a/src-ui/src/app/components/common/input/select/select.component.ts +++ b/src-ui/src/app/components/common/input/select/select.component.ts @@ -10,7 +10,7 @@ import { AbstractInputComponent } from '../abstract-input'; }], selector: 'app-input-select', templateUrl: './select.component.html', - styleUrls: ['./select.component.css'] + styleUrls: ['./select.component.scss'] }) export class SelectComponent extends AbstractInputComponent<number> { diff --git a/src-ui/src/app/components/common/input/tags/tags.component.css b/src-ui/src/app/components/common/input/tags/tags.component.scss similarity index 100% rename from src-ui/src/app/components/common/input/tags/tags.component.css rename to src-ui/src/app/components/common/input/tags/tags.component.scss diff --git a/src-ui/src/app/components/common/input/tags/tags.component.ts b/src-ui/src/app/components/common/input/tags/tags.component.ts index dd57d8e50..81bd9d470 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.ts +++ b/src-ui/src/app/components/common/input/tags/tags.component.ts @@ -15,7 +15,7 @@ import { TagService } from 'src/app/services/rest/tag.service'; }], selector: 'app-input-tags', templateUrl: './tags.component.html', - styleUrls: ['./tags.component.css'] + styleUrls: ['./tags.component.scss'] }) export class TagsComponent implements OnInit, ControlValueAccessor { diff --git a/src-ui/src/app/components/common/input/text/text.component.css b/src-ui/src/app/components/common/input/text/text.component.scss similarity index 100% rename from src-ui/src/app/components/common/input/text/text.component.css rename to src-ui/src/app/components/common/input/text/text.component.scss diff --git a/src-ui/src/app/components/common/input/text/text.component.ts b/src-ui/src/app/components/common/input/text/text.component.ts index d4dc1c587..ffb8c0c3d 100644 --- a/src-ui/src/app/components/common/input/text/text.component.ts +++ b/src-ui/src/app/components/common/input/text/text.component.ts @@ -11,7 +11,7 @@ import { AbstractInputComponent } from '../abstract-input'; }], selector: 'app-input-text', templateUrl: './text.component.html', - styleUrls: ['./text.component.css'] + styleUrls: ['./text.component.scss'] }) export class TextComponent extends AbstractInputComponent<string> { diff --git a/src-ui/src/app/components/common/page-header/page-header.component.css b/src-ui/src/app/components/common/page-header/page-header.component.scss similarity index 100% rename from src-ui/src/app/components/common/page-header/page-header.component.css rename to src-ui/src/app/components/common/page-header/page-header.component.scss diff --git a/src-ui/src/app/components/common/page-header/page-header.component.ts b/src-ui/src/app/components/common/page-header/page-header.component.ts index 6046a6452..d1218ec8f 100644 --- a/src-ui/src/app/components/common/page-header/page-header.component.ts +++ b/src-ui/src/app/components/common/page-header/page-header.component.ts @@ -3,7 +3,7 @@ import { Component, Input, OnInit } from '@angular/core'; @Component({ selector: 'app-page-header', templateUrl: './page-header.component.html', - styleUrls: ['./page-header.component.css'] + styleUrls: ['./page-header.component.scss'] }) export class PageHeaderComponent implements OnInit { diff --git a/src-ui/src/app/components/common/tag/tag.component.css b/src-ui/src/app/components/common/tag/tag.component.scss similarity index 100% rename from src-ui/src/app/components/common/tag/tag.component.css rename to src-ui/src/app/components/common/tag/tag.component.scss diff --git a/src-ui/src/app/components/common/tag/tag.component.ts b/src-ui/src/app/components/common/tag/tag.component.ts index a7f81fa0a..ec59e86f3 100644 --- a/src-ui/src/app/components/common/tag/tag.component.ts +++ b/src-ui/src/app/components/common/tag/tag.component.ts @@ -4,7 +4,7 @@ import { TAG_COLOURS, PaperlessTag } from 'src/app/data/paperless-tag'; @Component({ selector: 'app-tag', templateUrl: './tag.component.html', - styleUrls: ['./tag.component.css'] + styleUrls: ['./tag.component.scss'] }) export class TagComponent implements OnInit { diff --git a/src-ui/src/app/components/common/toasts/toasts.component.css b/src-ui/src/app/components/common/toasts/toasts.component.scss similarity index 100% rename from src-ui/src/app/components/common/toasts/toasts.component.css rename to src-ui/src/app/components/common/toasts/toasts.component.scss diff --git a/src-ui/src/app/components/common/toasts/toasts.component.ts b/src-ui/src/app/components/common/toasts/toasts.component.ts index 53abf1f1a..8f142646d 100644 --- a/src-ui/src/app/components/common/toasts/toasts.component.ts +++ b/src-ui/src/app/components/common/toasts/toasts.component.ts @@ -5,7 +5,7 @@ import { Toast, ToastService } from 'src/app/services/toast.service'; @Component({ selector: 'app-toasts', templateUrl: './toasts.component.html', - styleUrls: ['./toasts.component.css'] + styleUrls: ['./toasts.component.scss'] }) export class ToastsComponent implements OnInit, OnDestroy { diff --git a/src-ui/src/app/components/dashboard/dashboard.component.css b/src-ui/src/app/components/dashboard/dashboard.component.scss similarity index 100% rename from src-ui/src/app/components/dashboard/dashboard.component.css rename to src-ui/src/app/components/dashboard/dashboard.component.scss diff --git a/src-ui/src/app/components/dashboard/dashboard.component.ts b/src-ui/src/app/components/dashboard/dashboard.component.ts index f8d5fb0ae..5bedf6389 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.ts +++ b/src-ui/src/app/components/dashboard/dashboard.component.ts @@ -15,7 +15,7 @@ export interface Statistics { @Component({ selector: 'app-dashboard', templateUrl: './dashboard.component.html', - styleUrls: ['./dashboard.component.css'] + styleUrls: ['./dashboard.component.scss'] }) export class DashboardComponent implements OnInit { diff --git a/src-ui/src/app/components/document-detail/document-detail.component.css b/src-ui/src/app/components/document-detail/document-detail.component.scss similarity index 100% rename from src-ui/src/app/components/document-detail/document-detail.component.css rename to src-ui/src/app/components/document-detail/document-detail.component.scss diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index e87445705..00b840b48 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -21,7 +21,7 @@ import { TagEditDialogComponent } from '../manage/tag-list/tag-edit-dialog/tag-e @Component({ selector: 'app-document-detail', templateUrl: './document-detail.component.html', - styleUrls: ['./document-detail.component.css'] + styleUrls: ['./document-detail.component.scss'] }) export class DocumentDetailComponent implements OnInit { diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.css b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss similarity index 100% rename from src-ui/src/app/components/document-list/document-card-large/document-card-large.component.css rename to src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts index c05b1f039..1c1e110c0 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts @@ -6,7 +6,7 @@ import { DocumentService } from 'src/app/services/rest/document.service'; @Component({ selector: 'app-document-card-large', templateUrl: './document-card-large.component.html', - styleUrls: ['./document-card-large.component.css'] + styleUrls: ['./document-card-large.component.scss'] }) export class DocumentCardLargeComponent implements OnInit { diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.css b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss similarity index 100% rename from src-ui/src/app/components/document-list/document-card-small/document-card-small.component.css rename to src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts index 939f28bf7..1ca9847b9 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts @@ -5,7 +5,7 @@ import { DocumentService } from 'src/app/services/rest/document.service'; @Component({ selector: 'app-document-card-small', templateUrl: './document-card-small.component.html', - styleUrls: ['./document-card-small.component.css'] + styleUrls: ['./document-card-small.component.scss'] }) export class DocumentCardSmallComponent implements OnInit { diff --git a/src-ui/src/app/components/document-list/document-list.component.css b/src-ui/src/app/components/document-list/document-list.component.scss similarity index 100% rename from src-ui/src/app/components/document-list/document-list.component.css rename to src-ui/src/app/components/document-list/document-list.component.scss diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index 2ea2c9e3e..2b155c34f 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -11,7 +11,7 @@ import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-vi @Component({ selector: 'app-document-list', templateUrl: './document-list.component.html', - styleUrls: ['./document-list.component.css'] + styleUrls: ['./document-list.component.scss'] }) export class DocumentListComponent implements OnInit { diff --git a/src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.css b/src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.scss similarity index 100% rename from src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.css rename to src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.scss diff --git a/src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.ts b/src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.ts index 6fcdbd2c8..0dd351770 100644 --- a/src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.ts +++ b/src-ui/src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.ts @@ -5,7 +5,7 @@ import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; @Component({ selector: 'app-save-view-config-dialog', templateUrl: './save-view-config-dialog.component.html', - styleUrls: ['./save-view-config-dialog.component.css'] + styleUrls: ['./save-view-config-dialog.component.scss'] }) export class SaveViewConfigDialogComponent implements OnInit { diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.css b/src-ui/src/app/components/filter-editor/filter-editor.component.scss similarity index 100% rename from src-ui/src/app/components/filter-editor/filter-editor.component.css rename to src-ui/src/app/components/filter-editor/filter-editor.component.scss diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/filter-editor/filter-editor.component.ts index dc23450c5..9a104c465 100644 --- a/src-ui/src/app/components/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/filter-editor/filter-editor.component.ts @@ -12,7 +12,7 @@ import { TagService } from 'src/app/services/rest/tag.service'; @Component({ selector: 'app-filter-editor', templateUrl: './filter-editor.component.html', - styleUrls: ['./filter-editor.component.css'] + styleUrls: ['./filter-editor.component.scss'] }) export class FilterEditorComponent implements OnInit { diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.css b/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.scss similarity index 100% rename from src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.css rename to src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.scss diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.ts b/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.ts index f3f768044..855fc159c 100644 --- a/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.ts +++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.ts @@ -9,7 +9,7 @@ import { ToastService } from 'src/app/services/toast.service'; @Component({ selector: 'app-correspondent-edit-dialog', templateUrl: './correspondent-edit-dialog.component.html', - styleUrls: ['./correspondent-edit-dialog.component.css'] + styleUrls: ['./correspondent-edit-dialog.component.scss'] }) export class CorrespondentEditDialogComponent extends EditDialogComponent<PaperlessCorrespondent> { diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.css b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.scss similarity index 100% rename from src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.css rename to src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.scss diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts index f867a10e3..83aa5d2cc 100644 --- a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts +++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts @@ -8,7 +8,7 @@ import { CorrespondentEditDialogComponent } from './correspondent-edit-dialog/co @Component({ selector: 'app-correspondent-list', templateUrl: './correspondent-list.component.html', - styleUrls: ['./correspondent-list.component.css'] + styleUrls: ['./correspondent-list.component.scss'] }) export class CorrespondentListComponent extends GenericListComponent<PaperlessCorrespondent> { diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.css b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.scss similarity index 100% rename from src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.css rename to src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.scss diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.ts b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.ts index 88b7d6da5..087eede8c 100644 --- a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.ts +++ b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.ts @@ -9,7 +9,7 @@ import { ToastService } from 'src/app/services/toast.service'; @Component({ selector: 'app-document-type-edit-dialog', templateUrl: './document-type-edit-dialog.component.html', - styleUrls: ['./document-type-edit-dialog.component.css'] + styleUrls: ['./document-type-edit-dialog.component.scss'] }) export class DocumentTypeEditDialogComponent extends EditDialogComponent<PaperlessDocumentType> { diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-list.component.css b/src-ui/src/app/components/manage/document-type-list/document-type-list.component.scss similarity index 100% rename from src-ui/src/app/components/manage/document-type-list/document-type-list.component.css rename to src-ui/src/app/components/manage/document-type-list/document-type-list.component.scss diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts b/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts index ae502efc2..733d2c44b 100644 --- a/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts +++ b/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts @@ -8,7 +8,7 @@ import { DocumentTypeEditDialogComponent } from './document-type-edit-dialog/doc @Component({ selector: 'app-document-type-list', templateUrl: './document-type-list.component.html', - styleUrls: ['./document-type-list.component.css'] + styleUrls: ['./document-type-list.component.scss'] }) export class DocumentTypeListComponent extends GenericListComponent<PaperlessDocumentType> { diff --git a/src-ui/src/app/components/manage/logs/logs.component.css b/src-ui/src/app/components/manage/logs/logs.component.scss similarity index 100% rename from src-ui/src/app/components/manage/logs/logs.component.css rename to src-ui/src/app/components/manage/logs/logs.component.scss diff --git a/src-ui/src/app/components/manage/logs/logs.component.ts b/src-ui/src/app/components/manage/logs/logs.component.ts index f80aada21..d52b90a5a 100644 --- a/src-ui/src/app/components/manage/logs/logs.component.ts +++ b/src-ui/src/app/components/manage/logs/logs.component.ts @@ -6,7 +6,7 @@ import { LogService } from 'src/app/services/rest/log.service'; @Component({ selector: 'app-logs', templateUrl: './logs.component.html', - styleUrls: ['./logs.component.css'] + styleUrls: ['./logs.component.scss'] }) export class LogsComponent implements OnInit { diff --git a/src-ui/src/app/components/manage/settings/settings.component.css b/src-ui/src/app/components/manage/settings/settings.component.scss similarity index 100% rename from src-ui/src/app/components/manage/settings/settings.component.css rename to src-ui/src/app/components/manage/settings/settings.component.scss diff --git a/src-ui/src/app/components/manage/settings/settings.component.ts b/src-ui/src/app/components/manage/settings/settings.component.ts index e14c2bd7b..1b93268fc 100644 --- a/src-ui/src/app/components/manage/settings/settings.component.ts +++ b/src-ui/src/app/components/manage/settings/settings.component.ts @@ -8,7 +8,7 @@ import { SavedViewConfigService } from 'src/app/services/saved-view-config.servi @Component({ selector: 'app-settings', templateUrl: './settings.component.html', - styleUrls: ['./settings.component.css'] + styleUrls: ['./settings.component.scss'] }) export class SettingsComponent implements OnInit { diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.css b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.scss similarity index 100% rename from src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.css rename to src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.scss diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.ts b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.ts index 7aee39e77..bb0162608 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.ts +++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.ts @@ -9,7 +9,7 @@ import { ToastService } from 'src/app/services/toast.service'; @Component({ selector: 'app-tag-edit-dialog', templateUrl: './tag-edit-dialog.component.html', - styleUrls: ['./tag-edit-dialog.component.css'] + styleUrls: ['./tag-edit-dialog.component.scss'] }) export class TagEditDialogComponent extends EditDialogComponent<PaperlessTag> { diff --git a/src-ui/src/app/components/manage/tag-list/tag-list.component.css b/src-ui/src/app/components/manage/tag-list/tag-list.component.scss similarity index 100% rename from src-ui/src/app/components/manage/tag-list/tag-list.component.css rename to src-ui/src/app/components/manage/tag-list/tag-list.component.scss diff --git a/src-ui/src/app/components/manage/tag-list/tag-list.component.ts b/src-ui/src/app/components/manage/tag-list/tag-list.component.ts index 88fc03a59..761a9484c 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-list.component.ts +++ b/src-ui/src/app/components/manage/tag-list/tag-list.component.ts @@ -9,7 +9,7 @@ import { TagEditDialogComponent } from './tag-edit-dialog/tag-edit-dialog.compon @Component({ selector: 'app-tag-list', templateUrl: './tag-list.component.html', - styleUrls: ['./tag-list.component.css'] + styleUrls: ['./tag-list.component.scss'] }) export class TagListComponent extends GenericListComponent<PaperlessTag> { diff --git a/src-ui/src/app/components/not-found/not-found.component.css b/src-ui/src/app/components/not-found/not-found.component.scss similarity index 100% rename from src-ui/src/app/components/not-found/not-found.component.css rename to src-ui/src/app/components/not-found/not-found.component.scss diff --git a/src-ui/src/app/components/not-found/not-found.component.ts b/src-ui/src/app/components/not-found/not-found.component.ts index 33da0a492..7cb4124f1 100644 --- a/src-ui/src/app/components/not-found/not-found.component.ts +++ b/src-ui/src/app/components/not-found/not-found.component.ts @@ -3,7 +3,7 @@ import { Component, OnInit } from '@angular/core'; @Component({ selector: 'app-not-found', templateUrl: './not-found.component.html', - styleUrls: ['./not-found.component.css'] + styleUrls: ['./not-found.component.scss'] }) export class NotFoundComponent implements OnInit { diff --git a/src-ui/src/app/components/search/result-hightlight/result-hightlight.component.css b/src-ui/src/app/components/search/result-hightlight/result-hightlight.component.scss similarity index 100% rename from src-ui/src/app/components/search/result-hightlight/result-hightlight.component.css rename to src-ui/src/app/components/search/result-hightlight/result-hightlight.component.scss diff --git a/src-ui/src/app/components/search/result-hightlight/result-hightlight.component.ts b/src-ui/src/app/components/search/result-hightlight/result-hightlight.component.ts index 0f20c93cc..cd37448e0 100644 --- a/src-ui/src/app/components/search/result-hightlight/result-hightlight.component.ts +++ b/src-ui/src/app/components/search/result-hightlight/result-hightlight.component.ts @@ -4,7 +4,7 @@ import { SearchHitHighlight } from 'src/app/data/search-result'; @Component({ selector: 'app-result-hightlight', templateUrl: './result-hightlight.component.html', - styleUrls: ['./result-hightlight.component.css'] + styleUrls: ['./result-hightlight.component.scss'] }) export class ResultHightlightComponent implements OnInit { diff --git a/src-ui/src/app/components/search/search.component.css b/src-ui/src/app/components/search/search.component.scss similarity index 100% rename from src-ui/src/app/components/search/search.component.css rename to src-ui/src/app/components/search/search.component.scss diff --git a/src-ui/src/app/components/search/search.component.ts b/src-ui/src/app/components/search/search.component.ts index bd15611c6..f8c5d6cdc 100644 --- a/src-ui/src/app/components/search/search.component.ts +++ b/src-ui/src/app/components/search/search.component.ts @@ -6,7 +6,7 @@ import { SearchService } from 'src/app/services/rest/search.service'; @Component({ selector: 'app-search', templateUrl: './search.component.html', - styleUrls: ['./search.component.css'] + styleUrls: ['./search.component.scss'] }) export class SearchComponent implements OnInit { diff --git a/src-ui/src/styles.css b/src-ui/src/styles.scss similarity index 99% rename from src-ui/src/styles.css rename to src-ui/src/styles.scss index c7849912e..454f7d22a 100644 --- a/src-ui/src/styles.css +++ b/src-ui/src/styles.scss @@ -1,3 +1,5 @@ +@import "bootstrap"; + .toolbaricon { width: 1.2em; From 8650af05f7632d998b789cbf18c8787933e18dae Mon Sep 17 00:00:00 2001 From: Jonas Winkler <jonas.winkler@jpwinkler.de> Date: Sun, 22 Nov 2020 16:33:26 +0100 Subject: [PATCH 38/52] many layout and theme changes --- .../app-frame/app-frame.component.html | 24 +++-- .../app-frame/app-frame.component.scss | 9 +- .../common/input/check/check.component.html | 6 +- .../page-header/page-header.component.html | 5 +- .../page-header/page-header.component.ts | 3 + .../dashboard/dashboard.component.html | 94 ++++++++++++------- .../document-detail.component.html | 4 +- .../document-list.component.html | 18 ++-- .../correspondent-list.component.html | 2 +- .../document-type-list.component.html | 2 +- .../manage/logs/logs.component.html | 2 +- .../manage/tag-list/tag-list.component.html | 2 +- src-ui/src/index.html | 2 +- src-ui/src/styles.scss | 9 +- src-ui/src/theme.scss | 6 ++ 15 files changed, 114 insertions(+), 74 deletions(-) create mode 100644 src-ui/src/theme.scss diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 1232ecf12..05289becc 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -1,24 +1,14 @@ -<nav class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0 shadow"> +<nav class="navbar navbar-dark sticky-top bg-primary flex-md-nowrap p-0 shadow"> <span class="navbar-brand col-md-3 col-lg-2 mr-0 px-3" href="#">Paperless-ng</span> <button class="navbar-toggler position-absolute d-md-none collapsed" type="button" data-toggle="collapse" data-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation" (click)="isMenuCollapsed = !isMenuCollapsed"> <span class="navbar-toggler-icon"></span> </button> - <form (ngSubmit)="search()" class="w-100"> + <form (ngSubmit)="search()" class="w-100 m-1"> <input class="form-control form-control-dark" type="text" placeholder="Search" aria-label="Search" [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (selectItem)="itemSelected($event)"> </form> - <ul class="navbar-nav px-3"> - <li class="nav-item text-nowrap"> - <a class="nav-link" href="accounts/logout/"> - <svg class="buttonicon" fill="currentColor"> - <use xlink:href="assets/bootstrap-icons.svg#door-closed"/> - </svg> - Logout - </a> - </li> - </ul> </nav> <div class="container-fluid"> @@ -151,7 +141,15 @@ <svg class="sidebaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#link"/> </svg> - Github + GitHub + </a> + </li> + <li class="nav-item"> + <a class="nav-link" href="accounts/logout/"> + <svg class="sidebaricon" fill="currentColor"> + <use xlink:href="assets/bootstrap-icons.svg#door-open"/> + </svg> + Logout </a> </li> </ul> diff --git a/src-ui/src/app/components/app-frame/app-frame.component.scss b/src-ui/src/app/components/app-frame/app-frame.component.scss index c29213992..87dcb8fe3 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.scss +++ b/src-ui/src/app/components/app-frame/app-frame.component.scss @@ -1,4 +1,6 @@ +@import "/src/theme"; + /* * Sidebar */ @@ -15,14 +17,15 @@ @media (max-width: 767.98px) { .sidebar { - top: 5rem; + top: 3rem; } } .sidebar-sticky { position: relative; top: 0; - height: calc(100vh - 48px); + /* height: calc(100vh - 48px); */ + height: 100%; padding-top: .5rem; overflow-x: hidden; overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */ @@ -46,7 +49,7 @@ } .sidebar .nav-link.active { - color: #007bff; + color: $primary; } .sidebar .nav-link:hover .sidebaricon, diff --git a/src-ui/src/app/components/common/input/check/check.component.html b/src-ui/src/app/components/common/input/check/check.component.html index 4276eed7c..88bde649e 100644 --- a/src-ui/src/app/components/common/input/check/check.component.html +++ b/src-ui/src/app/components/common/input/check/check.component.html @@ -1,5 +1,5 @@ -<div class="form-group form-check"> - <input type="checkbox" class="form-check-input" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)" (blur)="onTouched()" [disabled]="disabled"> - <label class="form-check-label" [for]="inputId">{{title}}</label> +<div class="form-group custom-control custom-checkbox"> + <input type="checkbox" class="custom-control-input" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)" (blur)="onTouched()" [disabled]="disabled"> + <label class="custom-control-label" [for]="inputId">{{title}}</label> <small *ngIf="hint" class="form-text text-muted">{{hint}}</small> </div> \ No newline at end of file diff --git a/src-ui/src/app/components/common/page-header/page-header.component.html b/src-ui/src/app/components/common/page-header/page-header.component.html index 54386422a..fccce9b98 100644 --- a/src-ui/src/app/components/common/page-header/page-header.component.html +++ b/src-ui/src/app/components/common/page-header/page-header.component.html @@ -1,6 +1,7 @@ <div class="row pt-3 pb-1 mb-3 border-bottom align-items-center" > - <div class="col text-truncate"> - <h1 class="h2 text-truncate" style="line-height: 1.4">{{title}}</h1> + <div class="col-md text-truncate"> + <p class="h2 text-truncate" style="line-height: 1.4">{{title}}</p> + <p *ngIf="subTitle" class="h5 text-truncate" style="line-height: 1.4">{{subTitle}}</p> </div> <div class="btn-toolbar col-auto"> <ng-content></ng-content> diff --git a/src-ui/src/app/components/common/page-header/page-header.component.ts b/src-ui/src/app/components/common/page-header/page-header.component.ts index d1218ec8f..93ec3bfb7 100644 --- a/src-ui/src/app/components/common/page-header/page-header.component.ts +++ b/src-ui/src/app/components/common/page-header/page-header.component.ts @@ -12,6 +12,9 @@ export class PageHeaderComponent implements OnInit { @Input() title: string = "" + @Input() + subTitle: string = "" + ngOnInit(): void { } diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html index 033141f2a..3fad2af04 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.html +++ b/src-ui/src/app/components/dashboard/dashboard.component.html @@ -1,50 +1,74 @@ - -<app-page-header title="Dashboard"> +<app-page-header title="Dashboard" subTitle="Welcome to paperless-ng!"> + <img src="assets/logo.svg" height="80" class="m-2"> </app-page-header> -<p>Welcome to paperless-ng!</p> - <div class='row'> <div class="col-lg"> <ng-container *ngFor="let v of savedDashboardViews"> - <h4>{{v.viewConfig.title}}</h4> - <table class="table table-sm table-hover table-borderless"> - <thead> - <tr> - <th>Created</th> - <th scope="col">Title</th> - </tr> - </thead> - <tbody> - <tr *ngFor="let doc of v.documents" routerLink="/documents/{{doc.id}}"> - <td>{{doc.created | date}}</td> - <td>{{doc.title}}<app-tag [tag]="t" *ngFor="let t of doc.tags" class="ml-1"></app-tag> - </tr> - </tbody> - </table> + <div class="card mb-3"> + <div class="card-header"> + <h5 class="card-title mb-0">{{v.viewConfig.title}}</h5> + </div> + <div class="card-body text-dark"> + <table class="table table-sm table-hover table-borderless"> + <thead> + <tr> + <th>Created</th> + <th scope="col">Title</th> + </tr> + </thead> + <tbody> + <tr *ngFor="let doc of v.documents" routerLink="/documents/{{doc.id}}"> + <td>{{doc.created | date}}</td> + <td>{{doc.title}}<app-tag [tag]="t" *ngFor="let t of doc.tags" class="ml-1"></app-tag> + </tr> + </tbody> + </table> + </div> + </div> </ng-container> <ng-container *ngIf="savedDashboardViews.length == 0"> - <h4>Saved views</h4> - <p>This space is reserved to display your saved views. Go to your documents and save a view to have it displayed here!</p> + <div class="card mb-3"> + <div class="card-header"> + <h5 class="card-title mb-0">Saved views</h5> + </div> + <div class="card-body text-dark"> + <p class="card-text">This space is reserved to display your saved views. Go to your documents and save a view + to have it displayed + here!</p> + </div> + </div> + </ng-container> </div> <div class="col-lg"> - <h4>Statistics</h4> - <p>Documents in inbox: {{statistics.documents_inbox}}</p> - <p>Total documents: {{statistics.documents_total}}</p> - <h4>Upload new Document</h4> - <form> - <ngx-file-drop - dropZoneLabel="Drop documents here" - (onFileDrop)="dropped($event)" - (onFileOver)="fileOver($event)" - (onFileLeave)="fileLeave($event)" - dropZoneClassName="bg-light mt-4 card"> - </ngx-file-drop> - </form> + <div class="card mb-3"> + <div class="card-header"> + <h5 class="card-title mb-0">Statistics</h5> + </div> + <div class="card-body text-dark"> + <p class="card-text">Documents in inbox: {{statistics.documents_inbox}}</p> + <p class="card-text">Total documents: {{statistics.documents_total}}</p> + </div> + </div> + + <div class="card mb-3"> + <div class="card-header"> + <h5 class="card-title mb-0">Upload new documents</h5> + </div> + <div class="card-body text-dark"> + <form> + <ngx-file-drop dropZoneLabel="Drop documents here" (onFileDrop)="dropped($event)" + (onFileOver)="fileOver($event)" (onFileLeave)="fileLeave($event)" dropZoneClassName="bg-light mt-4 card"> + + </ngx-file-drop> + </form> + </div> + </div> + </div> -</div> +</div> \ No newline at end of file diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index b460e7f97..9e1f8ad71 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -5,13 +5,13 @@ </svg> <span class="d-none d-lg-inline"> Delete</span> </button> - <a [href]="downloadUrl" class="btn btn-sm btn-outline-secondary mr-2"> + <a [href]="downloadUrl" class="btn btn-sm btn-outline-primary mr-2"> <svg class="buttonicon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#download" /> </svg> <span class="d-none d-lg-inline"> Download</span> </a> - <button type="button" class="btn btn-sm btn-outline-secondary" (click)="close()"> + <button type="button" class="btn btn-sm btn-outline-primary" (click)="close()"> <svg class="buttonicon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#x" /> </svg> diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index 67bd6b7c2..17d0b7ff3 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -2,19 +2,19 @@ <div class="btn-group btn-group-toggle mr-2" ngbRadioGroup [(ngModel)]="displayMode" (ngModelChange)="saveDisplayMode()"> - <label ngbButtonLabel class="btn-outline-secondary btn-sm"> + <label ngbButtonLabel class="btn-outline-primary btn-sm"> <input ngbButton type="radio" class="btn btn-sm" value="details"> <svg class="toolbaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#list-ul" /> </svg> </label> - <label ngbButtonLabel class="btn-outline-secondary btn-sm"> + <label ngbButtonLabel class="btn-outline-primary btn-sm"> <input ngbButton type="radio" class="btn btn-sm" value="smallCards"> <svg class="toolbaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#grid" /> </svg> </label> - <label ngbButtonLabel class="btn-outline-secondary btn-sm"> + <label ngbButtonLabel class="btn-outline-primary btn-sm"> <input ngbButton type="radio" class="btn btn-sm" value="largeCards"> <svg class="toolbaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#hdd-stack" /> @@ -24,19 +24,19 @@ <div class="btn-group btn-group-toggle mr-2" ngbRadioGroup [(ngModel)]="docs.sortDirection" *ngIf="!docs.viewId"> <div ngbDropdown class="btn-group"> - <button class="btn btn-outline-secondary btn-sm" id="dropdownBasic1" ngbDropdownToggle>Sort by</button> + <button class="btn btn-outline-primary btn-sm" id="dropdownBasic1" ngbDropdownToggle>Sort by</button> <div ngbDropdownMenu aria-labelledby="dropdownBasic1"> <button *ngFor="let f of getSortFields()" ngbDropdownItem (click)="setSort(f.field)" [class.active]="docs.sortField == f.field">{{f.name}}</button> </div> </div> - <label ngbButtonLabel class="btn-outline-secondary btn-sm"> + <label ngbButtonLabel class="btn-outline-primary btn-sm"> <input ngbButton type="radio" class="btn btn-sm" value="asc"> <svg class="toolbaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#sort-alpha-down" /> </svg> </label> - <label ngbButtonLabel class="btn-outline-secondary btn-sm"> + <label ngbButtonLabel class="btn-outline-primary btn-sm"> <input ngbButton type="radio" class="btn btn-sm" value="des"> <svg class="toolbaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#sort-alpha-up-alt" /> @@ -45,7 +45,7 @@ </div> <div class="btn-group" *ngIf="!docs.viewId"> - <button type="button" class="btn btn-sm btn-outline-secondary" (click)="showFilter=!showFilter"> + <button type="button" class="btn btn-sm btn-outline-primary" (click)="showFilter=!showFilter"> <svg class="toolbaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#funnel" /> </svg> @@ -53,7 +53,7 @@ </button> <div class="btn-group" ngbDropdown role="group"> - <button class="btn btn-sm btn-outline-secondary dropdown-toggle-split" ngbDropdownToggle></button> + <button class="btn btn-sm btn-outline-primary dropdown-toggle-split" ngbDropdownToggle></button> <div class="dropdown-menu" ngbDropdownMenu> <button ngbDropdownItem *ngFor="let config of savedViewConfigService.getConfigs()" (click)="loadViewConfig(config)">{{config.title}}</button> <div class="dropdown-divider" *ngIf="savedViewConfigService.getConfigs().length > 0"></div> @@ -72,7 +72,7 @@ </div> <ngb-pagination [pageSize]="docs.currentPageSize" [collectionSize]="docs.collectionSize" [(page)]="docs.currentPage" [maxSize]="5" - [rotate]="true" [boundaryLinks]="true" (pageChange)="reload()" aria-label="Default pagination"></ngb-pagination> + [rotate]="true" (pageChange)="reload()" aria-label="Default pagination"></ngb-pagination> <div *ngIf="displayMode == 'largeCards'"> <app-document-card-large *ngFor="let d of docs.documents" [document]="d" [details]="d.content"> diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html index a790a18b3..6f7a1665f 100644 --- a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html +++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html @@ -1,5 +1,5 @@ <app-page-header title="Correspondents"> - <button type="button" class="btn btn-sm btn-outline-secondary" (click)="openCreateDialog()"> + <button type="button" class="btn btn-sm btn-outline-primary" (click)="openCreateDialog()"> Create </button> </app-page-header> diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-list.component.html b/src-ui/src/app/components/manage/document-type-list/document-type-list.component.html index a07f6c7e4..c239f494f 100644 --- a/src-ui/src/app/components/manage/document-type-list/document-type-list.component.html +++ b/src-ui/src/app/components/manage/document-type-list/document-type-list.component.html @@ -1,5 +1,5 @@ <app-page-header title="Document types"> - <button type="button" class="btn btn-sm btn-outline-secondary" (click)="openCreateDialog()"> + <button type="button" class="btn btn-sm btn-outline-primary" (click)="openCreateDialog()"> Create </button> </app-page-header> diff --git a/src-ui/src/app/components/manage/logs/logs.component.html b/src-ui/src/app/components/manage/logs/logs.component.html index f6738d373..6af482c66 100644 --- a/src-ui/src/app/components/manage/logs/logs.component.html +++ b/src-ui/src/app/components/manage/logs/logs.component.html @@ -1,7 +1,7 @@ <app-page-header title="Logs"> <div ngbDropdown class="btn-group"> - <button class="btn btn-outline-secondary btn-sm" id="dropdownBasic1" ngbDropdownToggle> + <button class="btn btn-outline-primary btn-sm" id="dropdownBasic1" ngbDropdownToggle> <svg class="toolbaricon" fill="currentColor"> <use xlink:href="assets/bootstrap-icons.svg#funnel" /> </svg> diff --git a/src-ui/src/app/components/manage/tag-list/tag-list.component.html b/src-ui/src/app/components/manage/tag-list/tag-list.component.html index d06748cec..850a41a0c 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-list.component.html +++ b/src-ui/src/app/components/manage/tag-list/tag-list.component.html @@ -1,5 +1,5 @@ <app-page-header title="Tags"> - <button type="button" class="btn btn-sm btn-outline-secondary" (click)="openCreateDialog()"> + <button type="button" class="btn btn-sm btn-outline-primary" (click)="openCreateDialog()"> Create </button> </app-page-header> diff --git a/src-ui/src/index.html b/src-ui/src/index.html index 6dde83373..681bcdfd7 100644 --- a/src-ui/src/index.html +++ b/src-ui/src/index.html @@ -4,7 +4,7 @@ <meta charset="utf-8"> <title>PaperlessUi - + diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 454f7d22a..b0b66b7f9 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -1,4 +1,6 @@ -@import "bootstrap"; +@import "theme"; + +@import "node_modules/bootstrap/scss/bootstrap"; .toolbaricon { @@ -27,12 +29,15 @@ body { border-color: rgba(255, 255, 255, .1); } +.form-control-dark::placeholder { + color: #fff; +} + .form-control-dark:focus { border-color: transparent; box-shadow: 0 0 0 3px rgba(255, 255, 255, .25); } - .asc { background-color: #f8f9fa!important; } diff --git a/src-ui/src/theme.scss b/src-ui/src/theme.scss new file mode 100644 index 000000000..88f3ae30f --- /dev/null +++ b/src-ui/src/theme.scss @@ -0,0 +1,6 @@ +$paperless-green: #17541f; +$primary: #17541f; + +$theme-colors: ( + "primary": $primary +); \ No newline at end of file From c11ce4e06f92f8e68ad33052f34233e619c1f36c Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Sun, 22 Nov 2020 16:38:27 +0100 Subject: [PATCH 39/52] favicon --- src-ui/src/favicon.ico | Bin 948 -> 111014 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src-ui/src/favicon.ico b/src-ui/src/favicon.ico index 997406ad22c29aae95893fb3d666c30258a09537..cb57d8b2b1e8b91c9aba08f0b59caa0f4f8808dc 100644 GIT binary patch literal 111014 zcmeEP2V73y8^3S6Xb(v{G$h%TC`nPIDI=A=LYWbIBO@a+vK1N0C?p~EN@TB$$au}j zD5XW}|2(&QbG`o}D$=j}`FwlMe9kk@J@=e*&wC>X79m7PNfAKRiE#o1F@_)rb@lRi zB&1)1G($rw-ijb*$`gdSd3jvNf*@485d@n}#YI?z)jVJT7xFXGCy2?5ScDG5ffb~b zrvo7f;Coi09AupOqvqU`GoTFc2Cx@k4&Vb+@S&MdX+waP0^S0`09OH6 zpT-9W2LQwYBoChYE2*;SK*s}aQgOOIQslo0@Es3`Bfqr(EVB-v07#|c%=$Es`H;r~ zROkZX>3|CWB`O_l$Xl0=qmCw2IhqIBzAqz955o5WbRAy09rE}Y<qRK7x^h=)JI;}PtZTa^D=@2#~;QNHGm{Q z7{E^uO92~M1hExR3MfOI0XzkK1QZkGIHL}&n*(tCAukwsDjR3YGr#{!$%Q(_0Q{*q zq3W#oeZe}4(d_}ndlUO5_I=Xcb<|`9Te=;fs6QK!0@w-=0MwZbDlQ5nZ@LYj{Q=Ve zI4(Q@^Z|TVrb6gi2rzgHLm1nA54a3a1;Q&nl3s7U5FnHiIvIszrCr2QB5TF4_Vx*Ts_z7SGpc|l~Z$S8$yn*E@ zvJb=mf9>h_91p1trS5^26l*4oU) zpm+}sqvMt2V_)0__?`B#P5>`&o;L8ghJF17wZ@xH*Z z=~rdI@x_8}hqsNN#p!w!#Xb@Yz;WVNzNWKC3Hj zpc;v)&i^-0!@21m0FE_3lLObFmG*&ZH2qx>;KS@Q_{<#+z_sp=WWf8d72rEwzZ*e5 zJZKH^?EvPpgelKQ$m@c2YV$&V3jF>=d1SzO^CkfO?83RNHI}K%3;8pE<8}z36oBJJ4?tZuPx8iRJ+?zZ0_Z+K4gmL6 z>b!kyK+UD1eUt;|0N5YY|5W>=4Bu>kGV}zXElB-K&C1Hk)XK_ACCgF7Usjew_yg3+ zxP&Btzl=k$00hDxV8-DPW@k8r+F1@E`5X#mB60}fvLZkUpcE>Vu>l<920*T*fHK5F zz*)d^KpyUg!j>xG38>hoXj%Pz3aIwog?a(-{@)3}`(Gb`Y2THruMt{Splkq+8=(Mx zz(1h_EhkW%bEEt?-uDLlYw|AxnRf&5d4X#`@}2=&FS`!UID_lOZ2%nry>1?Zva7?~ zMcq-MUg>~xF@Unq0^mGN5YPs|1>6Iu1L{>jqUE7zE1(Ae_^ieCdtU&`I}U*BChSwB z{7kA2X@BEl$^hXm09>EYbCpjJJ_k?$;M_7AfX_~UfH|NU$W5*%AzY!8hzo2||A9*t z`Y@y2_7F}3(6S@F47dfD48XF+fDZu7!*vj@r8ff{0g3=7KccdIg4$Gz(Vh~7lL1WG zF^uc*t$+joKF>M=uut)p`wVHQ=OO^kYj6$wO&8cxuI)<=-T_c;U|YceUYRg{2H*$a z0$KoY-B(#TX4-2ggZo7tVLbb;Z`Gy_z@L`i6vCC+6lKD>;v4|V)B(T&FlGHw80!Q9 z)Bto_go-n~Fs_A*;)!ztTxV4!hd<=uy#E0J$Du4hRb{J6zX^3*0KdJr&8x2ZxA`!R zmCve>3CEtnfE$4QfZ+go{Zf^zRiz`Z6##P04V3f;uUyrQ)AfP60jiSeJmlf?meykd zgsYOTy6H!OM@s;q=GgqVbs+1P+n}oZ7UjXQBp#3qkOAxjR9DU)mBTU306?g_$o-ZM zSQmi(F$PdsTR7*(=WHgx3cv?I+f`Qfchc|~*%m;`kMvhAs-75793Oex$Nm%xKpg@A zI0gy=aLveDwkGj&z)u4}>LA-uyxJ;Ib#RV`eVkW@0!Tx<<2@?@K>j#R;hc!KY)#__ zfu|IJ)Sd)K$b4bHy z8Z9?XaV+93TO08l;A=y-R~;YFWr5(y;(s*DC+0E}aLoE8qp7E_ApHe19khtr(P}bx?uubO71!>3m+w zD3A3<0+{c)YfBe=b|P<5HeMNc<5W3(zS7Sr-Xh<{X}&;l?41X|bysc4PV>QeiZI~2 zwkwNZ8D3OYj+w?Nk8R-m0^bGDZPy0H@uXsI#B95^!oZ)_8|Ot+0r(864cTcPcu!WV z5B-i#$|lOR{7A9?;kfz;K(}90x(j%bbBf=Quex=h4W=F)Abz5jbiikk9P+R03)^5o z7Qzbw9{@G+0oF%*)V25|e{6>V>`&H!%I4;97$w#fkc!UB*4V9pbna#j@%rTP3x`LQjf{z^<8FpT%bTmU^+tV(8P zIuV@e3=7A~1697y(s|sT|oU>Q(`x^n4!2g|7M_()eWCN;_9n(_*;()&) z_W!ksaxj4RMSs9;KvgL$ z18GQ$%Ghv%3Mq%A#Yok#Z7xMY5;6`F!f(Fd+!RI2Xv!{6hYDmq9MvkPaPkVvgBBEJ zQigQ|!P~G2?LA4)Vl>c!bX&C_L znm7k0=a4{uShpc$qK?#<83bV*Lvb9!^#eIRH3S`hqyw!d&@KRaevA9N!T`_yQOWAp3@vt89Zc0jXnoU&lQ?oWId~7`RSQr}G=40aU4bT+La51$u@9 z&<41VlMBFpuvGoqoV`Jr0CZ!{zi{7=129#yCeznK81={Jff0Z#+Yk&OCM(9&8^hWF z+`Gl|I%)t+QwQ7tpbeq`^Un%^y5c%;GXV4OYy$2(U>a$M?-)u!R}{iH*TZKV>PF)^gkJ(q1L$XeD+uFzQY^y-90vFTdH`eq6}C`u{{6F2b!|I_ zoan!W0PO3u-ZVx1HUJ6%jsRLWZwS+6Xo~X$+-q6^z_}v1X9kq6LufcF)wNpH;?I-` z!eaoaAFT(I;=FeR;2mHm0LKT^#|OYH#~c0x`DdweE!E(xC4?)W;#xarFRP_&C50HW z;QaOlfLDGxj`zL|0Ovh-0J4B401kjI^E3Jw%3_}|fO#c7j*_;hM6^0j0y$_updK-R zAIYBy`Njb3*ZADm1mLs(Bj88%D$B!hWF-Kft+Wh(+zyPkT_OClGc>oLi~-<2;4J{h zK72-0reAf_@cr^sz;|P1ojV>H&~%Ze?cNmHKUobuaGpc20UttG3lLY0dOx!TK3DOX zVuruB07{~+e|tf<1GFcAUf2Cd)+tcV6M$uo0dxURsrZlTRW~mSc+CQ!KPv1{SABue zMi7L5CL@mhCV+iZS$yVM0IGUc{ES~!`CozOUVu8FPHjN(7YCXGs7f9>9sP}S8nnYR z0QUW{06M?6X)5sU1gJ9`P`q*c%>?{Ne)P#`0KMi5gfPmr1n{GJHO<3y2llZ#u>r-m zH_)H;;cHN45a0s<*L*4flxYi~raJzt{u5|}UT=_gsIBpU(!m`lj=xpOfxce^H~}aC z(DPoLyX*#3Rj)SFA45B4fLiSb6u-$o(E;arI{~;JSPI|=pd5cvf3zjeHw~$_3Dto$ z-)EqBO$S<)uh926=Rv<81$;M0fjGVPt*U-)r=Ni~TL5b1KA`yEbF3;k>HFd)l)-Vb zqW@9hg+HiU70;TcZ-X{#Wv_|SVfasUz-L3R^0F0Y6KFelLON}un(9^E`k$fA3FY#j zKYo8-h2mig6nR$GKK6Tj?$P>I482rZ7oa(S%Iel-8qPmDP%>1=^s51?K8`6kr>RWd zm5?q#mHUo8l_m}J5}>lWb)9wq+7+YZ_)fO!MyR^-K+^!c@}7b;IjYS6v82<1E(Y+{ z{S$FqtGdy0{GWi;NTBNC^K37GDKp;l##BC`>eBrGxY2n)I|JSUnDzfu_%gKH3_#wW z|0m>666ktBaXewlJdcrAITtn@1t@P9?~gE^`b=JR8OE^}pI00MpTD;MK;j5xQ&u^jdI6&9r3 zu@0aO;5MKxb*+qdKD1e}M*Jn=p)k~i^Z2Cv_`D<6H^0II>nl4|>?QVu_)|b-a@J+q zdM&5&cbC5+JC;THc~L_iIH!F6hiz~V+EYS3TG+|`s`#~19@}Q14}>29>eA;--Z&q! zquZ|)g?@tx-9AN4fL;YK<*du_D!TpO5WUu_&@xc8HPFL=y40KIoeJ%c&w*M~<~JEp zy3_XqK3j0_=o5gJwJzve;BNpRe)e1Q-&E_Ds(m{VBiLrrb7q(13So0XL5+xLZV-98@h1W;3bdFw|5Pjb(K zZo9rvp5J6(f53Y+3Q(JU;TrJ!zY~7}e6#?6Ql(#R zi}{`#Wd-245Dxg+yQrVpqN%qTwtO&~lCfb&~g#~)F0|F(kJKN?`@Pw&rILt16;DtK+- zk98Z$#k2#2^#CUUygD=E4Z{CS|Dg>!jS#2r3Ec0-F@dQwGdv5+H?)gqJ&9}TuqtdY zsUhl*{-l4qjrsv_p1&Kw^gV_>01eIk75boV3@AdFIUeA1(hl<*@}>H1EseHl3%CzJ z-{%2L01a7vQa99*0o?b&wNoYl_Z%C%{-_}8Ndf1}y8w3p_#Wp!p+Aoe@I1~Ifbf4w zf5upVeV`M7uQL6~ejZn*qfyLa3D$u_(%qR1*G^4N~}5H^Rsfs3cPUvJy3#q6Dn65;K&DRaQz; z0w{dH0UI*I4?vKraDalOWn79PV~issGoZ)_DKbJzemMdu2L*!*Dv%T^kQ54%6j~~$ z9Ex%&T13$jik8uoU0%M7Ls2e8b0}Iw(GrT5@=*K_Lgb}}--%!Y;3E;=z7qkV+WJle zvaZR+_|}{z^j|-~5x`x*X+SdoIo8#rjmEn5*8+_HU9lK^&79NXp4KefGsQIkqVZ~Y z7GQj5!xjAfnWqonnO8ik+BgO@JlB&NsXJE8WpQnPj>rEv9z+1hxn1KJ&@elJ#<$;w zj|5-y?s?-mHwQrD7|<}?4}Puq{zymgGydM5IS;@vjtlaD#y-#h`+%BDh5vC5fWNup z^*_!9$@g2>2b6|1&IKD%H(>amUITosqz^oSG<*-#I0iH%=YuYkPjMeB7J&DD44|?z zgPS0oJm*!N-)b~4Gc{Osz^B*;lmPfX1IPdVfU#7V`7FqRG%G;k7|=j^gSvw(egAg@ zzkUL60qOv8z;OW6{}?_2kOYwDBN~Vq{?vQh*!2R7^G4DeO{8!f6|Empwh~Y-+u=Ze5(!E2EchA&iR`I z2u*TdxT1gbg*4uMp*N6bgk=$pSN*pD=_~5@x)2rt;CET4^j{VGcKZhWWzX2l8{vf5G2J||Bu7?zV|B3s8O&RAR z$?`aMAR4bfX93)k0bkI-eY`aQrf-?Pqr>Qbd@j)AdohIF0fZ*Z8P)Au5NhA%j`+;y z?Mv?=4cCcl0d4?20NHknzbHW{=X<7T+tsC-5SS++UzY1+v2(m zpVig%J^Jne0Oxow0T{2lSj_*}UQ zz~_WDKpH^HO;UB(qiVc<%>wvM-g1!l74GYN0#w&0bUE}5j@8ZpdcJ^tg8Xe8P*NzQ z09;QnpA|HZU!gcITmZ}lw1vKfJQ;nVaZIRgFJRcs6v8h6zv2hFEc)C5&r)<&<5x$0A07HG!uC31lRy@-a@vc0pDE5aN||A1!!9V#eJA$!0-A3{g3y)1c155 zi-x!$fY5NPcqcF&;=kL@kLux=a2ntVs5o;<|4yN5EBrTe7~d}81A7Gleq`I9<>Bu? z`cdO>0)rRM-M;yv{Ql%H@_)wnck;0>UIt7ARO~O3{h)ELxuUa!ZOHKvs3qV7;CF0I z*U5+aJz#E!eJPf~3&(kF0NobO2XJ0M*Q-m4>z8YQ@c=mhEd%^Nj-P)oy24NmFu<^d zB!o`@YU+nKzytSfa2|$x0W`mC2zLa~wx7?)qs!Da#XW^oz)%3W9sx?q(*RvtEtS;2 z{cZ#2`@g!@tIS__fR6%Y-*Z6eG9Mw_1%UR(|C7c6(0PBF;u_^Bz#M?%6rn`zcUH@2 z4Wtn2O##P5Ttn4{AC3VZoaZ$M+-LB_y|D+rE{80JZw<~ch zX3o=VVqbiIIHL^#ZUB7V()=z#80Q!)Km;S5&i~sK&!L0?aNnY0EKu&>FyAvVbi}h9 zp8&OC|L4Gmw%>Xlo;y`s$p6zW59zmf@@k{)+Tw8=WOD)F{&z(`XdsXL3N=tS^?~F* zFTU5UE!)KcFCl;|0M}bgzT+T{_QW~hpIY}Z+y9kt9>};1AO`@M%lE4)jQ5WRR9xUa ze7P@hPXzn>uh`~CWxXlB)Vsub`)=XN6X}S5aKv4pbeMv4wukFNELR`>{3Drgoj3{lHrjAquFi zc|WclJt)4jfii8G3~~DXJkqN?`OLEQIa~yCj%2DEV^z6-*2AZah4wb>@!#-F6Pp4|qZ=7n&k=^uL*;8}di9~MBFbF`Y+F_7Xp z2Pl(I9K^+`JZ!fEfVn5cEc-8oqd*Vr@1!l&9siv(CK})7F=Igf+OYX6;L`?x@Ae)u zc;Yh?`xxEdkk)3Os}0#}!s7_&(}LCy@2h{|qJOUqaxVeYM1Q*OM&N_{rki>A!TMQ| z57OZ5e#JRWdx+P@J0`mAzfF&VJ{5gH$+iBUIYfWJ*$mu+-2ten{qP-5XTSi!CkDUs z5Uv;>@xFjKVld!eS^HPlHqoF{GfF=~(?z;o4+E5(xQ@YRb9HsBsvO?)xZit&!RsxA zEdiv@X=6}vcc2Z;|7e@t0D6r>t6VQBK>+4E z!Rqw0f29oCVIF{-^VHYhqJVtp2N}Rk!0+1j3Dm{C5awF$3dA)5mG*xs-45ty!0)zO z-|OLfBM;{Pi)G#e^&w380pHKvs(~Nyd5`aOXn)`uuqS}@J+3i$FDlOrXg+|h*AVGT z&=2(_f3vLW@a-AMYm>hbN4WqvSGflG9sA?_4WILLUG(P&0O@HVkN z`_pzPd2qkxB%r2t#C=da)Ad8&qfk_tCO|oWnzs8d)W`Xtk1GAW1NhEmeYhw;;Mt~q zHT1(Js9!PPQwKeNbYZ*tF61{Qw$!lQf5G>(jG3UTDe8`>4;SNmjUhGg!?Oy#VBcG% z8t*?C|0~_A2JO;z{g>!z&==?1^`Z}yeAO6Z&Onm0!^8)qxe1Pfzo&x?Q-_?e!&p~J0 z2mfpHfHp2|X#XpL9O*ys!6(RbESFV*P;jaF`6jw;RR{9-6f1wwc5T?YzVcWL-=N_` zyDGcL{_;Ik8x^o^cn+Wp;Xm*}1mxjap<3;$G(Vti0JxV_U;R{9a^ZRq*I%`2*I(%u zblX6O04m!DKpDHx<$i_t&&nzdF8{7H{{KJV9{{K;JJ)Bv>p^dF4A5}+^X~>|zXDYQ zRQ66T6n@*lhu_Llv9JBPDx-azTm0AdgRo!XyMcDJ9e#Gt)I~nF4fh}mD4yYF-Yf4R zZiM-Wy1JAHR(xZ85VTVOV9M}c2nVUuv5kM^JKDcMdjoK;N83Ca!uXDu&>UEI-@jn2 z1#ynCnbAHiM?ATa3011hywnYDA*Q4z5KVSQy^RnXvjWi>W2Gowx~biZ(J$^RLHiW{fU58 ze?u0!O`tgMSPfv>sG*0qgTCZi@Nf7Z+NS-FV}LQn)!~0X{ppJ#2cA#f3ux$mr}g84 zu3D%+z8Cl#7q*K~c2r>DSw5A4HQ(#~RXuqA=M?Tu$Nfk80M5HcqyL$|nfq1y)ug-% z%=NHs{H~3Tm1<2`)u^uOx9=0+n4}KC^K&{hoK^B^4X zFTX>^y)JwY`kkJQk*Y0#wqig8!nl{l1^kQ8g9jit?srwKaAVH@S%CU~M$CTD9&8u` z_}BJ}@cRhld-&FF_4|9W|L25ykbw6M<6Ife4W|PBrM&`tAMB2LsQkw7tm}ab{<;V} zdxkhB;Cc3AfPZD3xCs5<(Ekr*U3V1HhG@rfNCKS(cmb&I^MZXKBkpt7wTT)$e--#2 z?`TI77Eq(V z#|U=C^NwKOc7RAgecLbi05Wv|G1->X7{w}KVs#^=t z|Kl6&O##oGh64WPIip~drSbl+Ti-Le<9;##pdJZJ;8naSw>; zp}G(A{0<5Gf*N2Q;7^YQhoQ~J^T2iQ@=9C4Ie`ZJ76W{MKtH$u{^YzM5!(Bszs0W9 z5dYysb?SfFfgk98faeRg0)F=0#M*dXT!3~O_WjN(EY4m^nb$x|ojQ0oF2Lu85Ev>hJ z-?{JUIynBT0xH^R4Afr>Fvb9QXG6{p?7(+5StCA#`pS*|ufIO8OfvKr1I7@(4xsPp zU!mzxPZ4=FUjLQ_&}R(jL-;JwOc zyd?5zy#7rKsC|2XgyR98DL4hFZXY-UWt#y0P30Sy$!`JBlf86<$_~bhS*NS;z7w{|Y@BgR|;F%KwztP@! zH4Fn z?+fu93BE_J&!RWVRtpwThX1Qh`5!6%<`16%_&>17^jc`5kwrbX0PSm__`d%(z^gp1 zaqU;nT~}-3!8uZT96;aqD))cg{}0YuQ=pMijVw^^Z?X@_LKx4L{}R74xcKo8Qk;%epjEILjK!~x$?`J_M`uU+Qw@BJmp4H67?MYM1K;B14Sb82H}EZ9uEDo>c?0MG>LM93;#>kQMx12Gh?5dC z;v_?6oc(?MGA>UXB_?|`B>*yH#!+Hs9G(1KoWm0bhTr9b#NWlCf$!oV@%M2yPn^RO z&*6y|@x)=|rmaa*7}>vzm+{8gJoy}+IG1TpV3orY$9s!;@y1Jd@=JdbFXPECTgBuK zVcz(4p8V@&LH2gm;$lkpvOv6VA4|vb(V9hw2tkV5)W|Ue(Wb1gWlv|{2k*Ddp6lj1 zyD;hciHLJef@L$5X0IL9XON6Vx(v6Kx!9sE8slYbFFg>j7$*`ClPEA`oankP_Tw~m z<%}P1t8rt@VP&N?p&O1Y+O|4ow2%7!4e6nyy7?qr+I%8&c9~JfBVM~gysmEZ8~dvG zRm64cT(?JlA68z=-kx`0!EnV*o*yqw9>MCpO@v+aG+;)Md)n@KX^V#oB*3qsx>px-@4ZoO3Ph*KdCoskt=I#@RxBtZmj>5-ye9lGWl}S zelIaCV3t(td7EZ>3YuqLdf^t(nphknb1>!Q6u*Fb{k<=w^%e*V*WdtmNA)N7t|oF9 zW%E52kL~ZBE*8D{EF>!}X*D+5oM`syNk)KdYvbuBI9pq)uz-uhNU^r6f~E;V@v+G} zmK)tLu%FS7Xd1D2`lp%CO^MmuUK|H`2{V~N8(;Whx|P2B{e{G1KkJy!NmG}3Y!cUJ zy*8cITXSRw&_Id%K-trwEg_kCBd!un_J!5%C}r$=Es(T z0>MH(E`2I2uqD=95{;2(4{zCT{%}b{g`B+M+}TTV-r5c=T)ER=)4mCOo{FNk?q>+< z?6eaxNXnD=C|7(lE_C3>i(V`cMoZ&W;f!#xrQy%63mkfOUQwYoG z-|grb(a0WZMMh^1&WcnPy<#(AE2=yCRnh@|wGCNcI8(W4oTCzX(+BXsYSnq5EY#~T z$u`wJ-I@p^>^7ZwGiI%4r|=Uj@dz7zCA(z>b9ekE$?~rjY_VBdYNGP6f`sUqR zK30&x^cNP~k%8f16MR~7o;3;H_c3*RZe0Y*iTt#+q_4FkzBj2f^OD2FSiK{SU&f~ zx~ymv=D8HsGe02F`!oAGzpZS-3pZ~EnWr2uhL6Y#+bN|Xq4zDHwIh01Pn}ZQ;b`A+ zhoJYL@1#3me&+3C%?jLXruvmG+|=7*+MN~b#q9;H!{&7AmMNDk8P)dj<@JFHr70sm zN$%Vh_$o@2J=|G6>By38hwMvzx@cw#Kek@be2(l5$InqF&{%uXCSO9imN`*G6ZR@% z%8r7(*49f!`n^b5JY3l_F-dq;>VDzZLOldO?e4TyIZn`U#e>z^Z~5{^@6X5!xGr_F za8K_MZHPQR#q&lBc6&%|6TGmGeq|X3i1AVvQ?lmI9oCgq4$%#E0S!$@_`z zqHg?Kd&jv42OR{%t$6ifosW1oIVpGk5|y0f*d<+9Lr6`m(n7L_2^|u=9dW}Td5cK? z7n3yAjF#YAtIih+m)La|C@Yl_ZF};G<%~i}!fFY&YZ0V5>p7Ndc>e zyS9_F^>9fQ;Ed~&`?Y)EwXIxhYwv^aqcw^OJwgl8CHSE`o}Uh`-jNl;H3`cclF?51 zvF)T~vswng5H(UyVcY3Ioe8=p@>xcz@k&N(1J{h7kkVsO@|6L3UqE*8o{7hW)iUkH zL~iYQJv;D&>7DF74t@nE?zGKfCwKVxWXWXNs$CWQ_;WutENZifpdB6GW?$3s8wKQQLBvzc%N;lN?)tyDXgb&boeu$XB$E*Z~CW#)zf;9 z5X^rZG|j({@7pBP_lNH+-Amkk5*v}F4Wn|0%R3Wa8YxH0Occ1Gdu^Dgn_d1;@3Hcd zrmUE^_ZGUCotVU0(eFrn^p^LhE+1}}#dZ=^C`}5V-P$ZrimScG(O<-b5DVF z0|Pq-m(@hrrPhy0B<7gr%n#r2S?{KcZI_^D zAA@cSbY>Y&=rz`3md>R%VmX~|EN2%I*ZcT+HP7@R+%#7Pz%4QA&Ff*o@ggp*uIW1H zw`(nsw^c$T^2x?^is?;bygxWD9^{(e1UgetNcLO3!G-eN3p$~;0kdSH@|2v8ZdgN@ z#=hAQ4XmpEJ;W{!lK1m8ihWYo$A9o*Q-AB+)4&ucW9YTl9lD`6brH~M@#;(q(RQpttAZhTl^d`@MG;2{B%TdE#wW19I&soNMWz9hg= zjCfum|4v^{inA|OOPi(EDkW!VCy)2(Vy0p-7rTePUIb4PB6nW*amo%8dYNTtZID_v zLw|suSN{tp0XGCkxg2TMfi-TXUBIUYlio6B|(c_og0pqYAc>15~gt^J$jw6{pRI;x}XicM3SCA$a7>ht+ZwI;;R4+^l86lurmA-HvG zLHr8c=B)W9hvFynH)M^II-gzWQmAn&PNt~kt7rS0sS1DKS5%Ii-z4T@j6qv*&aI;x z76gBd_L3d%C!8bbm3vP%N$8Ps=#*n;2NE;v5|fsX*eBl1Fw8zY@4flZ2O~R51gR~Q zTej`k)ubcx<|;>hM>kuz|L%t8#T$rw(g(hFDSo?LJ0OgBvDj+joZ-vA9&%DBoT+7_ zb5s1fP>+Rzg}2+7vF!M_D@8pmRr6|Mzv0D~Ub8<8GUuM-WCqBlsc!H|FX?lx`^&j* zc}+4D!mqu4{g%(A)2F9%<#R@ZXdSgUg}vwLz4ei0o9sM#T)5}brRGNBMVj7T51w}z zlX8|5O$Z53kp0?K6$wZK)rzdWa=VJHVwnkF7;oZqQ$YRf z^iPW?oIb4AVTh@4yu!q5H~W-u_Io)8Ufq9pPX{7{Z`8VLYxZ9^W?dgFU{Hg&Y~ z%6XiwTO(8V9xYk2Lc~0C@FQPWOJ`#E>xnIHsX9iz_;Pl;hkU|4CsFD7%Rzq!7gfIb zLQ=5ag|vbI2X1y;(fP<-S3RmujDTkJmpC3y+9y z!Af|3pVOP;VBVFp=JBfsxxrb3_9e~~U1@yX&aa=n!=*3?nb zTjNHrK`Tw0nNM(&Z`I%8ti;mZfj72f!~AAc%S(j;orzw$DWz)tCP-i4OL-Q|x~5im zritanb89|d+tpe>%Y#p5iGbRjw16bekf~ieL~sbhA$rf&nCObKL+6%my&N#`gRJv~ zHZi-7Yrh}6)+i)y!7HM>MEv3y@eWEGzKFCTQpTRmaLIFk@i}0@G6TVD#;>1Gwh7H% zVH%clp>UC*NAj*DAtQtL?VATp+pm5$YWtZx6Vp_itX@D^ekm-B&MYi@J(d++uvsv# z&s2Tmy|*Ks?y$1?>;{0Y&%6W@_vmQeFMfMyAEBQ$)ULf-u876n%|VCCCTJhIu>Z*P zm$zcW2IQYmD)=ys&AzZNP%QC&@p8j8&$eVeRFc0K_%XU)e?j#%hcBOAVSjnDX0V{U znBnG$rhbD;6DJQDDz_l>xfUm6{Hlw6wr=CBd^S&ivpD#6vlx44WQxS=k-3SV-g9hL zUzmM8dHtibB)tHJdZDZzAeEj6)7 zePE+6___D6`;P8~UU!C>9-G~5x1Yb++GzHQ^Us=bx@ze3yc-!rTuGWPt;cskdC!!O zJ*>|aF16Cn+Gf-4NdHu0jd#Y5K7r$sn(gV{I-<2NcmGnawIf0f>biX0XW3sMe3EUq z9kaHnUVm{uY2oZH-jiP_M{O@2o4+eBFp8a`5pS$8&)B=~gXirhod3!>7-?a!<>tZ8 z(!`=85pP!+#g04PHRW;iQa8(qhk8t66+iwkH9OHbFmc|%yZdJsiRHxGXM{dkV$ol~ ze2B8|D^_%m6pwilIg3LbC!1ey;Wy#4cT`UerOTiEx_eI`TTtSTiJWmZyO)*^>-D># zs*g*P4|6^)@mXdoHe~L0^>>dpy|gwyb2{wkRs}WwBW$UhwC!l>k1l~TcG|yP9Ja*b z8K3%>yzL(6o(i6x&YIF}S8oB6Ys1{1TTd@~_rR@-dzOtw&f1&Gro=1(a}fJdtv0x`}2$WMVpwv zIImyWZp5qi2i#e1+nppuikqeiz3_9j_i9Vbk@IeEuXb5svL4*Dpn6N6=-3dqX^9Bp#RA zZ!uy|j>g6`qs9mY!fw({=xV4>lJWG&7NiDf%4akJi8eeG|#fKQ#RFEFsBQE69V zioUSO-qZ3rolUN?_6`p)yUG^s(N0q`QmvZhmVxvtt zAHu|U$+X(2Hlb&Nk;SE>y-PdyxVQhpal$g|a!%Lnqr8YA<_RVeUSHOCYQ}Pn4OS44 zR>&{B{VqB0dKQZ{<3Rva;4kF6w`@XF!;L#k%9hwz3?+JQyD~cKS=gOjf<>Hh6EDBn zxy)q8QLF6%UsVSLx4QkMtI~$kfxF8hdOKL{PHg7a*NX3&R^W%bYXmvk&MyLW3L1Mch^a1;eL4%GxvC0&37A70 zB1P`Qy-1ab>;dXWBwZH2^?l>fx-V-`pIlY;#iCx^2{%3E-u3ZqBc-?fFk8Kc)yN~l z@j{<^p1HI8y1(WcIkugR!yH|oAh9WJY+mJY`f8|h|O1Q!t=V03<|z3us!tISEY+S&Ai18#cmIa zpVBI-N6)YG*ZL}ScpBVNK=iaC%i(2wY1W?BMUjIfru%kzBep%H=>S4a=&p5pxmI?5 zrY^g){9}$51P3~r-b_0%GJH6WLpzn%JhqtXBee(P%#b)wpssl!}loeBUdp=(Ntn$H}`_i1w zcKM_A!UYG6Eqm%G6uRx6Q)wLk#4mSr~b4ca(UwJHBZGhvzi5l@wZ*hln z6UD*`i7>u5p?z*Nk8N@L*&u~@A$`hv|2NL=x&$DDY`LY z@u!m8mkAdQwVD6r?E6m65B--kaZK=tm{THBWb4`N_N>q+)5lvTn3b~{a`+5KBw+p-QpDXehc@YT=ey<{I} z74rMrjqlr)Wli`;B>az6Xhf^MRi(c9Q1aGID3nmS-Z(W+Qy`x2bzYQ_bZ%Ot zZlu`mWj9ui>o;loWWi!Wvl~pT@9rMx*rUJ9AePZCdBN>AUtZa`?b2|{o)viD(8crl zow!<>Q{5c)j+OG(F?gqcMR1Tu3E|uIbqwF6bDs~r?J65(K&)xe_Ec)z?s?N%4J4ZR zHH*;n5H8)9eE7O%yksWd^n!tL!hr^y;c06N`YWZqnw&f#yZx#)vZqQXcn&= zueLHss>{KsFuTOQ_A^-NpYxxrPg2qBCL1AK5!0N8E1~^#Yk$sEeSR(_Sewlx}_I(HSVhv9Y>r>OFZMVb^nX)EcN8$0qtCjX6l|9 zXXGIj_aUR@Foob0K#2K%+!q-&JCh#9*gOIcG;kw%%MBt%)TPMI8gP*Z04< zkRRrli5QtAdG(lVWyi@ykydSGu?}UeA)gErNLf-0;|RP9X4|1^x_`FbITmvyz|97+sm$M36cvH?sYmO zd}NUQ`_4Mo+b;R2`%v_u`_m$k_@<-xnJiiE-!5lLw;ch2A*yA93DSd}j}o}u@%D?J zJ<>FKwqvs|$p+?L8XUoImCJ%UgDu<%XI^%L`UUu>>7eW}{p4}RJ??`@6?xUI;NnHoG-*TB5D z!_Cv9rtMC9{jSxnIZ48!mM^`vz%V_$bIDru6Y`wWlK4{>$An+;349O`$zJUJ`0YNo zRPO9`kCwI_F0VdZQkp;R@qxBO*@YS(CU|OdxKa6DLr!;j?{h~!NR-2EJJL4h3E#xD z#G9hy_)9H&Yd(q|XtpbN**R7-9py8TtFt|>8BODy9i?5o@8)cg_}I~%3XPqudn$J5 z`eb;>IE^Mfz4gup7F-(qLeAm6w2RmzL6IGo_eYwB3|MDBP&pyyW$0nE1nWLYVvZei zdU5#zGQpR=!rV)b4?l9^&Uk_CL=TavtzuZx0;A(am1ec(QCi_`1a$=FRkpe;_UPJTp6`50~UM`vUgdme==IH zrF+wZzPH34SX-O!xN^7osA%>pH|_?`lwI8y?6g?wZmKe<)5UBfPvPWAvkq>I){L20 z@Rhh6aBuItUg8!_=ZeQ$aIKAcvUdc`c;~)$w|?H|)*mnW-VQ!I>0{x=S2D(YrJQ{x zRwHcImv&*9ZGTp1XK|4~e5A=7wN@L}wbzQ1p3pH-bdZzOv9I$jS{e{rrp#8Bk~$uG zDWJ>vsXLopY-9Lor=sZNuh~xbz3=yq>~ck+d6Rkb1HB%YWCfjKFL!>SWHY;^M5;nY zwzft8(B~SN=iBTswpO@2_8<`~c%f{>yO4+dUoL1pP4wv*{SmW^pTyap=zOBRu^+s^ zfqe+eUN-k5ExiUGgRR}p`kkfOK0&!=TDvBWS$UshYkuSFz7;J*?yp$y>+Z1ZS!b&| zJum29m~f?6=wSYY=DuBHwvUiV>DVbISlhMpl#3sR*vqogO=3c~PLm3q-ZkW^$!Bfd zM3tr zm$!-VzjTq;Y=KbOg8Zj9Qx5jtV~{_C({Tk$w8#7G-Sax? zIueP3TlXfv|^^X<^KwjbR1-?`-toFB54>3=o{QEZo6sN=ef-HY4o5D5u_iC@U3u zFHECbt7DlKOS4xAB#1d1HaowU~`#(4*u(Ic) zQvJ76(&AV4QWNeWp5VUo;S%R|EkK;PcdDR0`jyc}W?l$kP7 zu3t~xn7y|<5tH_Nzw6R^nd}9JYg!2#q~tzjt;^oG*-Jge$1Qv7!_3oB=X=X39+~=Z zhIQ0AV z;tsu!Iz7V&EWbER@ZsCe>D^AJoGBQ7TV>xNLS1ob?$<|=xpFxUT;FiN#e27jzCSnc z!b-hvySk^oa0wt(bM(GEcv(2ZdC`$>+@XPA%q)<%37xx%Uw_LBBgqoI_^V?x;3;eqz^zla5-ITsn_+$b@APsH{K*%5H8OK}VKZC8Tmb^L(#lY7JF zk93zz5Q*2{{dC;nc?t(3+H)){&OfpA+9CJe^>)zN($48pR-!B0T-VgPeu{6Xh~0%R#`hVL5-9g-01<6{Vx#(?jIr4hw1Q2IU1Kjk2_KcU@`d139rw{%-tswNoHTf; z-fQ%#Rp-0pv=e%uZT#UB=g`!r>>x$uLAHsZP#@>4M#p|5f$F&%K0smo^@@qNrqsz z-GA@q{Ug$a&ruRrcdt*YAJCm$BXcGi%vxj8Aw>YQ!I@8=Fpm0sX z)e&l6F3%ZoS24vYYVy{J2TO`fzN{KE&@SCcQP5ZZT|2d{hBn^aOIO%0crPJ8SUajc zJMT>YHz{X28;+B|vN^P$7{R%bwPq}FU-)zT)moyquUp=iIN&<{u=?q>%6UU=ShiZ- zto63{n-|bsW38vgnf*7{aMRbvT#A=1@>uZp&!@>N%sug;SzaZa&_$eU{Ap z-5M=&I(Ta4m1=Rs=lg5l>?^Iit*5bqZ?`*_ebW2-9xdB=8=($oW{rA~|^Z82TUY71qi_X|=HrqcXmN>k0#?AH;LGPLnF>lUW51Ko< zYodtA7Lm2F#MYtbE*}sM<(sngMcx;7yNQ+}Ll2p27)YHES)lRJK6Q{muLAWm89B0C z)xednOQqgiTYJh!F6jBp#fb%dXrsk`q4t&&nal6=v zgthjoCcE=($gt3H`#jGxSH^Y2>801SkB80@j@H_BL$&j{6aE%4qXTa}-yQQzW6L|M zm3>ZqzBc^u=&K5^RA&ja`l@I0a;%=I#MVADdYcFM+uv!oTx93=VUq7Q=5R`ug&L~v z=6`C!z3;TuuPeNt_^@-;IG^A%E~^gvT`icdcIc+_?cD_N&OJ}=)IHO44@YK3EO=HV zekq!rDgC*`Wnng-YtV(sB4_%_`n)(d+ii!Syk+vP#Etu0UAyv?5h)>&GcL4u)adwz zv*hv(&dHqZ63r845t;n^H_kpAvXZkTpzY2fhfhw*_%K+6yUvy6YMB4xiMh^_!~xQ6 z!pHBjdej7Opc&kCkETr{B=t&~=iF1C-}~`APU^BOe|=54p{-|y;;Eb+>(lwTy_^j+ zgnAw-y&qI6#_1cXq}BYYRWA`>5S1UBeRs{U)qA~j9TqIl@!mXbu5^*3owAPC>5n%> zC5KEcl@ZwO$8};S2o^0Izx7KdCuw$Dm-~Z9`xINvxxKrYr(W*Y-6AJEL(GJV)HRlP zn&_S@YmyM)=*qRREen3EYN6%eZZ~dGXP2kfPX`WKxMk`h6B^ls+Rg+_d4Yl&J;B+mU|$UX7tc- z&2A02J9A#mh<|jZgHE#hxvy=!yX??-Gir9itl-x-y6=?unEW|liMDBEMAEu9nSvI} zawqO>ziT)*P0UY9G4XzAsnzFB2cV;$oa8jlIyiK~{b^G!ninawjuY3PqGguxy!nvR zGjHVu=d*m~$mH1V$+Y__lYE`H@?q?#MT-^K9TxJxb&$3hoZ#2!{~FHaq2NkbrJGmI2Fd%lX~Q1$NBt=PWgTv-RZLtp0`fj7(z!vk&0{_b)WpkoMe0@Dd7S_@n!ss*JKu51_?qiF?T3}G9=N?#@ioV;cCZqyga zuk0=t=V&4HCV0*?&Z@vaiIe+OVc3du8X2Xj1z_mNWuU~suMYa?#6@%F9HpxG*;20V zdpdC8G*5je@Flfnzmaj8S^$pRIb8ns9uABSJf`%88&U9d63Tl`Ro@Rny#J?xp%z9N zwxz$3VVYV1p1kp)^0yBYLUqf`2VMd&GB*#xp8+qO=4o$3L%jd!q>Y6QBO5fe0E}Z0 zI5_9`+j(tJ;!=Be%j{yFChioj-Tw{6q`O)mhr zu{nP~p*cvIIRvZ!bPRt5;IqJ591L4oZ=+6VdI8vSU_j8OgFc?_^Z5>NGjQMl=ps03Dt-Uqh2rp0DRa6j1uAj4wV}Oz6ao>AxkV84U7rEf&7M$ zzdv0b2Cmc6?+n`B->5T80D#Dky}366d#3qo9{9Y#11b=yz5R{)qv-`;>Dj@>Uz)$K zVfwuVe+Ya<0Teb>s3B+=S*7U(AdYe&*-u%)^pkyd0)G!|B~n@?PWv_tz^MJl-K8qg#Uo5-(2XD5wzfs>bwE!T1 za02k6X&k)+_&Z>$b5X^np@vZ}G`9c*VnP+4srXMq=U_(m?M{2lhEWx!)6=eDnfsji zC~Okp9zrqDHlU^ST*iyz8sJ^P7J#i^+nDyy4WlYdr?X8%BSH*(Q{a0F%YkQCBYU7IjCFmI-B}4sMJt3^s9NVg!P0 z6hDQy(L`XWMkB@zOLgN$4KYz;j0zZxq9KKdpZE#5@k0crP^5f9KO};h)ZDQ%ybhht z%t9#h|nu0K(bJ ztIkhEr!*UyrZWQ1k2+YkGqDi8Z<|mIN&$kzpKl{cNP=OQzXHz>vn+c)F)zO|Bou>E z2|-d_=qY#Y+yOu1a}XI?cU}%04)zz%anD(XZC{#~WreV!a$7k2Ug`?&CUEc0EtrkZ zL49MB)h!_K{H(*l_93D5tO0;BUnvYlo+;yss%n^&qjt6fZOa+}+FDO(~2>G z2dx@=JZ?DHP^;b7*Y1as5^uphBsh*s*z&MBd?e@I>-9kU>63PjP&^#5YTOb&x^6Cf z?674rmSHB5Fk!{Gv7rv!?qX#ei_L(XtwVqLX3L}$MI|kJ*w(rhx~tc&L&xP#?cQow zX_|gx$wMr3pRZIIr_;;O|8fAjd;1`nOeu5K(pCu7>^3E&D2OBBq?sYa(%S?GwG&_0-s%_v$L@R!5H_fc)lOb9ZoOO#p`Nn`KU z3LTTBtjwo`7(HA6 z7gmO$yTR!5L>Bsg!X8616{JUngg_@&85%>W=mChTR;x4`P=?PJ~oPuy5 zU-L`C@_!34D21{fD~Y8NVnR3t;aqZI3fIhmgmx}$oc-dKDC6Ap$Gy>a!`A*x2L1v0 WcZ@i?LyX}70000 Date: Sun, 22 Nov 2020 16:38:47 +0100 Subject: [PATCH 40/52] layout changes --- src-ui/src/app/components/dashboard/dashboard.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html index 3fad2af04..4c1089c64 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.html +++ b/src-ui/src/app/components/dashboard/dashboard.component.html @@ -1,5 +1,5 @@ - +
From 8e7a3d309f82e06e9b18568dd9ca090a991ef44c Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Sun, 22 Nov 2020 16:39:25 +0100 Subject: [PATCH 41/52] page title --- src-ui/src/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/index.html b/src-ui/src/index.html index 681bcdfd7..f82399ce6 100644 --- a/src-ui/src/index.html +++ b/src-ui/src/index.html @@ -2,7 +2,7 @@ - PaperlessUi + Paperless-ng From 6c0e0755b9539ac431f16287fd7b094b23ca1301 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Sun, 22 Nov 2020 17:48:54 +0100 Subject: [PATCH 42/52] menu closing on mobile --- .../app-frame/app-frame.component.html | 18 +++++++++--------- .../app-frame/app-frame.component.ts | 6 ++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 05289becc..31c14bd7b 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -17,7 +17,7 @@