From 999b36473c933c3029fb54c983d05babca9f39a6 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 03:29:23 +0100 Subject: [PATCH 01/48] more refactoring and bug fixing. --- .../filter-editor/filter-editor.component.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/filter-editor/filter-editor.component.ts index f98b9517f..a11f0736a 100644 --- a/src-ui/src/app/components/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/filter-editor/filter-editor.component.ts @@ -200,24 +200,21 @@ export class FilterEditorComponent implements OnInit, OnDestroy { } setDateFilter(date: NgbDateStruct, dateRuleTypeID: number) { - let filterRules = this.filterRules - let existingRule = filterRules.find(rule => rule.rule_type == dateRuleTypeID) + let existingRule = this.filterRules.find(rule => rule.rule_type == dateRuleTypeID) let newValue = this.dateParser.format(date) if (existingRule) { existingRule.value = newValue } else { - filterRules.push({rule_type: dateRuleTypeID, value: newValue}) + this.filterRules.push({rule_type: dateRuleTypeID, value: newValue}) } - - this.filterRules = filterRules } clearDateFilter(dateRuleTypeID: number) { - let filterRules = this.filterRules - let existingRule = filterRules.find(rule => rule.rule_type == dateRuleTypeID) - filterRules.splice(filterRules.indexOf(existingRule), 1) - this.filterRules = filterRules + let ruleIndex = this.filterRules.findIndex(rule => rule.rule_type == dateRuleTypeID) + if (ruleIndex != -1) { + this.filterRules.splice(ruleIndex, 1) + } } } From 6d39dfeb3b096d994d303e8f89faeb09142a2eaa Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 03:53:18 +0100 Subject: [PATCH 02/48] forgot to address this. --- .../components/document-list/document-list.component.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index acbfd3602..1b3596098 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -93,16 +93,16 @@ - {{(d.correspondent$ | async)?.name}} + {{(d.correspondent$ | async)?.name}} {{d.title | documentTitle}} - + - {{(d.document_type$ | async)?.name}} + {{(d.document_type$ | async)?.name}} From 164755c755a170678bfd648f6f72261eee7c4af6 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Mon, 14 Dec 2020 19:45:22 -0800 Subject: [PATCH 03/48] Breakpoints for screen sizes, icons for mobile --- .../document-list.component.html | 2 +- .../filter-dropdown.component.html | 14 +++++- .../filter-dropdown.component.ts | 2 +- .../filter-editor.component.html | 45 ++++++++++--------- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index acbfd3602..5f91deb5f 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -62,7 +62,7 @@ -
+
diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html index 5de9228bd..76ef360fd 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html +++ b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html @@ -1,6 +1,18 @@
diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.ts b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.ts index d675e14f1..b9d3fca6f 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.ts +++ b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.ts @@ -22,7 +22,7 @@ export class FilterDropdownComponent { title: string @Input() - display: string + icon: string @Output() toggle = new EventEmitter() diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.html b/src-ui/src/app/components/filter-editor/filter-editor.component.html index 80f10407c..5bf23f8bd 100644 --- a/src-ui/src/app/components/filter-editor/filter-editor.component.html +++ b/src-ui/src/app/components/filter-editor/filter-editor.component.html @@ -1,22 +1,27 @@ -
-
-
Filter by:
+
+
+
+ + +
-
- -
- - - - - - - - - +
+
+
+ + + + + +
+
+
+
+ +
From 30185d560ca363b53deed34c747b8a201ef58ab0 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Mon, 14 Dec 2020 22:33:50 -0800 Subject: [PATCH 04/48] Much cleaner way to set icon --- .../filter-dropdown/filter-dropdown.component.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html index 76ef360fd..8f7d14e81 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html +++ b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html @@ -11,6 +11,9 @@ +
+ +
From 49be87fe371d1f8a5895f42513a943b5bf2a5a7c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 12:06:24 +0100 Subject: [PATCH 05/48] code style --- src/documents/models.py | 6 +++++- src/documents/serialisers.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/documents/models.py b/src/documents/models.py index 1b1f697bc..b544f413d 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -340,7 +340,11 @@ class SavedViewFilterRule(models.Model): (17, "Does not have tag"), ] - saved_view = models.ForeignKey(SavedView, on_delete=models.CASCADE, related_name="filter_rules") + saved_view = models.ForeignKey( + SavedView, + on_delete=models.CASCADE, + related_name="filter_rules" + ) rule_type = models.PositiveIntegerField(choices=RULE_TYPES) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 2def07fdd..36878448c 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -167,14 +167,16 @@ class SavedViewSerializer(serializers.ModelSerializer): if rules_data: SavedViewFilterRule.objects.filter(saved_view=instance).delete() for rule_data in rules_data: - SavedViewFilterRule.objects.create(saved_view=instance, **rule_data) + SavedViewFilterRule.objects.create( + saved_view=instance, **rule_data) return instance def create(self, validated_data): rules_data = validated_data.pop('filter_rules') saved_view = SavedView.objects.create(**validated_data) for rule_data in rules_data: - SavedViewFilterRule.objects.create(saved_view=saved_view, **rule_data) + SavedViewFilterRule.objects.create( + saved_view=saved_view, **rule_data) return saved_view From 56204933b0c045de0a4b5156c781269cfc40c3c1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 13:16:28 +0100 Subject: [PATCH 06/48] bugfix, tests --- src/documents/serialisers.py | 2 +- src/documents/tests/test_api.py | 91 +++++++++++++++++++++++++++++-- src/documents/tests/test_index.py | 21 +++++++ 3 files changed, 109 insertions(+), 5 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 36878448c..ee0a42384 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -164,7 +164,7 @@ class SavedViewSerializer(serializers.ModelSerializer): else: rules_data = None super(SavedViewSerializer, self).update(instance, validated_data) - if rules_data: + if rules_data is not None: SavedViewFilterRule.objects.filter(saved_view=instance).delete() for rule_data in rules_data: SavedViewFilterRule.objects.create( diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index ab1716366..e0a64664f 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -4,12 +4,11 @@ import tempfile from unittest import mock from django.contrib.auth.models import User -from pathvalidate import ValidationError from rest_framework.test import APITestCase from whoosh.writing import AsyncWriter from documents import index -from documents.models import Document, Correspondent, DocumentType, Tag +from documents.models import Document, Correspondent, DocumentType, Tag, SavedView from documents.tests.utils import DirectoriesMixin @@ -18,8 +17,8 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): def setUp(self): super(TestDocumentApi, self).setUp() - user = User.objects.create_superuser(username="temp_admin") - self.client.force_login(user=user) + self.user = User.objects.create_superuser(username="temp_admin") + self.client.force_login(user=self.user) def testDocuments(self): @@ -515,3 +514,87 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertFalse(meta['has_archive_version']) self.assertGreater(len(meta['original_metadata']), 0) self.assertIsNone(meta['archive_metadata']) + + def test_saved_views(self): + u1 = User.objects.create_user("user1") + u2 = User.objects.create_user("user2") + + v1 = SavedView.objects.create(user=u1, name="test1", sort_field="", show_on_dashboard=False, show_in_sidebar=False) + v2 = SavedView.objects.create(user=u2, name="test2", sort_field="", show_on_dashboard=False, show_in_sidebar=False) + v3 = SavedView.objects.create(user=u2, name="test3", sort_field="", show_on_dashboard=False, show_in_sidebar=False) + + response = self.client.get("/api/saved_views/") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 0) + + self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 404) + + self.client.force_login(user=u1) + + response = self.client.get("/api/saved_views/") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 1) + + self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 200) + + self.client.force_login(user=u2) + + response = self.client.get("/api/saved_views/") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['count'], 2) + + self.assertEqual(self.client.get(f"/api/saved_views/{v1.id}/").status_code, 404) + + def test_create_update_patch(self): + + u1 = User.objects.create_user("user1") + + view = { + "name": "test", + "show_on_dashboard": True, + "show_in_sidebar": True, + "sort_field": "created2", + "filter_rules": [ + { + "rule_type": 4, + "value": "test" + } + ] + } + + response = self.client.post("/api/saved_views/", view, format='json') + self.assertEqual(response.status_code, 201) + + v1 = SavedView.objects.get(name="test") + self.assertEqual(v1.sort_field, "created2") + self.assertEqual(v1.filter_rules.count(), 1) + self.assertEqual(v1.user, self.user) + + response = self.client.patch(f"/api/saved_views/{v1.id}/", { + "show_in_sidebar": False + }, format='json') + + v1 = SavedView.objects.get(id=v1.id) + self.assertEqual(response.status_code, 200) + self.assertFalse(v1.show_in_sidebar) + self.assertEqual(v1.filter_rules.count(), 1) + + view['filter_rules'] = [{ + "rule_type": 12, + "value": "secret" + }] + + response = self.client.put(f"/api/saved_views/{v1.id}/", view, format='json') + self.assertEqual(response.status_code, 200) + + v1 = SavedView.objects.get(id=v1.id) + self.assertEqual(v1.filter_rules.count(), 1) + self.assertEqual(v1.filter_rules.first().value, "secret") + + view['filter_rules'] = [] + + response = self.client.put(f"/api/saved_views/{v1.id}/", view, format='json') + self.assertEqual(response.status_code, 200) + + v1 = SavedView.objects.get(id=v1.id) + self.assertEqual(v1.filter_rules.count(), 0) diff --git a/src/documents/tests/test_index.py b/src/documents/tests/test_index.py index 830fca0e0..2baa9621d 100644 --- a/src/documents/tests/test_index.py +++ b/src/documents/tests/test_index.py @@ -1,6 +1,9 @@ from django.test import TestCase +from documents import index from documents.index import JsonFormatter +from documents.models import Document +from documents.tests.utils import DirectoriesMixin class JsonFormatterTest(TestCase): @@ -12,3 +15,21 @@ class JsonFormatterTest(TestCase): self.assertListEqual(self.formatter.format([]), []) +class TestAutoComplete(DirectoriesMixin, TestCase): + + def test_auto_complete(self): + + doc1 = Document.objects.create(title="doc1", checksum="A", content="test test2 test3") + doc2 = Document.objects.create(title="doc2", checksum="B", content="test test2") + doc3 = Document.objects.create(title="doc3", checksum="C", content="test2") + + index.add_or_update_document(doc1) + index.add_or_update_document(doc2) + index.add_or_update_document(doc3) + + ix = index.open_index() + + self.assertListEqual(index.autocomplete(ix, "tes"), [b"test3", b"test", b"test2"]) + self.assertListEqual(index.autocomplete(ix, "tes", limit=3), [b"test3", b"test", b"test2"]) + self.assertListEqual(index.autocomplete(ix, "tes", limit=1), [b"test3"]) + self.assertListEqual(index.autocomplete(ix, "tes", limit=0), []) From 7e0aa7136aed44d7cf200c5a816a372c282acc4a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 13:26:01 +0100 Subject: [PATCH 07/48] more tests --- src/paperless_text/tests/samples/test.txt | 1 + src/paperless_text/tests/test_parser.py | 26 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 src/paperless_text/tests/samples/test.txt create mode 100644 src/paperless_text/tests/test_parser.py diff --git a/src/paperless_text/tests/samples/test.txt b/src/paperless_text/tests/samples/test.txt new file mode 100644 index 000000000..6de7b8c69 --- /dev/null +++ b/src/paperless_text/tests/samples/test.txt @@ -0,0 +1 @@ +This is a test file. diff --git a/src/paperless_text/tests/test_parser.py b/src/paperless_text/tests/test_parser.py new file mode 100644 index 000000000..413aa91cf --- /dev/null +++ b/src/paperless_text/tests/test_parser.py @@ -0,0 +1,26 @@ +import os + +from django.test import TestCase + +from documents.tests.utils import DirectoriesMixin +from paperless_text.parsers import TextDocumentParser + + +class TestTextParser(DirectoriesMixin, TestCase): + + def test_thumbnail(self): + + parser = TextDocumentParser(None) + + # just make sure that it does not crash + f = parser.get_thumbnail(os.path.join(os.path.dirname(__file__), "samples", "test.txt"), "text/plain") + self.assertTrue(os.path.isfile(f)) + + def test_parse(self): + + parser = TextDocumentParser(None) + + parser.parse(os.path.join(os.path.dirname(__file__), "samples", "test.txt"), "text/plain") + + self.assertEqual(parser.get_text(), "This is a test file.\n") + self.assertIsNone(parser.get_archive_path()) From b787983e42434daadae4ff0c309329f91efc8781 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 13:47:43 +0100 Subject: [PATCH 08/48] more tests --- src/documents/models.py | 2 +- src/documents/tests/test_sanity_check.py | 10 +++++++--- src/documents/tests/utils.py | 3 ++- src/paperless_mail/mail.py | 10 +++++----- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/documents/models.py b/src/documents/models.py index b544f413d..d81343afa 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -221,7 +221,7 @@ class Document(models.Model): else: fname = "{:07}{}".format(self.pk, self.file_type) if self.storage_type == self.STORAGE_TYPE_GPG: - fname += ".gpg" + fname += ".gpg" # pragma: no cover return os.path.join( settings.ORIGINALS_DIR, diff --git a/src/documents/tests/test_sanity_check.py b/src/documents/tests/test_sanity_check.py index 725e87617..0554cd7cd 100644 --- a/src/documents/tests/test_sanity_check.py +++ b/src/documents/tests/test_sanity_check.py @@ -2,6 +2,8 @@ import os import shutil from pathlib import Path +import filelock +from django.conf import settings from django.test import TestCase from documents.models import Document @@ -13,9 +15,11 @@ class TestSanityCheck(DirectoriesMixin, TestCase): def make_test_data(self): - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000001.pdf"), os.path.join(self.dirs.originals_dir, "0000001.pdf")) - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "archive", "0000001.pdf"), os.path.join(self.dirs.archive_dir, "0000001.pdf")) - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000001.png"), os.path.join(self.dirs.thumbnail_dir, "0000001.png")) + with filelock.FileLock(settings.MEDIA_LOCK): + # just make sure that the lockfile is present. + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000001.pdf"), os.path.join(self.dirs.originals_dir, "0000001.pdf")) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "archive", "0000001.pdf"), os.path.join(self.dirs.archive_dir, "0000001.pdf")) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000001.png"), os.path.join(self.dirs.thumbnail_dir, "0000001.png")) return Document.objects.create(title="test", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", content="test", pk=1, filename="0000001.pdf", mime_type="application/pdf") diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index 7f9d50ed5..dfefc4061 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -34,7 +34,8 @@ def setup_directories(): ARCHIVE_DIR=dirs.archive_dir, CONSUMPTION_DIR=dirs.consumption_dir, INDEX_DIR=dirs.index_dir, - MODEL_FILE=os.path.join(dirs.data_dir, "classification_model.pickle") + MODEL_FILE=os.path.join(dirs.data_dir, "classification_model.pickle"), + MEDIA_LOCK=os.path.join(dirs.media_dir, "media.lock") ) dirs.settings_override.enable() diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index a82c34f15..3c200362d 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -26,7 +26,7 @@ class BaseMailAction: return {} def post_consume(self, M, message_uids, parameter): - pass + pass # pragma: nocover class DeleteMailAction(BaseMailAction): @@ -69,7 +69,7 @@ def get_rule_action(rule): elif rule.action == MailRule.ACTION_MARK_READ: return MarkReadMailAction() else: - raise ValueError("Unknown action.") + raise NotImplementedError("Unknown action.") # pragma: nocover def make_criterias(rule): @@ -95,7 +95,7 @@ def get_mailbox(server, port, security): elif security == MailAccount.IMAP_SECURITY_SSL: mailbox = MailBox(server, port) else: - raise ValueError("Unknown IMAP security") + raise NotImplementedError("Unknown IMAP security") # pragma: nocover return mailbox @@ -119,7 +119,7 @@ class MailAccountHandler(LoggingMixin): return os.path.splitext(os.path.basename(att.filename))[0] else: - raise ValueError("Unknown title selector.") + raise NotImplementedError("Unknown title selector.") # pragma: nocover # NOQA: E501 def get_correspondent(self, message, rule): c_from = rule.assign_correspondent_from @@ -141,7 +141,7 @@ class MailAccountHandler(LoggingMixin): return rule.assign_correspondent else: - raise ValueError("Unknwown correspondent selector") + raise NotImplementedError("Unknwown correspondent selector") # pragma: nocover # NOQA: E501 def handle_mail_account(self, account): From 5894060dc5e63b9109427816d76b94d66a132de5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 13:52:35 +0100 Subject: [PATCH 09/48] fixes #25 --- src/documents/parsers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 228e2c86e..cbbb912de 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -163,8 +163,6 @@ def parse_date(filename, text): date = None - next_year = timezone.now().year + 5 # Arbitrary 5 year future limit - # if filename date parsing is enabled, search there first: if settings.FILENAME_DATE_ORDER: for m in re.finditer(DATE_REGEX, filename): @@ -176,7 +174,7 @@ def parse_date(filename, text): # Skip all matches that do not parse to a proper date continue - if date is not None and next_year > date.year > 1900: + if date and date.year > 1900 and date <= timezone.now(): return date # Iterate through all regex matches in text and try to parse the date @@ -189,7 +187,7 @@ def parse_date(filename, text): # Skip all matches that do not parse to a proper date continue - if date is not None and next_year > date.year > 1900: + if date and date.year > 1900 and date <= timezone.now(): break else: date = None From 31bea6a361e24d7ac2e42e70624edf08caaffc3c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 14:11:05 +0100 Subject: [PATCH 10/48] path sanitation --- src/documents/file_handling.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/documents/file_handling.py b/src/documents/file_handling.py index c49493991..5643756ac 100644 --- a/src/documents/file_handling.py +++ b/src/documents/file_handling.py @@ -99,6 +99,11 @@ def generate_filename(doc, counter=0): tags = defaultdictNoStr(lambda: slugify(None), many_to_dictionary(doc.tags)) + tag_list = pathvalidate.sanitize_filename( + ",".join([tag.name for tag in doc.tags.all()]), + replacement_text="-" + ) + if doc.correspondent: correspondent = pathvalidate.sanitize_filename( doc.correspondent.name, replacement_text="-" @@ -127,7 +132,7 @@ def generate_filename(doc, counter=0): added_month=f"{doc.added.month:02}" if doc.added else "none", added_day=f"{doc.added.day:02}" if doc.added else "none", tags=tags, - tag_list=",".join([tag.name for tag in doc.tags.all()]) + tag_list=tag_list ).strip() path = path.strip(os.sep) From a0c74025e3c1cda4b17c044c13f7251f6a7fe96c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 14:30:31 +0100 Subject: [PATCH 11/48] changelog and docs --- docs/advanced_usage.rst | 36 +++++++++++++++++++++--------------- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index b5ae254b3..48a86384c 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -263,10 +263,10 @@ using the identifier which it has assigned to each document. You will end up get files like ``0000123.pdf`` in your media directory. This isn't necessarily a bad thing, because you normally don't have to access these files manually. However, if you wish to name your files differently, you can do that by adjusting the -``PAPERLESS_FILENAME_FORMAT`` settings variable. +``PAPERLESS_FILENAME_FORMAT`` configuration option. -This variable allows you to configure the filename (folders are allowed!) using -placeholders. For example, setting +This variable allows you to configure the filename (folders are allowed) using +placeholders. For example, configuring this to .. code:: bash @@ -277,17 +277,16 @@ will create a directory structure as follows: .. code:: 2019/ - my_bank/ - statement-january-0000001.pdf - statement-february-0000002.pdf + My bank/ + Statement January.pdf + Statement February.pdf 2020/ - my_bank/ - statement-january-0000003.pdf - shoe_store/ - my_new_shoes-0000004.pdf - -Paperless appends the unique identifier of each document to the filename. This -avoids filename clashes. + My bank/ + Statement January.pdf + Letter.pdf + Letter_01.pdf + Shoe store/ + My new shoes.pdf .. danger:: @@ -299,6 +298,7 @@ Paperless provides the following placeholders withing filenames: * ``{correspondent}``: The name of the correspondent, or "none". * ``{document_type}``: The name of the document type, or "none". +* ``{tag_list}``: A comma separated list of all tags assigned to the document. * ``{title}``: The title of the document. * ``{created}``: The full date and time the document was created. * ``{created_year}``: Year created only. @@ -309,8 +309,14 @@ Paperless provides the following placeholders withing filenames: * ``{added_month}``: Month added only (number 1-12). * ``{added_day}``: Day added only (number 1-31). -Paperless will convert all values for the placeholders into values which are safe -for use in filenames. + +Paperless will try to conserve the information from your database as much as possible. +However, some characters that you can use in document titles and correspondent names (such +as ``: \ /`` and a couple more) are not allowed in filenames and will be replaced with dashes. + +If paperless detects that two documents share the same filename, paperless will automatically +append ``_01``, ``_02``, etc to the filename. This happens if all the placeholders in a filename +evaluate to the same value. .. hint:: diff --git a/docs/changelog.rst b/docs/changelog.rst index a50fc31d5..0e55cb144 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,40 @@ Changelog ********* + +paperless-ng 0.9.7 +################## + + +* Front end + + * Thanks to the hard work of `Michael Shamoon`_, paperless now comes with a much more streamlined UI for + filtering documents. + + * `Michael Shamoon`_ replaced the document preview with another component. This should fix compatibility with Safari browsers. + + * Paperless now stores your saved views on the server and associates them with your user account. You + will have to recreate your views. + +* Other additions and changes + + * The GitHub and documentation links now open in new tabs/windows. Thanks to `rYR79435`_. + * The new filename format field ``{tag_list}`` inserts a list of tags into the filename, separated by comma. + * The ``document_retagger`` no longer removes inbox tags or tags without matching rules. + * The new configuration option ``PAPERLESS_COOKIE_PREFIX`` allows you to run multiple instances of paperless on different ports. + This option enables you to be logged in into multiple instances by specifying different cookie names for each instance. + +* Fixes + + * Sometimes paperless would assign dates in the future to newly consumed documents. + * The filename format fields ``{created_month}`` and ``{created_day}`` now use a leading zero for single digit values. + * The filename format field ``{tags}`` can no longer be used without arguments. + * Paperless was not able to consume many images (especially images from mobile scanners) due to missing DPI information. + Paperless now assumes A4 paper size for PDF generation if no DPI information is present. + * Documents with empty titles could not be opened from the table view due to the link being empty. + * Fixed an issue with filenames containing special characters such as ``:`` not being accepted for upload. + + paperless-ng 0.9.6 ################## @@ -841,6 +875,8 @@ bulk of the work on this big change. * Initial release +.. _rYR79435: https://github.com/rYR79435 +.. _Michael Shamoon: https://github.com/shamoon .. _jayme-github: http://github.com/jayme-github .. _Brian Conn: https://github.com/TheConnMan .. _Christopher Luu: https://github.com/nuudles From 7dce57b9f76a5684f321d175a921fb6a8da0442f Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 14:34:48 +0100 Subject: [PATCH 12/48] changelog --- docs/changelog.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0e55cb144..84d04bc7a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,8 +17,9 @@ paperless-ng 0.9.7 * `Michael Shamoon`_ replaced the document preview with another component. This should fix compatibility with Safari browsers. - * Paperless now stores your saved views on the server and associates them with your user account. You - will have to recreate your views. + * Paperless now stores your saved views on the server and associates them with your user account. + This means that you can access your views on multiple devices and have separate views for different users. + You will have to recreate your views. * Other additions and changes From 55075619c10fd1dc3f2e607c4e6e418aa95a1146 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 14:43:07 +0100 Subject: [PATCH 13/48] documentation --- docs/usage_overview.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/usage_overview.rst b/docs/usage_overview.rst index bb9ecd452..d6f4cf9db 100644 --- a/docs/usage_overview.rst +++ b/docs/usage_overview.rst @@ -57,9 +57,6 @@ Adding documents to paperless ############################# Once you've got Paperless setup, you need to start feeding documents into it. -Currently, there are four options: the consumption directory, the dashboard, IMAP (email), and -HTTP POST. - When adding documents to paperless, it will perform the following operations on your documents: @@ -112,6 +109,17 @@ Dashboard upload The dashboard has a file drop field to upload documents to paperless. Simply drag a file onto this field or select a file with the file dialog. Multiple files are supported. + +Mobile upload +============= + +The mobile app over at ``_ allows Android users +to share any documents with paperless. This can be combined with any of the mobile +scanning apps out there, such as Office Lens. + +The app is still a little rough around the edges, +but it gets the job done. This will eventually be rolled into `Paperless App `_ as well. + .. _usage-email: IMAP (Email) From d208ab1e1299c30a9c98cad98c8810a3f18c6314 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Tue, 15 Dec 2020 15:03:00 +0100 Subject: [PATCH 14/48] Update README.md --- README.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 41f85af19..276521f5b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [Paperless](https://github.com/the-paperless-project/paperless) is an application by Daniel Quinn and others that indexes your scanned documents and allows you to easily search for documents and store metadata alongside your documents. -Paperless-ng is a fork of the original project, adding a new interface and many other changes under the hood. For a detailed list of changes, see below. +Paperless-ng is a fork of the original project, adding a new interface and many other changes under the hood. For a detailed list of changes, have a look at the changelog in the documentation. This project is still in development and some things may not work as expected. @@ -15,11 +15,13 @@ This project is still in development and some things may not work as expected. Paperless does not control your scanner, it only helps you deal with what your scanner produces. -1. Buy a document scanner that can write to a place on your network. If you need some inspiration, have a look at the [scanner recommendations](https://paperless-ng.readthedocs.io/en/latest/scanners.html) page. -2. Set it up to "scan to FTP" or something similar. It should be able to push scanned images to a server without you having to do anything. Of course if your scanner doesn't know how to automatically upload the file somewhere, you can always do that manually. Paperless doesn't care how the documents get into its local consumption directory. -3. Have the target server run the Paperless consumption script to OCR the file and index it into a local database. -4. Use the web frontend to sift through the database and find what you want. -5. Download the PDF you need/want via the web interface and do whatever you like with it. You can even print it and send it as if it's the original. In most cases, no one will care or notice. +1. Buy a document scanner that can write to a place on your network. If you need some inspiration, have a look at the [scanner recommendations](https://paperless-ng.readthedocs.io/en/latest/scanners.html) page. Set it up to "scan to FTP" or something similar. It should be able to push scanned images to a server without you having to do anything. Of course if your scanner doesn't know how to automatically upload the file somewhere, you can always do that manually. Paperless doesn't care how the documents get into its local consumption directory. + + - Alternatively, you can use any of the mobile scanning apps out there. We have an app that allows you to share documents with paperless, if you're on Android. + +2. Wait for paperless to process your files. OCR is expensive, and depending on the power of your machine, this might take a bit of time. +3. Use the web frontend to sift through the database and find what you want. +4. Download the PDF you need/want via the web interface and do whatever you like with it. You can even print it and send it as if it's the original. In most cases, no one will care or notice. Here's what you get: @@ -39,7 +41,6 @@ Here's what you get: * When adding documents from mails, paperless can move these mails to a new folder, mark them as read, flag them or delete them. * Machine learning powered document matching. * Paperless learns from your documents and will be able to automatically assign tags, correspondents and types to documents once you've stored a few documents in paperless. -* We have a mobile app that offers a 'Share with paperless' option over at https://github.com/qcasey/paperless_share. You can use that in combination with any of the mobile scanning apps out there. It's still a little rough around the edges, but it works! * A task processor that processes documents in parallel and also tells you when something goes wrong. On modern multi core systems, consumption is blazing fast. * Code cleanup in many, MANY areas. Some of the code from OG paperless was just overly complicated. * More tests, more stability. @@ -78,7 +79,7 @@ The recommended way to deploy paperless is docker-compose. Don't clone the repos Read the [documentation](https://paperless-ng.readthedocs.io/en/latest/setup.html#installation) on how to get started. -Alternatively, you can install the dependencies and setup apache and a database server yourself. The documenation has information about the individual components of paperless that you need to take care of. +Alternatively, you can install the dependencies and setup apache and a database server yourself. The documenation has a step by step guide on how to do it. # Migrating to paperless-ng @@ -102,13 +103,15 @@ If you want to implement something big: Please start a discussion about that in Paperless has been around a while now, and people are starting to build stuff on top of it. If you're one of those people, we can add your project to this list: -* [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless. We're working on making this compatible. +* [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless. Updated to work with paperless-ng. +* [Paperless Share](https://github.com/qcasey/paperless_share). Share any files from your Android application with paperless. Very simple, but works with all of the mobile scanning apps out there that allow you to share scanned documents. + +These projects also exist, but their status and compatibility with paperless is unknown. + * [Paperless Desktop](https://github.com/thomasbrueggemann/paperless-desktop): A desktop UI for your Paperless installation. Runs on Mac, Linux, and Windows. * [ansible-role-paperless](https://github.com/ovv/ansible-role-paperless): An easy way to get Paperless running via Ansible. * [paperless-cli](https://github.com/stgarf/paperless-cli): A golang command line binary to interact with a Paperless instance. -Compatibility with Paperless-ng is unknown. - # Important Note Document scanners are typically used to scan sensitive documents. Things like your social insurance number, tax records, invoices, etc. Everything is stored in the clear without encryption by default (it needs to be searchable, so if someone has ideas on how to do that on encrypted data, I'm all ears). This means that Paperless should never be run on an untrusted host. Instead, I recommend that if you do want to use it, run it locally on a server in your own home. From 02e67d25b46ae17614a6d2f474564b1c10f0f87f Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Tue, 15 Dec 2020 15:03:34 +0100 Subject: [PATCH 15/48] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 276521f5b..95dd83752 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Paperless does not control your scanner, it only helps you deal with what your s 1. Buy a document scanner that can write to a place on your network. If you need some inspiration, have a look at the [scanner recommendations](https://paperless-ng.readthedocs.io/en/latest/scanners.html) page. Set it up to "scan to FTP" or something similar. It should be able to push scanned images to a server without you having to do anything. Of course if your scanner doesn't know how to automatically upload the file somewhere, you can always do that manually. Paperless doesn't care how the documents get into its local consumption directory. - - Alternatively, you can use any of the mobile scanning apps out there. We have an app that allows you to share documents with paperless, if you're on Android. + - Alternatively, you can use any of the mobile scanning apps out there. We have an app that allows you to share documents with paperless, if you're on Android. See the section on affiliated projects. 2. Wait for paperless to process your files. OCR is expensive, and depending on the power of your machine, this might take a bit of time. 3. Use the web frontend to sift through the database and find what you want. From 71c58c4b05b4f4e62f4a6fe24a4f8677accbc570 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Tue, 15 Dec 2020 15:04:28 +0100 Subject: [PATCH 16/48] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 95dd83752..e8ae8feb2 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Paperless has been around a while now, and people are starting to build stuff on * [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless. Updated to work with paperless-ng. * [Paperless Share](https://github.com/qcasey/paperless_share). Share any files from your Android application with paperless. Very simple, but works with all of the mobile scanning apps out there that allow you to share scanned documents. -These projects also exist, but their status and compatibility with paperless is unknown. +These projects also exist, but their status and compatibility with paperless-ng is unknown. * [Paperless Desktop](https://github.com/thomasbrueggemann/paperless-desktop): A desktop UI for your Paperless installation. Runs on Mac, Linux, and Windows. * [ansible-role-paperless](https://github.com/ovv/ansible-role-paperless): An easy way to get Paperless running via Ansible. From beffde1051669769d1ab38e63372e0f0ba962638 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Tue, 15 Dec 2020 07:10:31 -0800 Subject: [PATCH 17/48] quick filter button badges --- .../filter-dropdown.component.html | 15 +++++---------- .../filter-dropdown.component.scss | 6 ++++++ .../filter-editor/filter-editor.component.html | 8 ++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html index 8f7d14e81..d0cbfc3c9 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html +++ b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.html @@ -1,21 +1,16 @@
diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.scss b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.scss index d34729eee..40c93838f 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.scss +++ b/src-ui/src/app/components/filter-editor/filter-dropdown/filter-dropdown.component.scss @@ -1,3 +1,9 @@ +.badge-corner { + position: absolute; + top: -8px; + right: -8px; +} + .dropdown-menu { min-width: 250px; diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.html b/src-ui/src/app/components/filter-editor/filter-editor.component.html index 5bf23f8bd..6847a2902 100644 --- a/src-ui/src/app/components/filter-editor/filter-editor.component.html +++ b/src-ui/src/app/components/filter-editor/filter-editor.component.html @@ -8,10 +8,10 @@
- - - - + + + +
From 677cfb7a1e20032b0755a722759e8c88e70c43f2 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Tue, 15 Dec 2020 14:31:18 -0800 Subject: [PATCH 18/48] Use bootstrap row-cols-* classes to keep card list view full width --- .../document-card-small/document-card-small.component.html | 6 +++--- .../components/document-list/document-list.component.html | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html index 2647e702c..da0829349 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html @@ -1,4 +1,4 @@ -
+
@@ -22,7 +22,7 @@
-
+
@@ -42,7 +42,7 @@
- {{document.created | date}} + {{document.created | date}}
diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index 1b3596098..31b00f482 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -116,6 +116,6 @@ -
+
From b2a9cf47098f06235e42777baf0cec29ef4807bc Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 15 Dec 2020 23:35:10 +0100 Subject: [PATCH 19/48] docs --- docs/usage_overview.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage_overview.rst b/docs/usage_overview.rst index d6f4cf9db..7a4fd7740 100644 --- a/docs/usage_overview.rst +++ b/docs/usage_overview.rst @@ -117,8 +117,8 @@ The mobile app over at ``_ allows And to share any documents with paperless. This can be combined with any of the mobile scanning apps out there, such as Office Lens. -The app is still a little rough around the edges, -but it gets the job done. This will eventually be rolled into `Paperless App `_ as well. +Furthermore, there is the `Paperless App `_ as well, +which no only has document upload, but also document editing and browsing. .. _usage-email: From 22e56f09baf19fd7d3da5d06cbaa247236225fbf Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 00:14:32 +0100 Subject: [PATCH 20/48] fixes some issues regarding #139 --- .../document-card-large/document-card-large.component.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss index d6be8837e..11fb10562 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss @@ -1,5 +1,6 @@ .result-content { color: darkgray; + overflow-wrap: anywhere; } .doc-img { From 1a526ac31e230d46f37926facb4d302170a5d057 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 02:12:58 +0100 Subject: [PATCH 21/48] fixes #140 --- .../correspondent-list.component.html | 23 ++++++++++++++++--- .../correspondent-list.component.ts | 14 ++++++++++- .../document-type-list.component.html | 21 +++++++++++++++-- .../document-type-list.component.ts | 14 ++++++++++- .../manage/tag-list/tag-list.component.html | 23 ++++++++++++++++--- .../manage/tag-list/tag-list.component.ts | 15 +++++++++++- 6 files changed, 99 insertions(+), 11 deletions(-) diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html index 27aa4d366..2efd1c58d 100644 --- a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html +++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.html @@ -26,9 +26,26 @@ {{ correspondent.last_correspondence | date }}
- - -
+ + + +
diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts index effae2826..a128340b9 100644 --- a/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts +++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-list.component.ts @@ -1,6 +1,9 @@ import { Component, OnInit } from '@angular/core'; +import { Router } from '@angular/router'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; +import { FILTER_CORRESPONDENT } from 'src/app/data/filter-rule-type'; import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'; +import { DocumentListViewService } from 'src/app/services/document-list-view.service'; import { CorrespondentService } from 'src/app/services/rest/correspondent.service'; import { GenericListComponent } from '../generic-list/generic-list.component'; import { CorrespondentEditDialogComponent } from './correspondent-edit-dialog/correspondent-edit-dialog.component'; @@ -12,7 +15,10 @@ import { CorrespondentEditDialogComponent } from './correspondent-edit-dialog/co }) export class CorrespondentListComponent extends GenericListComponent { - constructor(correspondentsService: CorrespondentService, modalService: NgbModal,) { + constructor(correspondentsService: CorrespondentService, modalService: NgbModal, + private router: Router, + private list: DocumentListViewService + ) { super(correspondentsService,modalService,CorrespondentEditDialogComponent) } @@ -20,4 +26,10 @@ export class CorrespondentListComponent extends GenericListComponent{{ document_type.document_count }}
- - + + +
diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts b/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts index 16cdd88a9..d18a19226 100644 --- a/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts +++ b/src-ui/src/app/components/manage/document-type-list/document-type-list.component.ts @@ -1,6 +1,9 @@ import { Component } from '@angular/core'; +import { Router } from '@angular/router'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; +import { FILTER_DOCUMENT_TYPE } from 'src/app/data/filter-rule-type'; import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'; +import { DocumentListViewService } from 'src/app/services/document-list-view.service'; import { DocumentTypeService } from 'src/app/services/rest/document-type.service'; import { GenericListComponent } from '../generic-list/generic-list.component'; import { DocumentTypeEditDialogComponent } from './document-type-edit-dialog/document-type-edit-dialog.component'; @@ -12,7 +15,10 @@ import { DocumentTypeEditDialogComponent } from './document-type-edit-dialog/doc }) export class DocumentTypeListComponent extends GenericListComponent { - constructor(service: DocumentTypeService, modalService: NgbModal) { + constructor(service: DocumentTypeService, modalService: NgbModal, + private router: Router, + private list: DocumentListViewService + ) { super(service, modalService, DocumentTypeEditDialogComponent) } @@ -20,4 +26,10 @@ export class DocumentTypeListComponent extends GenericListComponent
- +
@@ -28,8 +28,25 @@ diff --git a/src-ui/src/app/components/manage/tag-list/tag-list.component.ts b/src-ui/src/app/components/manage/tag-list/tag-list.component.ts index 32093e0a8..e3f151550 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-list.component.ts +++ b/src-ui/src/app/components/manage/tag-list/tag-list.component.ts @@ -1,6 +1,9 @@ import { Component } from '@angular/core'; +import { Router } from '@angular/router'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; +import { FILTER_HAS_TAG } from 'src/app/data/filter-rule-type'; import { TAG_COLOURS, PaperlessTag } from 'src/app/data/paperless-tag'; +import { DocumentListViewService } from 'src/app/services/document-list-view.service'; import { TagService } from 'src/app/services/rest/tag.service'; import { GenericListComponent } from '../generic-list/generic-list.component'; import { TagEditDialogComponent } from './tag-edit-dialog/tag-edit-dialog.component'; @@ -12,7 +15,10 @@ import { TagEditDialogComponent } from './tag-edit-dialog/tag-edit-dialog.compon }) export class TagListComponent extends GenericListComponent { - constructor(tagService: TagService, modalService: NgbModal) { + constructor(tagService: TagService, modalService: NgbModal, + private router: Router, + private list: DocumentListViewService + ) { super(tagService, modalService, TagEditDialogComponent) } @@ -23,4 +29,11 @@ export class TagListComponent extends GenericListComponent { getObjectName(object: PaperlessTag) { return `tag '${object.name}'` } + + filterDocuments(object: PaperlessTag) { + this.list.documentListView.filter_rules = [ + {rule_type: FILTER_HAS_TAG, value: object.id.toString()} + ] + this.router.navigate(["documents"]) + } } From e528a587cc89e6529479d34d6fa3f8a53b9b52bc Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 13:41:02 +0100 Subject: [PATCH 22/48] fixed some issues with the test cases. --- src/paperless_mail/tests/test_mail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 2a391a268..9c0f52c53 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -399,7 +399,7 @@ class TestMail(TestCase): c = Correspondent.objects.get(name="amazon@amazon.de") # should work - self.assertEquals(kwargs['override_correspondent_id'], c.id) + self.assertEqual(kwargs['override_correspondent_id'], c.id) self.async_task.reset_mock() self.reset_bogus_mailbox() @@ -411,7 +411,7 @@ class TestMail(TestCase): args, kwargs = self.async_task.call_args self.async_task.assert_called_once() - self.assertEquals(kwargs['override_correspondent_id'], None) + self.assertEqual(kwargs['override_correspondent_id'], None) def test_filters(self): From 8bd82f5c69fa4de1ac02d4c29741253ee30311ce Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 13:49:48 +0100 Subject: [PATCH 23/48] fixing some test case warnings, case insensitive sorting for tags, correspondents and types. --- src/documents/models.py | 7 ++++++- src/documents/tests/test_file_handling.py | 11 ++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/documents/models.py b/src/documents/models.py index d81343afa..245bba6e9 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -11,6 +11,7 @@ import dateutil.parser from django.conf import settings from django.contrib.auth.models import User from django.db import models +from django.db.models.functions import Lower from django.utils import timezone from django.utils.text import slugify @@ -61,7 +62,7 @@ class MatchingModel(models.Model): class Meta: abstract = True - ordering = ("name",) + ordering = (Lower("name"),) def __str__(self): return self.name @@ -308,6 +309,10 @@ class Log(models.Model): class SavedView(models.Model): + class Meta: + + ordering = (Lower("name"),) + user = models.ForeignKey(User, on_delete=models.CASCADE) name = models.CharField(max_length=128) diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index dec89c45b..2e60065f1 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -9,6 +9,7 @@ from unittest import mock from django.conf import settings from django.db import DatabaseError from django.test import TestCase, override_settings +from django.utils import timezone from .utils import DirectoriesMixin from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \ @@ -298,23 +299,23 @@ class TestFileHandling(DirectoriesMixin, TestCase): @override_settings(PAPERLESS_FILENAME_FORMAT="{created_year}-{created_month}-{created_day}") def test_created_year_month_day(self): - d1 = datetime.datetime(2020, 3, 6, 1, 1, 1) + d1 = timezone.make_aware(datetime.datetime(2020, 3, 6, 1, 1, 1)) doc1 = Document.objects.create(title="doc1", mime_type="application/pdf", created=d1) self.assertEqual(generate_filename(doc1), "2020-03-06.pdf") - doc1.created = datetime.datetime(2020, 11, 16, 1, 1, 1) + doc1.created = timezone.make_aware(datetime.datetime(2020, 11, 16, 1, 1, 1)) self.assertEqual(generate_filename(doc1), "2020-11-16.pdf") @override_settings(PAPERLESS_FILENAME_FORMAT="{added_year}-{added_month}-{added_day}") def test_added_year_month_day(self): - d1 = datetime.datetime(232, 1, 9, 1, 1, 1) + d1 = timezone.make_aware(datetime.datetime(232, 1, 9, 1, 1, 1)) doc1 = Document.objects.create(title="doc1", mime_type="application/pdf", added=d1) self.assertEqual(generate_filename(doc1), "232-01-09.pdf") - doc1.added = datetime.datetime(2020, 11, 16, 1, 1, 1) + doc1.added = timezone.make_aware(datetime.datetime(2020, 11, 16, 1, 1, 1)) self.assertEqual(generate_filename(doc1), "2020-11-16.pdf") @@ -599,7 +600,7 @@ class TestFilenameGeneration(TestCase): PAPERLESS_FILENAME_FORMAT="{created}" ) def test_date(self): - doc = Document.objects.create(title="does not matter", created=datetime.datetime(2020,5,21, 7,36,51, 153), mime_type="application/pdf", pk=2, checksum="2") + doc = Document.objects.create(title="does not matter", created=timezone.make_aware(datetime.datetime(2020,5,21, 7,36,51, 153)), mime_type="application/pdf", pk=2, checksum="2") self.assertEqual(generate_filename(doc), "2020-05-21.pdf") From e47b105185912efc08160d9ee28dc57dae58ef3d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 14:17:05 +0100 Subject: [PATCH 24/48] fixes #7 and some test cases. --- src/paperless_text/parsers.py | 60 ++++++----------------------------- 1 file changed, 10 insertions(+), 50 deletions(-) diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py index f8f369ab0..646c5c549 100644 --- a/src/paperless_text/parsers.py +++ b/src/paperless_text/parsers.py @@ -1,6 +1,7 @@ import os import subprocess +from PIL import ImageDraw, ImageFont, Image from django.conf import settings from documents.parsers import DocumentParser, ParseError @@ -12,63 +13,22 @@ class TextDocumentParser(DocumentParser): """ def get_thumbnail(self, document_path, mime_type): - """ - The thumbnail of a text file is just a 500px wide image of the text - rendered onto a letter-sized page. - """ - # The below is heavily cribbed from https://askubuntu.com/a/590951 - - bg_color = "white" # bg color - text_color = "black" # text color - psize = [500, 647] # icon size - n_lines = 50 # number of lines to show - out_path = os.path.join(self.tempdir, "convert.png") - - temp_bg = os.path.join(self.tempdir, "bg.png") - temp_txlayer = os.path.join(self.tempdir, "tx.png") - picsize = "x".join([str(n) for n in psize]) - txsize = "x".join([str(n - 8) for n in psize]) - - def create_bg(): - work_size = ",".join([str(n - 1) for n in psize]) - r = str(round(psize[0] / 10)) - rounded = ",".join([r, r]) - run_command( - settings.CONVERT_BINARY, - "-size ", picsize, - ' xc:none -draw ', - '"fill ', bg_color, ' roundrectangle 0,0,', work_size, ",", rounded, '" ', # NOQA: E501 - temp_bg - ) def read_text(): with open(document_path, 'r') as src: lines = [line.strip() for line in src.readlines()] - text = "\n".join([line for line in lines[:n_lines]]) + text = "\n".join([line for line in lines[:50]]) return text.replace('"', "'") - def create_txlayer(): - run_command( - settings.CONVERT_BINARY, - "-background none", - "-fill", - text_color, - "-pointsize", "12", - "-border 4 -bordercolor none", - "-size ", txsize, - ' caption:"', read_text(), '" ', - temp_txlayer - ) + img = Image.new("RGB", (500, 700), color="white") + draw = ImageDraw.Draw(img) + font = ImageFont.truetype( + "/usr/share/fonts/liberation/LiberationSerif-Regular.ttf", 20, + layout_engine=ImageFont.LAYOUT_BASIC) + draw.text((5, 5), read_text(), font=font, fill="black") - create_txlayer() - create_bg() - run_command( - settings.CONVERT_BINARY, - temp_bg, - temp_txlayer, - "-background None -layers merge ", - out_path - ) + out_path = os.path.join(self.tempdir, "thumb.png") + img.save(out_path) return out_path From b2e0a8c88401c33077192ce6285057e60e1afd74 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 14:19:11 +0100 Subject: [PATCH 25/48] thumbnail generation --- src/paperless_text/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py index 646c5c549..7e488ca37 100644 --- a/src/paperless_text/parsers.py +++ b/src/paperless_text/parsers.py @@ -17,8 +17,8 @@ class TextDocumentParser(DocumentParser): def read_text(): with open(document_path, 'r') as src: lines = [line.strip() for line in src.readlines()] - text = "\n".join([line for line in lines[:50]]) - return text.replace('"', "'") + text = "\n".join(lines[:50]) + return text img = Image.new("RGB", (500, 700), color="white") draw = ImageDraw.Draw(img) From 9f5e6d1969f5af9742e0399414a297f81324021c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 14:23:10 +0100 Subject: [PATCH 26/48] fixes metadata display --- .../components/document-detail/document-detail.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index c0114f709..f4a64c2cc 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -110,8 +110,8 @@
Name{{ tag.document_count }}
- - + + +
- - + + From b13ec571f84cef60bd54dab2f59b61650a435b92 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 16 Dec 2020 14:41:57 +0100 Subject: [PATCH 27/48] fixes #134 --- docs/changelog.rst | 2 ++ src-ui/src/app/components/app-frame/app-frame.component.html | 5 +++++ src-ui/src/app/components/app-frame/app-frame.component.ts | 3 +++ src-ui/src/environments/environment.prod.ts | 3 ++- src-ui/src/environments/environment.ts | 3 ++- 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 84d04bc7a..b508bae13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,7 @@ paperless-ng 0.9.7 * The ``document_retagger`` no longer removes inbox tags or tags without matching rules. * The new configuration option ``PAPERLESS_COOKIE_PREFIX`` allows you to run multiple instances of paperless on different ports. This option enables you to be logged in into multiple instances by specifying different cookie names for each instance. + * Added a small version indicator to the front end. * Fixes @@ -38,6 +39,7 @@ paperless-ng 0.9.7 Paperless now assumes A4 paper size for PDF generation if no DPI information is present. * Documents with empty titles could not be opened from the table view due to the link being empty. * Fixed an issue with filenames containing special characters such as ``:`` not being accepted for upload. + * Fixed issues with thumbnail generation for plain text files. paperless-ng 0.9.6 diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 7876150af..3d315ec32 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -17,6 +17,11 @@