From fdcbf9c75c6c6fa360841e7472a46140fd4aaca3 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Sun, 6 Dec 2020 23:30:51 +0100 Subject: [PATCH 01/37] Update CONTRIBUTING.md --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd6080d35..a8fb1f8e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,3 +24,7 @@ feature-X branches is for experimental stuff that will eventually be merged into I'm trying to get most of paperless tested, so please do the same for your code! I know its a hassle, but it makes sure that your code works now and will allow us to detect regressions easily. To test your code, execute `pytest` in the src/ directory. Executing that in the project root is no good. This also generates a html coverage report, which you can use to see if you missed anything important during testing. + +## More info: + +... is available in the documentation. https://paperless-ng.readthedocs.io/en/latest/extending.html From 1357ee83a0485268403eb8e8081e9936aaed8180 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 8 Dec 2020 21:20:05 +0100 Subject: [PATCH 02/37] version bump --- docker/hub/docker-compose.postgres.yml | 2 +- docker/hub/docker-compose.sqlite.yml | 2 +- src/paperless/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml index 295d981e1..24f0e118f 100644 --- a/docker/hub/docker-compose.postgres.yml +++ b/docker/hub/docker-compose.postgres.yml @@ -15,7 +15,7 @@ services: POSTGRES_PASSWORD: paperless webserver: - image: jonaswinkler/paperless-ng:0.9.5 + image: jonaswinkler/paperless-ng:0.9.6 restart: always depends_on: - db diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml index 80df40596..6ae619fd6 100644 --- a/docker/hub/docker-compose.sqlite.yml +++ b/docker/hub/docker-compose.sqlite.yml @@ -5,7 +5,7 @@ services: restart: always webserver: - image: jonaswinkler/paperless-ng:0.9.5 + image: jonaswinkler/paperless-ng:0.9.6 restart: always depends_on: - broker diff --git a/src/paperless/version.py b/src/paperless/version.py index 26e46fea8..527e0668d 100644 --- a/src/paperless/version.py +++ b/src/paperless/version.py @@ -1 +1 @@ -__version__ = (0, 9, 5) +__version__ = (0, 9, 6) From ba7bf9b2d2d77643cbb30f0aa4f138a6de7befca Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 00:04:37 +0100 Subject: [PATCH 03/37] removed slugs entirely, since their only purpose was purely cosmetic anyway. --- src/documents/admin.py | 8 +----- src/documents/consumer.py | 2 +- .../management/commands/document_consumer.py | 5 +--- .../migrations/1006_auto_20201208_2209.py | 25 +++++++++++++++++++ src/documents/models.py | 11 ++------ src/documents/serialisers.py | 19 +++++++++++--- src/documents/signals/handlers.py | 4 +-- src/documents/tests/test_consumer.py | 12 ++++----- src/paperless_mail/mail.py | 5 +--- 9 files changed, 55 insertions(+), 36 deletions(-) create mode 100644 src/documents/migrations/1006_auto_20201208_2209.py diff --git a/src/documents/admin.py b/src/documents/admin.py index 2a4fb0031..055a6fd93 100755 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -17,8 +17,6 @@ class CorrespondentAdmin(admin.ModelAdmin): list_filter = ("matching_algorithm",) list_editable = ("match", "matching_algorithm") - readonly_fields = ("slug",) - class TagAdmin(admin.ModelAdmin): @@ -31,8 +29,6 @@ class TagAdmin(admin.ModelAdmin): list_filter = ("colour", "matching_algorithm") list_editable = ("colour", "match", "matching_algorithm") - readonly_fields = ("slug", ) - class DocumentTypeAdmin(admin.ModelAdmin): @@ -44,8 +40,6 @@ class DocumentTypeAdmin(admin.ModelAdmin): list_filter = ("matching_algorithm",) list_editable = ("match", "matching_algorithm") - readonly_fields = ("slug",) - class DocumentAdmin(admin.ModelAdmin): @@ -106,7 +100,7 @@ class DocumentAdmin(admin.ModelAdmin): for tag in obj.tags.all(): r += self._html_tag( "span", - tag.slug + ", " + tag.name + ", " ) return r diff --git a/src/documents/consumer.py b/src/documents/consumer.py index f52dd5a7d..19ca3ed7e 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -259,7 +259,7 @@ class Consumer(LoggingMixin): relevant_tags = set(file_info.tags) if relevant_tags: - tag_names = ", ".join([t.slug for t in relevant_tags]) + tag_names = ", ".join([t.name for t in relevant_tags]) self.log("debug", "Tagging with {}".format(tag_names)) document.tags.add(*relevant_tags) diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index 5cecd6bf9..b2f689aed 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -29,10 +29,7 @@ def _tags_from_path(filepath): path_parts = Path(filepath).relative_to( settings.CONSUMPTION_DIR).parent.parts for part in path_parts: - tag_ids.add(Tag.objects.get_or_create( - slug=slugify(part), - defaults={"name": part}, - )[0].pk) + tag_ids.add(Tag.objects.get_or_create(name=part)[0].pk) return tag_ids diff --git a/src/documents/migrations/1006_auto_20201208_2209.py b/src/documents/migrations/1006_auto_20201208_2209.py new file mode 100644 index 000000000..49f8c8dfe --- /dev/null +++ b/src/documents/migrations/1006_auto_20201208_2209.py @@ -0,0 +1,25 @@ +# Generated by Django 3.1.4 on 2020-12-08 22:09 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('documents', '1005_checksums'), + ] + + operations = [ + migrations.RemoveField( + model_name='correspondent', + name='slug', + ), + migrations.RemoveField( + model_name='documenttype', + name='slug', + ), + migrations.RemoveField( + model_name='tag', + name='slug', + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 366cb215d..f0678a843 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -36,7 +36,6 @@ class MatchingModel(models.Model): ) name = models.CharField(max_length=128, unique=True) - slug = models.SlugField(blank=True, editable=False) match = models.CharField(max_length=256, blank=True) matching_algorithm = models.PositiveIntegerField( @@ -69,7 +68,6 @@ class MatchingModel(models.Model): def save(self, *args, **kwargs): self.match = self.match.lower() - self.slug = slugify(self.name) models.Model.save(self, *args, **kwargs) @@ -384,9 +382,7 @@ class FileInfo: def _get_correspondent(cls, name): if not name: return None - return Correspondent.objects.get_or_create(name=name, defaults={ - "slug": slugify(name) - })[0] + return Correspondent.objects.get_or_create(name=name)[0] @classmethod def _get_title(cls, title): @@ -396,10 +392,7 @@ class FileInfo: def _get_tags(cls, tags): r = [] for t in tags.split(","): - r.append(Tag.objects.get_or_create( - slug=slugify(t), - defaults={"name": t} - )[0]) + r.append(Tag.objects.get_or_create(name=t)[0]) return tuple(r) @classmethod diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 5aedeeb58..600645061 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1,4 +1,5 @@ import magic +from django.utils.text import slugify from pathvalidate import validate_filename, ValidationError from rest_framework import serializers from rest_framework.fields import SerializerMethodField @@ -7,12 +8,16 @@ from .models import Correspondent, Tag, Document, Log, DocumentType from .parsers import is_mime_type_supported -class CorrespondentSerializer(serializers.HyperlinkedModelSerializer): +class CorrespondentSerializer(serializers.ModelSerializer): document_count = serializers.IntegerField(read_only=True) last_correspondence = serializers.DateTimeField(read_only=True) + def get_slug(self, obj): + return slugify(obj.name) + slug = SerializerMethodField() + class Meta: model = Correspondent fields = ( @@ -27,10 +32,14 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer): ) -class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer): +class DocumentTypeSerializer(serializers.ModelSerializer): document_count = serializers.IntegerField(read_only=True) + def get_slug(self, obj): + return slugify(obj.name) + slug = SerializerMethodField() + class Meta: model = DocumentType fields = ( @@ -44,10 +53,14 @@ class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer): ) -class TagSerializer(serializers.HyperlinkedModelSerializer): +class TagSerializer(serializers.ModelSerializer): document_count = serializers.IntegerField(read_only=True) + def get_slug(self, obj): + return slugify(obj.name) + slug = SerializerMethodField() + class Meta: model = Tag fields = ( diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 8a9ce18d7..8121072bf 100755 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -136,7 +136,7 @@ def set_tags(sender, message = 'Tagging "{}" with "{}"' logger( - message.format(document, ", ".join([t.slug for t in relevant_tags])), + message.format(document, ", ".join([t.name for t in relevant_tags])), logging_group ) @@ -165,7 +165,7 @@ def run_post_consume_script(sender, document, **kwargs): reverse("document-download", kwargs={"pk": document.pk}), reverse("document-thumb", kwargs={"pk": document.pk}), str(document.correspondent), - str(",".join(document.tags.all().values_list("slug", flat=True))) + str(",".join(document.tags.all().values_list("name", flat=True))) )).wait() diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index f828d3e11..b4b19be4c 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -27,7 +27,7 @@ class TestAttributes(TestCase): self.assertEqual(file_info.title, title, filename) - self.assertEqual(tuple([t.slug for t in file_info.tags]), tags, filename) + self.assertEqual(tuple([t.name for t in file_info.tags]), tags, filename) def test_guess_attributes_from_name0(self): self._test_guess_attributes_from_name( @@ -188,7 +188,7 @@ class TestFieldPermutations(TestCase): self.assertEqual(info.tags, (), filename) else: self.assertEqual( - [t.slug for t in info.tags], tags.split(','), + [t.name for t in info.tags], tags.split(','), filename ) @@ -342,8 +342,8 @@ class TestFieldPermutations(TestCase): info = FileInfo.from_filename(filename) self.assertEqual(info.title, "0001") self.assertEqual(len(info.tags), 2) - self.assertEqual(info.tags[0].slug, "tag1") - self.assertEqual(info.tags[1].slug, "tag2") + self.assertEqual(info.tags[0].name, "tag1") + self.assertEqual(info.tags[1].name, "tag2") self.assertIsNone(info.created) # Complex transformation with date in replacement string @@ -356,8 +356,8 @@ class TestFieldPermutations(TestCase): info = FileInfo.from_filename(filename) self.assertEqual(info.title, "0001") self.assertEqual(len(info.tags), 2) - self.assertEqual(info.tags[0].slug, "tag1") - self.assertEqual(info.tags[1].slug, "tag2") + self.assertEqual(info.tags[0].name, "tag1") + self.assertEqual(info.tags[1].name, "tag2") self.assertEqual(info.created.year, 2019) self.assertEqual(info.created.month, 9) self.assertEqual(info.created.day, 8) diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 08f7365da..a82c34f15 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -103,10 +103,7 @@ class MailAccountHandler(LoggingMixin): def _correspondent_from_name(self, name): try: - return Correspondent.objects.get_or_create( - name=name, defaults={ - "slug": slugify(name) - })[0] + return Correspondent.objects.get_or_create(name=name)[0] except DatabaseError as e: self.log( "error", From 33ca08f794f3fc6aad459b528e07f94692811b68 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 00:07:22 +0100 Subject: [PATCH 04/37] tags from folders: case insensitive --- src/documents/management/commands/document_consumer.py | 4 +++- src/documents/tests/test_management_consumer.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index b2f689aed..8ac60aa6d 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -29,7 +29,9 @@ def _tags_from_path(filepath): path_parts = Path(filepath).relative_to( settings.CONSUMPTION_DIR).parent.parts for part in path_parts: - tag_ids.add(Tag.objects.get_or_create(name=part)[0].pk) + tag_ids.add(Tag.objects.get_or_create(name__iexact=part, defaults={ + "name": part + })[0].pk) return tag_ids diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index 6973fdacf..b6a61a167 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -230,7 +230,7 @@ class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase): tag_names = ("existingTag", "Space Tag") # Create a Tag prior to consuming a file using it in path - tag_ids = [Tag.objects.create(name=tag_names[0]).pk,] + tag_ids = [Tag.objects.create(name="existingtag").pk,] self.t_start() From 234915a23a999c60ddae50d27b4c122ea7ac314a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 13:27:02 +0100 Subject: [PATCH 05/37] fixed test case. fixed bug with the decryption logic. --- .../management/commands/decrypt_documents.py | 3 ++- src/documents/tests/test_management_archiver.py | 16 +++++++--------- src/documents/tests/test_management_decrypt.py | 10 +++++----- src/documents/tests/test_management_exporter.py | 12 +++++------- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/documents/management/commands/decrypt_documents.py b/src/documents/management/commands/decrypt_documents.py index 2287bfa72..918f1a175 100644 --- a/src/documents/management/commands/decrypt_documents.py +++ b/src/documents/management/commands/decrypt_documents.py @@ -82,7 +82,8 @@ class Command(BaseCommand): with open(document.thumbnail_path, "wb") as f: f.write(raw_thumb) - document.save(update_fields=("storage_type", "filename")) + Document.objects.filter(id=document.id).update( + storage_type=document.storage_type, filename=document.filename) for path in old_paths: os.unlink(path) diff --git a/src/documents/tests/test_management_archiver.py b/src/documents/tests/test_management_archiver.py index fdb588acf..0828f05ff 100644 --- a/src/documents/tests/test_management_archiver.py +++ b/src/documents/tests/test_management_archiver.py @@ -16,25 +16,23 @@ sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf") class TestArchiver(DirectoriesMixin, TestCase): def make_models(self): - self.d1 = Document.objects.create(checksum="A", title="A", content="first document", pk=1, mime_type="application/pdf") - #self.d2 = Document.objects.create(checksum="B", title="B", content="second document") - #self.d3 = Document.objects.create(checksum="C", title="C", content="unrelated document") + return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf") def test_archiver(self): - shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf")) - self.make_models() + doc = self.make_models() + shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) call_command('document_archiver') def test_handle_document(self): - shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf")) - self.make_models() + doc = self.make_models() + shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) - handle_document(self.d1.pk) + handle_document(doc.pk) - doc = Document.objects.get(id=self.d1.id) + doc = Document.objects.get(id=doc.id) self.assertIsNotNone(doc.checksum) self.assertTrue(os.path.isfile(doc.archive_path)) diff --git a/src/documents/tests/test_management_decrypt.py b/src/documents/tests/test_management_decrypt.py index f68ea7cc1..1d64b1105 100644 --- a/src/documents/tests/test_management_decrypt.py +++ b/src/documents/tests/test_management_decrypt.py @@ -35,20 +35,20 @@ class TestDecryptDocuments(TestCase): PASSPHRASE="test" ).enable() - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg")) + doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) - Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg")) call_command('decrypt_documents') - doc = Document.objects.get(id=2) + doc.refresh_from_db() self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED) self.assertEqual(doc.filename, "0000002.pdf") self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf"))) self.assertTrue(os.path.isfile(doc.source_path)) - self.assertTrue(os.path.isfile(os.path.join(thumb_dir, "0000002.png"))) + self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png"))) self.assertTrue(os.path.isfile(doc.thumbnail_path)) with doc.source_file as f: diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index ab9733dc4..22d6fc7f6 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -24,13 +24,14 @@ class TestExportImport(DirectoriesMixin, TestCase): file = os.path.join(self.dirs.originals_dir, "0000001.pdf") - Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", id=1, mime_type="application/pdf") - Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) + Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf") + Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) Tag.objects.create(name="t") DocumentType.objects.create(name="dt") Correspondent.objects.create(name="c") target = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, target) call_command('document_exporter', target) @@ -66,9 +67,6 @@ class TestExportImport(DirectoriesMixin, TestCase): def test_export_missing_files(self): target = tempfile.mkdtemp() - Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", id=3, mime_type="application/pdf") + self.addCleanup(shutil.rmtree, target) + Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", mime_type="application/pdf") self.assertRaises(FileNotFoundError, call_command, 'document_exporter', target) - - def test_duplicate_titles(self): - # TODO - pass From 3677d0193f8dc2ad12e6e055d5c47ca3d54fd0ec Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 13:44:37 +0100 Subject: [PATCH 06/37] shadows --- .../app/components/common/input/tags/tags.component.html | 2 +- .../widgets/widget-frame/widget-frame.component.html | 2 +- .../document-detail/document-detail.component.html | 2 +- .../components/document-list/document-list.component.html | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src-ui/src/app/components/common/input/tags/tags.component.html b/src-ui/src/app/components/common/input/tags/tags.component.html index b2ad0944f..8029dd860 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.html +++ b/src-ui/src/app/components/common/input/tags/tags.component.html @@ -8,7 +8,7 @@
-
+
diff --git a/src-ui/src/app/components/dashboard/widgets/widget-frame/widget-frame.component.html b/src-ui/src/app/components/dashboard/widgets/widget-frame/widget-frame.component.html index d0f637935..1d7d2d906 100644 --- a/src-ui/src/app/components/dashboard/widgets/widget-frame/widget-frame.component.html +++ b/src-ui/src/app/components/dashboard/widgets/widget-frame/widget-frame.component.html @@ -1,4 +1,4 @@ -
+
{{title}}
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index 9f4c72cdd..e0b5c6da9 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -17,7 +17,7 @@
-
+
Download original
diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index 8608ed92b..1a8c7a781 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -24,7 +24,7 @@
-
+
@@ -53,7 +53,7 @@
-
+
@@ -85,7 +85,7 @@
- +
From b28d1bed0e669335554a71aff7d1889c252c1d82 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 22:16:57 +0100 Subject: [PATCH 07/37] fixes #112 --- src/documents/consumer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 19ca3ed7e..e4da51f1d 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -248,7 +248,7 @@ class Consumer(LoggingMixin): with open(self.path, "rb") as f: document = Document.objects.create( correspondent=file_info.correspondent, - title=file_info.title, + title=(self.override_title or file_info.title)[:127], content=text, mime_type=mime_type, checksum=hashlib.md5(f.read()).hexdigest(), @@ -265,12 +265,11 @@ class Consumer(LoggingMixin): self.apply_overrides(document) + document.save() + return document def apply_overrides(self, document): - if self.override_title: - document.title = self.override_title - if self.override_correspondent_id: document.correspondent = Correspondent.objects.get( pk=self.override_correspondent_id) From 206145461f93880fbc8f59f27f52c49bb3067096 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 22:17:17 +0100 Subject: [PATCH 08/37] locking media directory while deleting files --- src/documents/signals/handlers.py | 43 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 8121072bf..4fbbe8f8a 100755 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -171,29 +171,30 @@ def run_post_consume_script(sender, document, **kwargs): @receiver(models.signals.post_delete, sender=Document) def cleanup_document_deletion(sender, instance, using, **kwargs): - for f in (instance.source_path, - instance.archive_path, - instance.thumbnail_path): - if os.path.isfile(f): - try: - os.unlink(f) - logging.getLogger(__name__).debug( - f"Deleted file {f}.") - except OSError as e: - logging.getLogger(__name__).warning( - f"While deleting document {str(instance)}, the file " - f"{f} could not be deleted: {e}" - ) + with FileLock(settings.MEDIA_LOCK): + for f in (instance.source_path, + instance.archive_path, + instance.thumbnail_path): + if os.path.isfile(f): + try: + os.unlink(f) + logging.getLogger(__name__).debug( + f"Deleted file {f}.") + except OSError as e: + logging.getLogger(__name__).warning( + f"While deleting document {str(instance)}, the file " + f"{f} could not be deleted: {e}" + ) - delete_empty_directories( - os.path.dirname(instance.source_path), - root=settings.ORIGINALS_DIR - ) + delete_empty_directories( + os.path.dirname(instance.source_path), + root=settings.ORIGINALS_DIR + ) - delete_empty_directories( - os.path.dirname(instance.archive_path), - root=settings.ARCHIVE_DIR - ) + delete_empty_directories( + os.path.dirname(instance.archive_path), + root=settings.ARCHIVE_DIR + ) def validate_move(instance, old_path, new_path): From 80e6fa37370e84917cf1111aac4da7e02741f639 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 22:18:03 +0100 Subject: [PATCH 09/37] removed a janky test case that caused other test cases to fail --- src/documents/tests/test_file_handling.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 6d407a7ab..719b0078a 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -3,7 +3,6 @@ import hashlib import os import random import uuid -from concurrent.futures.thread import ThreadPoolExecutor from pathlib import Path from unittest import mock @@ -15,7 +14,6 @@ from .utils import DirectoriesMixin from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \ generate_unique_filename from ..models import Document, Correspondent -from ..sanity_checker import check_sanity class TestFileHandling(DirectoriesMixin, TestCase): @@ -573,21 +571,3 @@ def run(): for i in range(30): doc.title = str(random.randrange(1, 5)) doc.save() - - -class TestSuperMassive(DirectoriesMixin, TestCase): - - @override_settings(PAPERLESS_FILENAME_FORMAT="{title}") - def test_super_massive(self): - # try to save as many documents in parallel as possible. - # try to make the system fail. - - with ThreadPoolExecutor(max_workers=16) as executor: - results = [executor.submit(run) for i in range(16)] - - for r in results: - if r.exception(): - raise r.exception() - - # nope, everything still good. Thank you, lockfiles. - self.assertEqual(len(check_sanity()), 0) From f00193d28775e5483eeee8b2b7bef6b882b56798 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 22:38:52 +0100 Subject: [PATCH 10/37] layout fix --- .../document-card-small/document-card-small.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html index 95cf2e191..da469ebc4 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html @@ -1,6 +1,6 @@
-
+
From 1c30697e3b866235ad598750185c761468ae4b8f Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 9 Dec 2020 23:45:53 +0100 Subject: [PATCH 11/37] fixes #113 --- docker/docker-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index e2338842b..4832675ab 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -114,13 +114,13 @@ install_languages() { done } -initialize - # Install additional languages if specified if [[ ! -z "$PAPERLESS_OCR_LANGUAGES" ]]; then install_languages "$PAPERLESS_OCR_LANGUAGES" fi +initialize + if [[ "$1" != "/"* ]]; then exec sudo -HEu paperless python3 manage.py "$@" else From 61d93c5b9dee1f82abfad08a3d314c1fa803c91d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 00:02:45 +0100 Subject: [PATCH 12/37] added a progress bar to the reindex command. --- src/documents/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 65d767efc..8c9b00dd6 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -1,5 +1,6 @@ import logging +import tqdm from django.conf import settings from whoosh.writing import AsyncWriter @@ -23,7 +24,7 @@ def index_reindex(): ix = index.open_index(recreate=True) with AsyncWriter(ix) as writer: - for document in documents: + for document in tqdm.tqdm(documents): index.update_document(writer, document) From 0539df31405a3cfdf41e4ed7c599c41da91918cd Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 00:10:36 +0100 Subject: [PATCH 13/37] added progress bar to the document renamer. --- src/documents/management/commands/document_renamer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/documents/management/commands/document_renamer.py b/src/documents/management/commands/document_renamer.py index ba9e74de5..5d7d0d90c 100644 --- a/src/documents/management/commands/document_renamer.py +++ b/src/documents/management/commands/document_renamer.py @@ -1,3 +1,6 @@ +import logging + +import tqdm from django.core.management.base import BaseCommand from documents.models import Document @@ -18,6 +21,8 @@ class Command(Renderable, BaseCommand): self.verbosity = options["verbosity"] - for document in Document.objects.all(): + logging.getLogger().handlers[0].level = logging.ERROR + + for document in tqdm.tqdm(Document.objects.all()): # Saving the document again will generate a new filename and rename document.save() From a04cdc9ab5f11886ca8c3c5ea8b2c29eebfd8744 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 00:29:47 +0100 Subject: [PATCH 14/37] excluded the lockfile from the sanity checker. --- src/documents/sanity_checker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/documents/sanity_checker.py b/src/documents/sanity_checker.py index e3c4b1aec..bc0b689d4 100644 --- a/src/documents/sanity_checker.py +++ b/src/documents/sanity_checker.py @@ -46,6 +46,10 @@ def check_sanity(): for f in files: present_files.append(os.path.normpath(os.path.join(root, f))) + lockfile = os.path.normpath(settings.MEDIA_LOCK) + if lockfile in present_files: + present_files.remove(lockfile) + for doc in Document.objects.all(): # Check sanity of the thumbnail if not os.path.isfile(doc.thumbnail_path): From 87525669934f3017ced68d70f329a936a7b2415b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 00:30:35 +0100 Subject: [PATCH 15/37] changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 96578ac75..ce5cfe59a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,7 @@ This release focusses primarily on many small issues with the UI. conserve the original correspondents, types and titles as much as possible. * The filename formatter does not include the document ID in filenames anymore. It will rather append ``_01``, ``_02``, etc when it detects duplicate filenames. + * The docker image was trying check for installed languages before actually installing them. .. note:: From 660ff8ad45c08ce28b9fb6dd88bc0eed344e7cee Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 00:54:37 +0100 Subject: [PATCH 16/37] fixed an issue with the docker entrypoint script. --- docker/docker-entrypoint.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4832675ab..13a0ba035 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -25,6 +25,11 @@ wait_for_postgres() { host="${PAPERLESS_DBHOST}" port="${PAPERLESS_DBPORT}" + if [[ -z $port ]] ; + then + port="5432" + fi + while ! Date: Thu, 10 Dec 2020 00:59:03 +0100 Subject: [PATCH 17/37] a print() command somehow sneaked past my commit checks. --- src/documents/filters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/documents/filters.py b/src/documents/filters.py index 64ef826ce..b3c92eba3 100755 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -51,7 +51,6 @@ class TagsFilter(Filter): return qs for tag_id in tag_ids: - print(self.exclude, tag_id) if self.exclude: qs = qs.exclude(tags__id=tag_id) else: From e83ca71009a0203ef9df9db350a87ecd621c6f53 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 01:12:30 +0100 Subject: [PATCH 18/37] changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ce5cfe59a..a50fc31d5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,12 +32,12 @@ This release focusses primarily on many small issues with the UI. * Other * Fixed an issue with the docker image when a non-standard PostgreSQL port was used. + * The docker image was trying check for installed languages before actually installing them. * ``FILENAME_FORMAT`` placeholder for document types. * The filename formatter is now less restrictive with file names and tries to conserve the original correspondents, types and titles as much as possible. * The filename formatter does not include the document ID in filenames anymore. It will rather append ``_01``, ``_02``, etc when it detects duplicate filenames. - * The docker image was trying check for installed languages before actually installing them. .. note:: From 2a6401ba2ea01b1fca347e7051d22afb8bb21c25 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 02:14:26 +0100 Subject: [PATCH 19/37] added another library that's required to get this running on raspberry pi --- docker/local/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile index 461b9e4fc..d6e77da1d 100644 --- a/docker/local/Dockerfile +++ b/docker/local/Dockerfile @@ -20,6 +20,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ + libxslt-dev \ optipng \ pngquant \ qpdf \ From 2f5614c04bb36ff170d637e6f8ebf6088090a14b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 02:24:36 +0100 Subject: [PATCH 20/37] revert last commit. --- docker/local/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile index d6e77da1d..461b9e4fc 100644 --- a/docker/local/Dockerfile +++ b/docker/local/Dockerfile @@ -20,7 +20,6 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ - libxslt-dev \ optipng \ pngquant \ qpdf \ From 0c6c4a62d8db3dab486fbdda325d7ba8a93d02f1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 14:57:53 +0100 Subject: [PATCH 21/37] moved metadata extraction to the parsers --- src/documents/parsers.py | 4 ++++ src/documents/views.py | 36 +++++++----------------------- src/paperless_tesseract/parsers.py | 28 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 36ede3cce..228e2c86e 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -210,6 +210,7 @@ class DocumentParser(LoggingMixin): def __init__(self, logging_group): super().__init__() self.logging_group = logging_group + os.makedirs(settings.SCRATCH_DIR, exist_ok=True) self.tempdir = tempfile.mkdtemp( prefix="paperless-", dir=settings.SCRATCH_DIR) @@ -217,6 +218,9 @@ class DocumentParser(LoggingMixin): self.text = None self.date = None + def extract_metadata(self, document_path, mime_type): + return [] + def parse(self, document_path, mime_type): raise NotImplementedError() diff --git a/src/documents/views.py b/src/documents/views.py index 8dbb61dc7..b42ae1f96 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -1,11 +1,8 @@ -import logging import os -import re import tempfile from datetime import datetime from time import mktime -import pikepdf from django.conf import settings from django.db.models import Count, Max from django.http import HttpResponse, HttpResponseBadRequest, Http404 @@ -42,6 +39,7 @@ from .filters import ( LogFilterSet ) from .models import Correspondent, Document, Log, Tag, DocumentType +from .parsers import get_parser_class_for_mime_type from .serialisers import ( CorrespondentSerializer, DocumentSerializer, @@ -163,34 +161,16 @@ class DocumentViewSet(RetrieveModelMixin, disposition, filename) return response - def get_metadata(self, file, type): + def get_metadata(self, file, mime_type): if not os.path.isfile(file): return None - namespace_pattern = re.compile(r"\{(.*)\}(.*)") - - result = [] - if type == 'application/pdf': - pdf = pikepdf.open(file) - meta = pdf.open_metadata() - for key, value in meta.items(): - if isinstance(value, list): - value = " ".join([str(e) for e in value]) - value = str(value) - try: - m = namespace_pattern.match(key) - result.append({ - "namespace": m.group(1), - "prefix": meta.REVERSE_NS[m.group(1)], - "key": m.group(2), - "value": value - }) - except Exception as e: - logging.getLogger(__name__).warning( - f"Error while reading metadata {key}: {value}. Error: " - f"{e}" - ) - return result + parser_class = get_parser_class_for_mime_type(mime_type) + if parser_class: + parser = parser_class(logging_group=None) + return parser.extract_metadata(file, mime_type) + else: + return [] @action(methods=['get'], detail=True) def metadata(self, request, pk=None): diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index ebd706cdd..1cf6a769c 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -5,6 +5,7 @@ import subprocess import ocrmypdf import pdftotext +import pikepdf from PIL import Image from django.conf import settings from ocrmypdf import InputFileError, EncryptedPdfError @@ -18,6 +19,33 @@ class RasterisedDocumentParser(DocumentParser): image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.) """ + def extract_metadata(self, document_path, mime_type): + namespace_pattern = re.compile(r"\{(.*)\}(.*)") + + result = [] + if mime_type == 'application/pdf': + pdf = pikepdf.open(document_path) + meta = pdf.open_metadata() + for key, value in meta.items(): + if isinstance(value, list): + value = " ".join([str(e) for e in value]) + value = str(value) + try: + m = namespace_pattern.match(key) + result.append({ + "namespace": m.group(1), + "prefix": meta.REVERSE_NS[m.group(1)], + "key": m.group(2), + "value": value + }) + except Exception as e: + self.log( + "warning", + f"Error while reading metadata {key}: {value}. Error: " + f"{e}" + ) + return result + def get_thumbnail(self, document_path, mime_type): """ The thumbnail of a PDF is just a 500px wide image of the first page. From 3a8c612af36b0b72f9ee0ff05f9c5c08aaf2c6fc Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 10 Dec 2020 16:25:27 +0100 Subject: [PATCH 22/37] fixes #91 --- src/documents/serialisers.py | 7 ------- src/documents/tests/test_api.py | 10 ---------- 2 files changed, 17 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 600645061..db0e610d1 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1,6 +1,5 @@ import magic from django.utils.text import slugify -from pathvalidate import validate_filename, ValidationError from rest_framework import serializers from rest_framework.fields import SerializerMethodField @@ -179,12 +178,6 @@ class PostDocumentSerializer(serializers.Serializer): ) def validate_document(self, document): - - try: - validate_filename(document.name) - except ValidationError: - raise serializers.ValidationError("Invalid filename.") - document_data = document.file.read() mime_type = magic.from_buffer(document_data, mime=True) diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 572667406..ab1716366 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -403,16 +403,6 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, 400) m.assert_not_called() - @mock.patch("documents.views.async_task") - @mock.patch("documents.serialisers.validate_filename") - def test_upload_invalid_filename(self, validate_filename, async_task): - validate_filename.side_effect = ValidationError() - with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f: - response = self.client.post("/api/documents/post_document/", {"document": f}) - self.assertEqual(response.status_code, 400) - - async_task.assert_not_called() - @mock.patch("documents.views.async_task") def test_upload_with_title(self, async_task): with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f: From 77be7e0fe42ee48ea03ab333c4e4726d1a52fd15 Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Thu, 10 Dec 2020 16:28:02 +0100 Subject: [PATCH 23/37] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e754669a8..41f85af19 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Here's what you get: # Features * Performs OCR on your documents, adds selectable text to image only documents and adds tags, correspondents and document types to your documents. +* Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and can be configured freely. * Single page application front end. Should be pretty snappy. Will be mobile friendly in the future. * Includes a dashboard that shows basic statistics and has document upload. * Filtering by tags, correspondents, types, and more. From e02838b19893d299b598f29385f5a92de3e678fe Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Fri, 11 Dec 2020 17:49:32 +0100 Subject: [PATCH 24/37] fixes #122 --- docs/configuration.rst | 10 ++++++++++ paperless.conf.example | 1 + src/paperless/settings.py | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 2ec34f803..d3f47215b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -152,6 +152,16 @@ PAPERLESS_AUTO_LOGIN_USERNAME= Defaults to none, which disables this feature. + +PAPERLESS_COOKIE_PREFIX= + Specify a prefix that is added to the cookies used by paperless to identify + the currently logged in user. This is useful for when you're running two + instances of paperless on the same host. + + After changing this, you will have to login again. + + Defaults to ``""``, which does not alter the cookie names. + .. _configuration-ocr: OCR settings diff --git a/paperless.conf.example b/paperless.conf.example index 32c0e56b4..910fc22a0 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -30,6 +30,7 @@ #PAPERLESS_FORCE_SCRIPT_NAME= #PAPERLESS_STATIC_URL=/static/ #PAPERLESS_AUTO_LOGIN_USERNAME= +#PAPERLESS_COOKIE_PREFIX= # OCR settings diff --git a/src/paperless/settings.py b/src/paperless/settings.py index cf0c3e28d..1a6b80a0c 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -210,6 +210,12 @@ AUTH_PASSWORD_VALIDATORS = [ DATA_UPLOAD_MAX_NUMBER_FIELDS = None +COOKIE_PREFIX = os.getenv("PAPERLESS_COOKIE_PREFIX", "") + +CSRF_COOKIE_NAME = f"{COOKIE_PREFIX}csrftoken" +SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid" +LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language" + ############################################################################### # Database # ############################################################################### From 6d924e6f5c51923c44f64a0ff14dd3f908d21744 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Fri, 11 Dec 2020 17:57:56 +0100 Subject: [PATCH 25/37] fixes #115 --- src-ui/src/app/app.module.ts | 4 +++- .../document-card-large.component.html | 2 +- .../document-card-small.component.html | 2 +- .../document-list/document-list.component.html | 2 +- src-ui/src/app/pipes/document-title.pipe.spec.ts | 8 ++++++++ src-ui/src/app/pipes/document-title.pipe.ts | 16 ++++++++++++++++ 6 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 src-ui/src/app/pipes/document-title.pipe.spec.ts create mode 100644 src-ui/src/app/pipes/document-title.pipe.ts diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index ad12c9c47..675c882a7 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -48,6 +48,7 @@ import { WidgetFrameComponent } from './components/dashboard/widgets/widget-fram import { WelcomeWidgetComponent } from './components/dashboard/widgets/welcome-widget/welcome-widget.component'; import { YesNoPipe } from './pipes/yes-no.pipe'; import { FileSizePipe } from './pipes/file-size.pipe'; +import { DocumentTitlePipe } from './pipes/document-title.pipe'; @NgModule({ declarations: [ @@ -88,7 +89,8 @@ import { FileSizePipe } from './pipes/file-size.pipe'; WidgetFrameComponent, WelcomeWidgetComponent, YesNoPipe, - FileSizePipe + FileSizePipe, + DocumentTitlePipe ], imports: [ BrowserModule, diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html index bfc59b526..8f3fced66 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html @@ -12,7 +12,7 @@ {{(document.correspondent$ | async)?.name}} {{(document.correspondent$ | async)?.name}}: - {{document.title}} + {{document.title | documentTitle}}
#{{document.archive_serial_number}}
diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html index da469ebc4..86e28442c 100644 --- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html +++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html @@ -17,7 +17,7 @@ {{(document.correspondent$ | async)?.name}}: - {{document.title}} + {{document.title | documentTitle}}

diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index 1a8c7a781..c4fa0d4d7 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -105,7 +105,7 @@
ASN Correspondent - {{d.title}} + {{d.title | documentTitle}} diff --git a/src-ui/src/app/pipes/document-title.pipe.spec.ts b/src-ui/src/app/pipes/document-title.pipe.spec.ts new file mode 100644 index 000000000..29835abd6 --- /dev/null +++ b/src-ui/src/app/pipes/document-title.pipe.spec.ts @@ -0,0 +1,8 @@ +import { DocumentTitlePipe } from './document-title.pipe'; + +describe('DocumentTitlePipe', () => { + it('create an instance', () => { + const pipe = new DocumentTitlePipe(); + expect(pipe).toBeTruthy(); + }); +}); diff --git a/src-ui/src/app/pipes/document-title.pipe.ts b/src-ui/src/app/pipes/document-title.pipe.ts new file mode 100644 index 000000000..09445f595 --- /dev/null +++ b/src-ui/src/app/pipes/document-title.pipe.ts @@ -0,0 +1,16 @@ +import { Pipe, PipeTransform } from '@angular/core'; + +@Pipe({ + name: 'documentTitle' +}) +export class DocumentTitlePipe implements PipeTransform { + + transform(value: string): unknown { + if (value) { + return value + } else { + return "(no title)" + } + } + +} From 963043d8e9bc9a81e502904172605fa571fdf7b1 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Fri, 11 Dec 2020 14:22:02 -0800 Subject: [PATCH 26/37] Use ng2-pdf-viewer And remove now-unused safeUrl pipe --- src-ui/package-lock.json | 63 ++++++++++++++++++- src-ui/package.json | 1 + src-ui/src/app/app.module.ts | 6 +- .../document-detail.component.html | 12 ++-- .../document-detail.component.scss | 8 +++ src-ui/src/app/pipes/safe.pipe.spec.ts | 8 --- src-ui/src/app/pipes/safe.pipe.ts | 19 ------ 7 files changed, 78 insertions(+), 39 deletions(-) delete mode 100644 src-ui/src/app/pipes/safe.pipe.spec.ts delete mode 100644 src-ui/src/app/pipes/safe.pipe.ts diff --git a/src-ui/package-lock.json b/src-ui/package-lock.json index b6b66e1c6..5eca0b3c0 100644 --- a/src-ui/package-lock.json +++ b/src-ui/package-lock.json @@ -2215,6 +2215,11 @@ "integrity": "sha512-UV1/ZJMC+HcP902wWdpC43cAcGu0IQk/I5bXjP2aSuCjsk3cE74mDvFrLKga7oDC170ugOAYBwfT4DSQW3akDA==", "dev": true }, + "@types/pdfjs-dist": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@types/pdfjs-dist/-/pdfjs-dist-2.1.7.tgz", + "integrity": "sha512-nQIwcPUhkAIyn7x9NS0lR/qxYfd5unRtfGkMjvpgF4Sh28IXftRymaNmFKTTdejDNY25NDGSIyjwj/BRwAPexg==" + }, "@types/q": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz", @@ -3023,6 +3028,16 @@ "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", "dev": true }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "blob": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz", @@ -5508,6 +5523,13 @@ "schema-utils": "^2.6.5" } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "optional": true + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -8208,6 +8230,13 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, + "nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", + "dev": true, + "optional": true + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -8260,6 +8289,23 @@ "moment": "2.18.1" } }, + "ng2-pdf-viewer": { + "version": "6.3.2", + "resolved": "https://registry.npmjs.org/ng2-pdf-viewer/-/ng2-pdf-viewer-6.3.2.tgz", + "integrity": "sha512-H2tBhDd+Lq6CUzK2g54HsCcZDR2wTn1sDjYqKY3yF0Ydasl2R5ppCKynZBU/zge4EKvmHglJI120FbQMpJKDYQ==", + "requires": { + "@types/pdfjs-dist": "^2.1.4", + "pdfjs-dist": "^2.4.456", + "tslib": "^1.10.0" + }, + "dependencies": { + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + } + } + }, "ngx-cookie-service": { "version": "10.1.1", "resolved": "https://registry.npmjs.org/ngx-cookie-service/-/ngx-cookie-service-10.1.1.tgz", @@ -9270,6 +9316,11 @@ "sha.js": "^2.4.8" } }, + "pdfjs-dist": { + "version": "2.5.207", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.5.207.tgz", + "integrity": "sha512-xGDUhnCYPfHy+unMXCLCJtlpZaaZ17Ew3WIL0tnSgKFUZXHAPD49GO9xScyszSsQMoutNDgRb+rfBXIaX/lJbw==" + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -13228,7 +13279,11 @@ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", "dev": true, - "optional": true + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } }, "glob-parent": { "version": "3.1.0", @@ -13832,7 +13887,11 @@ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", "dev": true, - "optional": true + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } }, "glob-parent": { "version": "3.1.0", diff --git a/src-ui/package.json b/src-ui/package.json index af3334db9..6293f2672 100644 --- a/src-ui/package.json +++ b/src-ui/package.json @@ -23,6 +23,7 @@ "@ng-bootstrap/ng-bootstrap": "^8.0.0", "bootstrap": "^4.5.0", "ng-bootstrap": "^1.6.3", + "ng2-pdf-viewer": "^6.3.2", "ngx-cookie-service": "^10.1.1", "ngx-file-drop": "^10.0.0", "ngx-infinite-scroll": "^9.1.0", diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index 7f2e8414e..40c0991e7 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -14,7 +14,6 @@ import { LogsComponent } from './components/manage/logs/logs.component'; import { SettingsComponent } from './components/manage/settings/settings.component'; import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { DatePipe } from '@angular/common'; -import { SafePipe } from './pipes/safe.pipe'; import { NotFoundComponent } from './components/not-found/not-found.component'; import { CorrespondentListComponent } from './components/manage/correspondent-list/correspondent-list.component'; import { DeleteDialogComponent } from './components/common/delete-dialog/delete-dialog.component'; @@ -45,6 +44,7 @@ import { SavedViewWidgetComponent } from './components/dashboard/widgets/saved-v import { StatisticsWidgetComponent } from './components/dashboard/widgets/statistics-widget/statistics-widget.component'; import { UploadFileWidgetComponent } from './components/dashboard/widgets/upload-file-widget/upload-file-widget.component'; import { WidgetFrameComponent } from './components/dashboard/widgets/widget-frame/widget-frame.component'; +import { PdfViewerModule } from 'ng2-pdf-viewer'; @NgModule({ declarations: [ @@ -57,7 +57,6 @@ import { WidgetFrameComponent } from './components/dashboard/widgets/widget-fram DocumentTypeListComponent, LogsComponent, SettingsComponent, - SafePipe, NotFoundComponent, CorrespondentEditDialogComponent, DeleteDialogComponent, @@ -92,7 +91,8 @@ import { WidgetFrameComponent } from './components/dashboard/widgets/widget-fram FormsModule, ReactiveFormsModule, NgxFileDropModule, - InfiniteScrollModule + InfiniteScrollModule, + PdfViewerModule ], providers: [ DatePipe, diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index 5a5563571..93aec64bc 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -14,14 +14,14 @@ Download - +
Download original
- + @@ -66,10 +66,8 @@
- -

Your browser does not support PDFs. - Download the PDF.

- - +
+ +
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.scss b/src-ui/src/app/components/document-detail/document-detail.component.scss index e69de29bb..b4d720018 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.scss +++ b/src-ui/src/app/components/document-detail/document-detail.component.scss @@ -0,0 +1,8 @@ +.pdf-viewer-container { + height: calc(100vh - 160px); + top: 70px; + position: sticky; + padding: 10px; + background-color: gray; + overflow-y: scroll; +} diff --git a/src-ui/src/app/pipes/safe.pipe.spec.ts b/src-ui/src/app/pipes/safe.pipe.spec.ts deleted file mode 100644 index 49ee0ad14..000000000 --- a/src-ui/src/app/pipes/safe.pipe.spec.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { SafePipe } from './safe.pipe'; - -describe('SafePipe', () => { - it('create an instance', () => { - const pipe = new SafePipe(); - expect(pipe).toBeTruthy(); - }); -}); diff --git a/src-ui/src/app/pipes/safe.pipe.ts b/src-ui/src/app/pipes/safe.pipe.ts deleted file mode 100644 index f2d77a72d..000000000 --- a/src-ui/src/app/pipes/safe.pipe.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Pipe, PipeTransform } from '@angular/core'; -import { DomSanitizer } from '@angular/platform-browser'; - -@Pipe({ - name: 'safe' -}) -export class SafePipe implements PipeTransform { - - constructor(private sanitizer: DomSanitizer) { } - - transform(url) { - if (url == null) { - return this.sanitizer.bypassSecurityTrustResourceUrl("") - } else { - return this.sanitizer.bypassSecurityTrustResourceUrl(url); - } - } - -} \ No newline at end of file From 23354f8a0009fcc278b0f1bcde26b4ed006caa9c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 12 Dec 2020 01:19:22 +0100 Subject: [PATCH 27/37] fixes #125 --- src/documents/signals/handlers.py | 12 ++++++++---- src/documents/tests/test_management_retagger.py | 14 +++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 4fbbe8f8a..27aa37908 100755 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -7,6 +7,7 @@ from django.contrib.admin.models import ADDITION, LogEntry from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.db import models, DatabaseError +from django.db.models import Q from django.dispatch import receiver from django.utils import timezone from filelock import FileLock @@ -121,11 +122,14 @@ def set_tags(sender, classifier=None, replace=False, **kwargs): + if replace: - document.tags.clear() - current_tags = set([]) - else: - current_tags = set(document.tags.all()) + document.tags.exclude( + Q(is_inbox_tag=True) | + (Q(match="") & ~Q(matching_algorithm=Tag.MATCH_AUTO)) + ).delete() + + current_tags = set(document.tags.all()) matched_tags = matching.match_tags(document.content, classifier) diff --git a/src/documents/tests/test_management_retagger.py b/src/documents/tests/test_management_retagger.py index 2346b6527..2397b0cc8 100644 --- a/src/documents/tests/test_management_retagger.py +++ b/src/documents/tests/test_management_retagger.py @@ -14,6 +14,11 @@ class TestRetagger(DirectoriesMixin, TestCase): self.tag_first = Tag.objects.create(name="tag1", match="first", matching_algorithm=Tag.MATCH_ANY) self.tag_second = Tag.objects.create(name="tag2", match="second", matching_algorithm=Tag.MATCH_ANY) + self.tag_inbox = Tag.objects.create(name="test", is_inbox_tag=True) + self.tag_no_match = Tag.objects.create(name="test2") + + self.d3.tags.add(self.tag_inbox) + self.d3.tags.add(self.tag_no_match) self.correspondent_first = Correspondent.objects.create( name="c1", match="first", matching_algorithm=Correspondent.MATCH_ANY) @@ -38,7 +43,7 @@ class TestRetagger(DirectoriesMixin, TestCase): self.assertEqual(d_first.tags.count(), 1) self.assertEqual(d_second.tags.count(), 1) - self.assertEqual(d_unrelated.tags.count(), 0) + self.assertEqual(d_unrelated.tags.count(), 2) self.assertEqual(d_first.tags.first(), self.tag_first) self.assertEqual(d_second.tags.first(), self.tag_second) @@ -56,3 +61,10 @@ class TestRetagger(DirectoriesMixin, TestCase): self.assertEqual(d_first.correspondent, self.correspondent_first) self.assertEqual(d_second.correspondent, self.correspondent_second) + + def test_force_preserve_inbox(self): + call_command('document_retagger', '--tags', '--overwrite') + + d_first, d_second, d_unrelated = self.get_updated_docs() + + self.assertCountEqual([tag.id for tag in d_unrelated.tags.all()], [self.tag_inbox.id, self.tag_no_match.id]) From 4e4145df4c7098b6c5983c3707d9ba4a99864a74 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 12 Dec 2020 01:23:26 +0100 Subject: [PATCH 28/37] tests --- src/documents/tests/test_management_retagger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/documents/tests/test_management_retagger.py b/src/documents/tests/test_management_retagger.py index 2397b0cc8..2d2533341 100644 --- a/src/documents/tests/test_management_retagger.py +++ b/src/documents/tests/test_management_retagger.py @@ -67,4 +67,6 @@ class TestRetagger(DirectoriesMixin, TestCase): d_first, d_second, d_unrelated = self.get_updated_docs() + self.assertCountEqual([tag.id for tag in d_first.tags.all()], [self.tag_first.id]) + self.assertCountEqual([tag.id for tag in d_second.tags.all()], [self.tag_second.id]) self.assertCountEqual([tag.id for tag in d_unrelated.tags.all()], [self.tag_inbox.id, self.tag_no_match.id]) From 2bd9bb0d9ccbc9bda4b99a5adc0ef4ebd3cfa51f Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 12 Dec 2020 02:06:43 +0100 Subject: [PATCH 29/37] made a serious mistake. fixed. --- src/documents/signals/handlers.py | 6 +++--- src/documents/tests/test_management_retagger.py | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 27aa37908..586897585 100755 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -124,9 +124,9 @@ def set_tags(sender, **kwargs): if replace: - document.tags.exclude( - Q(is_inbox_tag=True) | - (Q(match="") & ~Q(matching_algorithm=Tag.MATCH_AUTO)) + Document.tags.through.objects.filter(document=document).exclude( + Q(tag__is_inbox_tag=True)).exclude( + Q(tag__match="") & ~Q(tag__matching_algorithm=Tag.MATCH_AUTO) ).delete() current_tags = set(document.tags.all()) diff --git a/src/documents/tests/test_management_retagger.py b/src/documents/tests/test_management_retagger.py index 2d2533341..907a23d09 100644 --- a/src/documents/tests/test_management_retagger.py +++ b/src/documents/tests/test_management_retagger.py @@ -20,6 +20,7 @@ class TestRetagger(DirectoriesMixin, TestCase): self.d3.tags.add(self.tag_inbox) self.d3.tags.add(self.tag_no_match) + self.correspondent_first = Correspondent.objects.create( name="c1", match="first", matching_algorithm=Correspondent.MATCH_ANY) self.correspondent_second = Correspondent.objects.create( @@ -62,11 +63,16 @@ class TestRetagger(DirectoriesMixin, TestCase): self.assertEqual(d_first.correspondent, self.correspondent_first) self.assertEqual(d_second.correspondent, self.correspondent_second) - def test_force_preserve_inbox(self): + def test_overwrite_preserve_inbox(self): + self.d1.tags.add(self.tag_second) + call_command('document_retagger', '--tags', '--overwrite') d_first, d_second, d_unrelated = self.get_updated_docs() + self.assertIsNotNone(Tag.objects.get(id=self.tag_second.id)) + self.assertCountEqual([tag.id for tag in d_first.tags.all()], [self.tag_first.id]) self.assertCountEqual([tag.id for tag in d_second.tags.all()], [self.tag_second.id]) self.assertCountEqual([tag.id for tag in d_unrelated.tags.all()], [self.tag_inbox.id, self.tag_no_match.id]) + From 4910573b42bad3c7b7a1b82aad3d6cea3aa7ff7e Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Sat, 12 Dec 2020 08:43:03 -0800 Subject: [PATCH 30/37] Fix PDF column width layout issues --- .../document-detail/document-detail.component.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index bec8a59a2..bdd1132fa 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -35,7 +35,7 @@
-
+
@@ -171,9 +171,9 @@
-
+
-
\ No newline at end of file +
From 0416e3ee65de1d42308015b013222c1272ba1dc9 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Sat, 12 Dec 2020 08:50:52 -0800 Subject: [PATCH 31/37] Bottom margin on columns for mobile stacking --- .../components/document-detail/document-detail.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index bdd1132fa..6f1aacdf5 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -35,7 +35,7 @@
-
+
@@ -171,7 +171,7 @@
-
+
From 45d31f973536192a8aec7e17e7d60c6ebdca7f7b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 12 Dec 2020 18:25:15 +0100 Subject: [PATCH 32/37] fixes bauerj/paperless_app#23 and most of all other scanner apps out there. --- src/paperless_tesseract/parsers.py | 24 ++++++++++++++++++++ src/paperless_tesseract/tests/test_parser.py | 15 +++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index 1cf6a769c..80e200f27 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -110,6 +110,24 @@ class RasterisedDocumentParser(DocumentParser): f"Error while getting DPI from image {image}: {e}") return None + def calculate_a4_dpi(self, image): + try: + with Image.open(image) as im: + width, height = im.size + # divide image width by A4 width (210mm) in inches. + dpi = int(width / (21 / 2.54)) + self.log( + 'debug', + f"Estimated DPI {dpi} based on image width {width}" + ) + return dpi + + except Exception as e: + self.log( + 'warning', + f"Error while calculating DPI for image {image}: {e}") + return None + def parse(self, document_path, mime_type): mode = settings.OCR_MODE @@ -162,6 +180,7 @@ class RasterisedDocumentParser(DocumentParser): if self.is_image(mime_type): dpi = self.get_dpi(document_path) + a4_dpi = self.calculate_a4_dpi(document_path) if dpi: self.log( "debug", @@ -170,6 +189,8 @@ class RasterisedDocumentParser(DocumentParser): ocr_args['image_dpi'] = dpi elif settings.OCR_IMAGE_DPI: ocr_args['image_dpi'] = settings.OCR_IMAGE_DPI + elif a4_dpi: + ocr_args['image_dpi'] = a4_dpi else: raise ParseError( f"Cannot produce archive PDF for image {document_path}, " @@ -241,6 +262,9 @@ def strip_excess_whitespace(text): def get_text_from_pdf(pdf_file): + if not os.path.isfile(pdf_file): + return None + with open(pdf_file, "rb") as f: try: pdf = pdftotext.PDF(f) diff --git a/src/paperless_tesseract/tests/test_parser.py b/src/paperless_tesseract/tests/test_parser.py index 8834ec755..7be176663 100644 --- a/src/paperless_tesseract/tests/test_parser.py +++ b/src/paperless_tesseract/tests/test_parser.py @@ -164,8 +164,21 @@ class TestParser(DirectoriesMixin, TestCase): self.assertRaises(ParseError, f) + @mock.patch("paperless_tesseract.parsers.ocrmypdf.ocr") + def test_image_calc_a4_dpi(self, m): + parser = RasterisedDocumentParser(None) - def test_image_no_dpi_fail(self): + parser.parse(os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"), "image/png") + + m.assert_called_once() + + args, kwargs = m.call_args + + self.assertEqual(kwargs['image_dpi'], 62) + + @mock.patch("paperless_tesseract.parsers.RasterisedDocumentParser.calculate_a4_dpi") + def test_image_dpi_fail(self, m): + m.return_value = None parser = RasterisedDocumentParser(None) def f(): From 1c3ecccea251c6296bee85adf299c3dfa50da097 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 12 Dec 2020 20:32:11 +0100 Subject: [PATCH 33/37] made the file renamer somewhat faster. --- src/documents/management/commands/document_renamer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/documents/management/commands/document_renamer.py b/src/documents/management/commands/document_renamer.py index 5d7d0d90c..745d2d03d 100644 --- a/src/documents/management/commands/document_renamer.py +++ b/src/documents/management/commands/document_renamer.py @@ -2,6 +2,7 @@ import logging import tqdm from django.core.management.base import BaseCommand +from django.db.models.signals import post_save from documents.models import Document from ...mixins import Renderable @@ -24,5 +25,4 @@ class Command(Renderable, BaseCommand): logging.getLogger().handlers[0].level = logging.ERROR for document in tqdm.tqdm(Document.objects.all()): - # Saving the document again will generate a new filename and rename - document.save() + post_save.send(Document, instance=document) From 7f0d5f45f1157bf405786a711c6f2b8ac318e783 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 12 Dec 2020 22:56:44 +0100 Subject: [PATCH 34/37] a couple adjustments for the document viewer. --- .../document-detail/document-detail.component.html | 6 +++--- .../document-detail/document-detail.component.scss | 2 -- .../components/document-detail/document-detail.component.ts | 4 ++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index 6f1aacdf5..f9f6e57ef 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -35,7 +35,7 @@
-
+
@@ -172,8 +172,8 @@
-
- +
+
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.scss b/src-ui/src/app/components/document-detail/document-detail.component.scss index b4d720018..998653bab 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.scss +++ b/src-ui/src/app/components/document-detail/document-detail.component.scss @@ -2,7 +2,5 @@ height: calc(100vh - 160px); top: 70px; position: sticky; - padding: 10px; background-color: gray; - overflow-y: scroll; } diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index 329077693..c80a8b1ce 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -59,6 +59,10 @@ export class DocumentDetailComponent implements OnInit { private documentListViewService: DocumentListViewService, private titleService: Title) { } + getContentType() { + return this.metadata?.has_archive_version ? 'application/pdf' : this.metadata?.original_mime_type + } + ngOnInit(): void { this.documentForm.valueChanges.subscribe(wow => { Object.assign(this.document, this.documentForm.value) From 57ead7d3552d29fb398cc237ea84c0117117898a Mon Sep 17 00:00:00 2001 From: rYR79435 <60985157+rYR79435@users.noreply.github.com> Date: Sun, 13 Dec 2020 13:30:30 +0100 Subject: [PATCH 35/37] Open GitHub and Documentation links in a new tab --- src-ui/src/app/components/app-frame/app-frame.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 3f326afdd..1cedeefde 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -132,7 +132,7 @@
-

{{message}}

-

{{message2}}

+

{{messageBold}}

+

{{message}}

- +
\ No newline at end of file diff --git a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.scss b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.scss similarity index 100% rename from src-ui/src/app/components/common/delete-dialog/delete-dialog.component.scss rename to src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.scss diff --git a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.spec.ts b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.spec.ts similarity index 52% rename from src-ui/src/app/components/common/delete-dialog/delete-dialog.component.spec.ts rename to src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.spec.ts index 33c7d6e88..fe08dc57a 100644 --- a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.spec.ts +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.spec.ts @@ -1,20 +1,20 @@ import { ComponentFixture, TestBed } from '@angular/core/testing'; -import { DeleteDialogComponent } from './delete-dialog.component'; +import { ConfirmDialogComponent } from './confirm-dialog.component'; -describe('DeleteDialogComponent', () => { - let component: DeleteDialogComponent; - let fixture: ComponentFixture; +describe('ConfirmDialogComponent', () => { + let component: ConfirmDialogComponent; + let fixture: ComponentFixture; beforeEach(async () => { await TestBed.configureTestingModule({ - declarations: [ DeleteDialogComponent ] + declarations: [ ConfirmDialogComponent ] }) .compileComponents(); }); beforeEach(() => { - fixture = TestBed.createComponent(DeleteDialogComponent); + fixture = TestBed.createComponent(ConfirmDialogComponent); component = fixture.componentInstance; fixture.detectChanges(); }); diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts new file mode 100644 index 000000000..e207f4598 --- /dev/null +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts @@ -0,0 +1,37 @@ +import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; + +@Component({ + selector: 'app-confirm-dialog', + templateUrl: './confirm-dialog.component.html', + styleUrls: ['./confirm-dialog.component.scss'] +}) +export class ConfirmDialogComponent implements OnInit { + + constructor(public activeModal: NgbActiveModal) { } + + @Output() + public confirmClicked = new EventEmitter() + + @Input() + title = "Confirmation" + + @Input() + messageBold + + @Input() + message + + @Input() + btnClass = "btn-primary" + + @Input() + btnCaption = "Confirm" + + ngOnInit(): void { + } + + cancelClicked() { + this.activeModal.close() + } +} diff --git a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts deleted file mode 100644 index 20114c78c..000000000 --- a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; -import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; - -@Component({ - selector: 'app-delete-dialog', - templateUrl: './delete-dialog.component.html', - styleUrls: ['./delete-dialog.component.scss'] -}) -export class DeleteDialogComponent implements OnInit { - - constructor(public activeModal: NgbActiveModal) { } - - @Output() - public deleteClicked = new EventEmitter() - - @Input() - title = "Delete confirmation" - - @Input() - message = "Do you really want to delete this?" - - @Input() - message2 - - ngOnInit(): void { - } - - cancelClicked() { - this.activeModal.close() - } -} diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index c80a8b1ce..4aac9c769 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -13,7 +13,7 @@ import { CorrespondentService } from 'src/app/services/rest/correspondent.servic import { DocumentTypeService } from 'src/app/services/rest/document-type.service'; import { DocumentService } from 'src/app/services/rest/document.service'; import { environment } from 'src/environments/environment'; -import { DeleteDialogComponent } from '../common/delete-dialog/delete-dialog.component'; +import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component'; import { CorrespondentEditDialogComponent } from '../manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component'; import { DocumentTypeEditDialogComponent } from '../manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component'; @@ -155,10 +155,13 @@ export class DocumentDetailComponent implements OnInit { } delete() { - let modal = this.modalService.open(DeleteDialogComponent, {backdrop: 'static'}) - modal.componentInstance.message = `Do you really want to delete document '${this.document.title}'?` - modal.componentInstance.message2 = `The files for this document will be deleted permanently. This operation cannot be undone.` - modal.componentInstance.deleteClicked.subscribe(() => { + let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'}) + modal.componentInstance.title = "Confirm delete" + modal.componentInstance.messageBold = `Do you really want to delete document '${this.document.title}'?` + modal.componentInstance.message = `The files for this document will be deleted permanently. This operation cannot be undone.` + modal.componentInstance.btnClass = "btn-danger" + modal.componentInstance.btnCaption = "Delete document" + modal.componentInstance.confirmClicked.subscribe(() => { this.documentsService.delete(this.document).subscribe(() => { modal.close() this.close() diff --git a/src-ui/src/app/components/manage/generic-list/generic-list.component.ts b/src-ui/src/app/components/manage/generic-list/generic-list.component.ts index d5477d010..59a5f09ed 100644 --- a/src-ui/src/app/components/manage/generic-list/generic-list.component.ts +++ b/src-ui/src/app/components/manage/generic-list/generic-list.component.ts @@ -4,7 +4,7 @@ import { MatchingModel, MATCHING_ALGORITHMS, MATCH_AUTO } from 'src/app/data/mat import { ObjectWithId } from 'src/app/data/object-with-id'; import { SortableDirective, SortEvent } from 'src/app/directives/sortable.directive'; import { AbstractPaperlessService } from 'src/app/services/rest/abstract-paperless-service'; -import { DeleteDialogComponent } from '../../common/delete-dialog/delete-dialog.component'; +import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component'; @Directive() export abstract class GenericListComponent implements OnInit { @@ -88,10 +88,13 @@ export abstract class GenericListComponent implements On } openDeleteDialog(object: T) { - var activeModal = this.modalService.open(DeleteDialogComponent, {backdrop: 'static'}) - activeModal.componentInstance.message = `Do you really want to delete ${this.getObjectName(object)}?` - activeModal.componentInstance.message2 = "Associated documents will not be deleted." - activeModal.componentInstance.deleteClicked.subscribe(() => { + var activeModal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'}) + activeModal.componentInstance.title = "Confirm delete" + activeModal.componentInstance.messageBold = `Do you really want to delete ${this.getObjectName(object)}?` + activeModal.componentInstance.message = "Associated documents will not be deleted." + activeModal.componentInstance.btnClass = "btn-danger" + activeModal.componentInstance.btnCaption = "Delete" + activeModal.componentInstance.confirmPressed.subscribe(() => { this.service.delete(object).subscribe(_ => { activeModal.close() this.reloadData() From f65fb857b33cd00e1c5a16e6a2ba8242ae378004 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 13 Dec 2020 14:56:44 +0100 Subject: [PATCH 37/37] refactored metadata views --- src-ui/src/app/app.module.ts | 4 +- .../document-detail.component.html | 49 +------------------ .../metadata-collapse.component.html | 23 +++++++++ .../metadata-collapse.component.scss | 0 .../metadata-collapse.component.spec.ts | 25 ++++++++++ .../metadata-collapse.component.ts | 23 +++++++++ 6 files changed, 76 insertions(+), 48 deletions(-) create mode 100644 src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.html create mode 100644 src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.scss create mode 100644 src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.spec.ts create mode 100644 src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.ts diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index a1ae10d14..5b92364d2 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -49,6 +49,7 @@ import { WelcomeWidgetComponent } from './components/dashboard/widgets/welcome-w import { YesNoPipe } from './pipes/yes-no.pipe'; import { FileSizePipe } from './pipes/file-size.pipe'; import { DocumentTitlePipe } from './pipes/document-title.pipe'; +import { MetadataCollapseComponent } from './components/document-detail/metadata-collapse/metadata-collapse.component'; @NgModule({ declarations: [ @@ -89,7 +90,8 @@ import { DocumentTitlePipe } from './pipes/document-title.pipe'; WelcomeWidgetComponent, YesNoPipe, FileSizePipe, - DocumentTitlePipe + DocumentTitlePipe, + MetadataCollapseComponent ], imports: [ BrowserModule, diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index f9f6e57ef..c0114f709 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -110,53 +110,8 @@
-
- - Original document metadata -
- -
- - - - - - - -
{{m.prefix}}:{{m.key}}{{m.value}}
-
- -
- - Archived document metadata -
- -
- - - - - - - -
{{m.prefix}}:{{m.key}}{{m.value}}
-
+ + diff --git a/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.html b/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.html new file mode 100644 index 000000000..e8fda1d0b --- /dev/null +++ b/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.html @@ -0,0 +1,23 @@ +
+ + {{title}} +
+ +
+ + + + + + + +
{{m.prefix}}:{{m.key}}{{m.value}}
+
\ No newline at end of file diff --git a/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.scss b/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.spec.ts b/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.spec.ts new file mode 100644 index 000000000..2bd96760b --- /dev/null +++ b/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { MetadataCollapseComponent } from './metadata-collapse.component'; + +describe('MetadataCollapseComponent', () => { + let component: MetadataCollapseComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ MetadataCollapseComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(MetadataCollapseComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.ts b/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.ts new file mode 100644 index 000000000..160274e41 --- /dev/null +++ b/src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.ts @@ -0,0 +1,23 @@ +import { Component, Input, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-metadata-collapse', + templateUrl: './metadata-collapse.component.html', + styleUrls: ['./metadata-collapse.component.scss'] +}) +export class MetadataCollapseComponent implements OnInit { + + constructor() { } + + expand = false + + @Input() + metadata + + @Input() + title = "Metadata" + + ngOnInit(): void { + } + +}