diff --git a/docs/configuration.md b/docs/configuration.md index d3874256f..722db8a4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1095,6 +1095,27 @@ barcode. Defaults to "ASN" +`PAPERLESS_CONSUMER_BARCODE_UPSCALE=` + +: Defines the upscale factor used in barcode detection. +Improves the detection of small barcodes, i.e. with a value of 1.5 by +upscaling the document beforce the detection process. Upscaling will +only take place if value is bigger than 1.0. Otherwise upscaling will +not be performed to save resources. Try using in combination with +PAPERLESS_CONSUMER_BARCODE_DPI set to a value higher than default. + + Defaults to 0.0 + +`PAPERLESS_CONSUMER_BARCODE_DPI=` + +: During barcode detection every page from a PDF document needs +to be converted to an image. A dpi value can be specified in the +conversion process. Default is 300. If the detection of small barcodes +fails a bigger dpi value i.e. 600 can fix the issue. Try using in +combination with PAPERLESS_CONSUMER_BARCODE_UPSCALE bigger than 1.0. + + Defaults to "300" + ## Binaries There are a few external software packages that Paperless expects to diff --git a/paperless.conf.example b/paperless.conf.example index 6bd70697e..9b168db0c 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -66,6 +66,8 @@ #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false #PAPERLESS_CONSUMER_ENABLE_BARCODES=false #PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT +#PAPERLESS_CONSUMER_BARCODE_UPSCALE=0.0 +#PAPERLESS_CONSUMER_BARCODE_DPI=300 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh #PAPERLESS_FILENAME_DATE_ORDER=YMD diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index 848239b9f..051ce707e 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -5,7 +5,7 @@ export const environment = { apiBaseUrl: document.baseURI + 'api/', apiVersion: '3', appTitle: 'Paperless-ngx', - version: '1.16.4', + version: '1.16.4-dev', webSocketHost: window.location.host, webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:', webSocketBaseUrl: base_url.pathname + 'ws/', diff --git a/src/documents/barcodes.py b/src/documents/barcodes.py index f3d59bc5b..3650593ae 100644 --- a/src/documents/barcodes.py +++ b/src/documents/barcodes.py @@ -203,11 +203,21 @@ class BarcodeReader: try: pages_from_path = convert_from_path( self.pdf_file, - dpi=300, + dpi=settings.CONSUMER_BARCODE_DPI, output_folder=self.temp_dir.name, ) for current_page_number, page in enumerate(pages_from_path): + factor = settings.CONSUMER_BARCODE_UPSCALE + if factor > 1.0: + logger.debug( + f"Upscaling image by {factor} for better barcode detection", + ) + x, y = page.size + page = page.resize( + (int(round(x * factor)), (int(round(y * factor)))), + ) + for barcode_value in reader(page): self.barcodes.append( Barcode(current_page_number, barcode_value), diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 6c28f720b..a5aec39c4 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -213,15 +213,12 @@ class OwnedObjectSerializer(serializers.ModelSerializer, SetPermissionsMixin): # other methods in mixin def create(self, validated_data): - if self.user and ( - "owner" not in validated_data or validated_data["owner"] is None - ): + # default to current user if not set + if "owner" not in validated_data and self.user: validated_data["owner"] = self.user permissions = None if "set_permissions" in validated_data: permissions = validated_data.pop("set_permissions") - if "user" not in permissions or permissions["user"] is None: - validated_data["owner"] = None instance = super().create(validated_data) if permissions is not None: self._set_permissions(permissions, instance) diff --git a/src/documents/tests/samples/barcodes/barcode-qr-asn-000123-upscale-dpi.pdf b/src/documents/tests/samples/barcodes/barcode-qr-asn-000123-upscale-dpi.pdf new file mode 100644 index 000000000..3d6d8eacb Binary files /dev/null and b/src/documents/tests/samples/barcodes/barcode-qr-asn-000123-upscale-dpi.pdf differ diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 90684b338..40a1ca4a3 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -28,6 +28,7 @@ from django.contrib.auth.models import User from django.test import override_settings from django.utils import timezone from guardian.shortcuts import assign_perm +from guardian.shortcuts import get_perms from rest_framework import status from rest_framework.test import APITestCase from whoosh.writing import AsyncWriter @@ -3855,7 +3856,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase): status.HTTP_200_OK, ) - def test_object_permissions(self): + def test_api_get_object_permissions(self): user1 = User.objects.create_user(username="test1") user2 = User.objects.create_user(username="test2") user1.user_permissions.add(*Permission.objects.filter(codename="view_document")) @@ -3886,18 +3887,16 @@ class TestApiAuth(DirectoriesMixin, APITestCase): status.HTTP_404_NOT_FOUND, ) - def test_api_set_permissions(self): + def test_api_default_owner(self): """ GIVEN: - - API request to create an object (Tag) that supplies set_permissions object + - API request to create an object (Tag) WHEN: - - owner is passed as null or as a user id - - view > users is set + - owner is not set at all THEN: - - Object permissions are set appropriately + - Object created with current user as owner """ user1 = User.objects.create_superuser(username="user1") - user2 = User.objects.create(username="user2") self.client.force_authenticate(user1) @@ -3907,11 +3906,73 @@ class TestApiAuth(DirectoriesMixin, APITestCase): { "name": "test1", "matching_algorithm": MatchingModel.MATCH_AUTO, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + tag1 = Tag.objects.filter(name="test1").first() + self.assertEqual(tag1.owner, user1) + + def test_api_set_no_owner(self): + """ + GIVEN: + - API request to create an object (Tag) + WHEN: + - owner is passed as None + THEN: + - Object created with no owner + """ + user1 = User.objects.create_superuser(username="user1") + + self.client.force_authenticate(user1) + + response = self.client.post( + "/api/tags/", + json.dumps( + { + "name": "test1", + "matching_algorithm": MatchingModel.MATCH_AUTO, + "owner": None, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + + tag1 = Tag.objects.filter(name="test1").first() + self.assertEqual(tag1.owner, None) + + def test_api_set_owner_w_permissions(self): + """ + GIVEN: + - API request to create an object (Tag) that supplies set_permissions object + WHEN: + - owner is passed as user id + - view > users is set & view > groups is set + THEN: + - Object permissions are set appropriately + """ + user1 = User.objects.create_superuser(username="user1") + user2 = User.objects.create(username="user2") + group1 = Group.objects.create(name="group1") + + self.client.force_authenticate(user1) + + response = self.client.post( + "/api/tags/", + json.dumps( + { + "name": "test1", + "matching_algorithm": MatchingModel.MATCH_AUTO, + "owner": user1.id, "set_permissions": { - "owner": None, "view": { - "users": None, - "groups": None, + "users": [user2.id], + "groups": [group1.id], }, "change": { "users": None, @@ -3926,19 +3987,43 @@ class TestApiAuth(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_201_CREATED) tag1 = Tag.objects.filter(name="test1").first() - self.assertEqual(tag1.owner, None) - response = self.client.post( - "/api/tags/", + from guardian.core import ObjectPermissionChecker + + checker = ObjectPermissionChecker(user2) + self.assertEqual(checker.has_perm("view_tag", tag1), True) + self.assertIn("view_tag", get_perms(group1, tag1)) + + def test_api_set_doc_permissions(self): + """ + GIVEN: + - API request to update doc permissions and owner + WHEN: + - owner is set + - view > users is set & view > groups is set + THEN: + - Object permissions are set appropriately + """ + doc = Document.objects.create( + title="test", + mime_type="application/pdf", + content="this is a document", + ) + user1 = User.objects.create_superuser(username="user1") + user2 = User.objects.create(username="user2") + group1 = Group.objects.create(name="group1") + + self.client.force_authenticate(user1) + + response = self.client.patch( + f"/api/documents/{doc.id}/", json.dumps( { - "name": "test2", - "matching_algorithm": MatchingModel.MATCH_AUTO, + "owner": user1.id, "set_permissions": { - "owner": user1.id, "view": { "users": [user2.id], - "groups": None, + "groups": [group1.id], }, "change": { "users": None, @@ -3950,12 +4035,15 @@ class TestApiAuth(DirectoriesMixin, APITestCase): content_type="application/json", ) - tag2 = Tag.objects.filter(name="test2").first() + self.assertEqual(response.status_code, status.HTTP_200_OK) + doc = Document.objects.get(pk=doc.id) + self.assertEqual(doc.owner, user1) from guardian.core import ObjectPermissionChecker checker = ObjectPermissionChecker(user2) - self.assertEqual(checker.has_perm("view_tag", tag2), True) + self.assertTrue(checker.has_perm("view_document", doc)) + self.assertIn("view_document", get_perms(group1, doc)) def test_dynamic_permissions_fields(self): user1 = User.objects.create_user(username="user1") diff --git a/src/documents/tests/test_barcodes.py b/src/documents/tests/test_barcodes.py index eda97554d..70f7807cc 100644 --- a/src/documents/tests/test_barcodes.py +++ b/src/documents/tests/test_barcodes.py @@ -906,6 +906,47 @@ class TestAsnBarcode(DirectoriesMixin, TestCase): input_doc, ) + @override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR") + def test_scan_file_for_qrcode_without_upscale(self): + """ + GIVEN: + - A printed and scanned PDF document with a rather small QR code + WHEN: + - ASN barcode detection is run with default settings + - pyzbar is used for detection, as zxing would behave differently, and detect the QR code + THEN: + - ASN is not detected + """ + + test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf" + + with BarcodeReader(test_file, "application/pdf") as reader: + reader.detect() + self.assertEqual(len(reader.barcodes), 0) + + @override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR") + @override_settings(CONSUMER_BARCODE_DPI=600) + @override_settings(CONSUMER_BARCODE_UPSCALE=1.5) + def test_scan_file_for_qrcode_with_upscale(self): + """ + GIVEN: + - A printed and scanned PDF document with a rather small QR code + WHEN: + - ASN barcode detection is run with 600dpi and an upscale factor of 1.5 and pyzbar + - pyzbar is used for detection, as zxing would behave differently. + Upscaling is a workaround for detection problems with pyzbar, + when you cannot switch to zxing (aarch64 build problems of zxing) + THEN: + - ASN 123 is detected + """ + + test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf" + + with BarcodeReader(test_file, "application/pdf") as reader: + reader.detect() + self.assertEqual(len(reader.barcodes), 1) + self.assertEqual(reader.asn, 123) + @pytest.mark.skipif( not HAS_ZXING_LIB, diff --git a/src/paperless/settings.py b/src/paperless/settings.py index ab33e6b1a..2b7a32091 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -781,6 +781,16 @@ CONSUMER_ASN_BARCODE_PREFIX: Final[str] = os.getenv( ) +CONSUMER_BARCODE_UPSCALE: Final[float] = float( + os.getenv("PAPERLESS_CONSUMER_BARCODE_UPSCALE", 0.0), +) + + +CONSUMER_BARCODE_DPI: Final[str] = int( + os.getenv("PAPERLESS_CONSUMER_BARCODE_DPI", 300), +) + + OCR_PAGES = int(os.getenv("PAPERLESS_OCR_PAGES", 0)) # The default language that tesseract will attempt to use when parsing