paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-25 10:49:30 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev'

Browse Source
This commit is contained in:
shamoon 2023-06-27 14:51:08 -07:00
commit 252abb41c3
9 changed files with 195 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
docs/configuration.md
View File

@ -1095,6 +1095,27 @@ barcode.
Defaults to "ASN"
`PAPERLESS_CONSUMER_BARCODE_UPSCALE=<float>`
: Defines the upscale factor used in barcode detection.
Improves the detection of small barcodes, i.e. with a value of 1.5 by
upscaling the document beforce the detection process. Upscaling will
only take place if value is bigger than 1.0. Otherwise upscaling will
not be performed to save resources. Try using in combination with
PAPERLESS_CONSUMER_BARCODE_DPI set to a value higher than default.
Defaults to 0.0
`PAPERLESS_CONSUMER_BARCODE_DPI=<int>`
: During barcode detection every page from a PDF document needs
to be converted to an image. A dpi value can be specified in the
conversion process. Default is 300. If the detection of small barcodes
fails a bigger dpi value i.e. 600 can fix the issue. Try using in
combination with PAPERLESS_CONSUMER_BARCODE_UPSCALE bigger than 1.0.
Defaults to "300"
## Binaries
There are a few external software packages that Paperless expects to

2
paperless.conf.example
View File

@ -66,6 +66,8 @@
#PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
#PAPERLESS_CONSUMER_ENABLE_BARCODES=false
#PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
#PAPERLESS_CONSUMER_BARCODE_UPSCALE=0.0
#PAPERLESS_CONSUMER_BARCODE_DPI=300
#PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
#PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
#PAPERLESS_FILENAME_DATE_ORDER=YMD

2
src-ui/src/environments/environment.prod.ts
View File

@ -5,7 +5,7 @@ export const environment = {
apiBaseUrl: document.baseURI + 'api/',
apiVersion: '3',
appTitle: 'Paperless-ngx',
version: '1.16.4',
version: '1.16.4-dev',
webSocketHost: window.location.host,
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
webSocketBaseUrl: base_url.pathname + 'ws/',

12
src/documents/barcodes.py
View File

@ -203,11 +203,21 @@ class BarcodeReader:
try:
pages_from_path = convert_from_path(
self.pdf_file,
dpi=300,
dpi=settings.CONSUMER_BARCODE_DPI,
output_folder=self.temp_dir.name,
)
for current_page_number, page in enumerate(pages_from_path):
factor = settings.CONSUMER_BARCODE_UPSCALE
if factor > 1.0:
logger.debug(
f"Upscaling image by {factor} for better barcode detection",
)
x, y = page.size
page = page.resize(
(int(round(x * factor)), (int(round(y * factor)))),
)
for barcode_value in reader(page):
self.barcodes.append(
Barcode(current_page_number, barcode_value),

7
src/documents/serialisers.py
View File

@ -213,15 +213,12 @@ class OwnedObjectSerializer(serializers.ModelSerializer, SetPermissionsMixin):
# other methods in mixin
def create(self, validated_data):
if self.user and (
"owner" not in validated_data or validated_data["owner"] is None
):
# default to current user if not set
if "owner" not in validated_data and self.user:
validated_data["owner"] = self.user
permissions = None
if "set_permissions" in validated_data:
permissions = validated_data.pop("set_permissions")
if "user" not in permissions or permissions["user"] is None:
validated_data["owner"] = None
instance = super().create(validated_data)
if permissions is not None:
self._set_permissions(permissions, instance)

BIN
src/documents/tests/samples/barcodes/barcode-qr-asn-000123-upscale-dpi.pdf Normal file
View File

Binary file not shown.

126
src/documents/tests/test_api.py
View File

@ -28,6 +28,7 @@ from django.contrib.auth.models import User
from django.test import override_settings
from django.utils import timezone
from guardian.shortcuts import assign_perm
from guardian.shortcuts import get_perms
from rest_framework import status
from rest_framework.test import APITestCase
from whoosh.writing import AsyncWriter
@ -3855,7 +3856,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
status.HTTP_200_OK,
)
def test_object_permissions(self):
def test_api_get_object_permissions(self):
user1 = User.objects.create_user(username="test1")
user2 = User.objects.create_user(username="test2")
user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
@ -3886,18 +3887,16 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
status.HTTP_404_NOT_FOUND,
)
def test_api_set_permissions(self):
def test_api_default_owner(self):
"""
GIVEN:
- API request to create an object (Tag) that supplies set_permissions object
- API request to create an object (Tag)
WHEN:
- owner is passed as null or as a user id
- view > users is set
- owner is not set at all
THEN:
- Object permissions are set appropriately
- Object created with current user as owner
"""
user1 = User.objects.create_superuser(username="user1")
user2 = User.objects.create(username="user2")
self.client.force_authenticate(user1)
@ -3907,11 +3906,73 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
{
"name": "test1",
"matching_algorithm": MatchingModel.MATCH_AUTO,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
tag1 = Tag.objects.filter(name="test1").first()
self.assertEqual(tag1.owner, user1)
def test_api_set_no_owner(self):
"""
GIVEN:
- API request to create an object (Tag)
WHEN:
- owner is passed as None
THEN:
- Object created with no owner
"""
user1 = User.objects.create_superuser(username="user1")
self.client.force_authenticate(user1)
response = self.client.post(
"/api/tags/",
json.dumps(
{
"name": "test1",
"matching_algorithm": MatchingModel.MATCH_AUTO,
"owner": None,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
tag1 = Tag.objects.filter(name="test1").first()
self.assertEqual(tag1.owner, None)
def test_api_set_owner_w_permissions(self):
"""
GIVEN:
- API request to create an object (Tag) that supplies set_permissions object
WHEN:
- owner is passed as user id
- view > users is set & view > groups is set
THEN:
- Object permissions are set appropriately
"""
user1 = User.objects.create_superuser(username="user1")
user2 = User.objects.create(username="user2")
group1 = Group.objects.create(name="group1")
self.client.force_authenticate(user1)
response = self.client.post(
"/api/tags/",
json.dumps(
{
"name": "test1",
"matching_algorithm": MatchingModel.MATCH_AUTO,
"owner": user1.id,
"set_permissions": {
"owner": None,
"view": {
"users": None,
"groups": None,
"users": [user2.id],
"groups": [group1.id],
},
"change": {
"users": None,
@ -3926,19 +3987,43 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
tag1 = Tag.objects.filter(name="test1").first()
self.assertEqual(tag1.owner, None)
response = self.client.post(
"/api/tags/",
from guardian.core import ObjectPermissionChecker
checker = ObjectPermissionChecker(user2)
self.assertEqual(checker.has_perm("view_tag", tag1), True)
self.assertIn("view_tag", get_perms(group1, tag1))
def test_api_set_doc_permissions(self):
"""
GIVEN:
- API request to update doc permissions and owner
WHEN:
- owner is set
- view > users is set & view > groups is set
THEN:
- Object permissions are set appropriately
"""
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="this is a document",
)
user1 = User.objects.create_superuser(username="user1")
user2 = User.objects.create(username="user2")
group1 = Group.objects.create(name="group1")
self.client.force_authenticate(user1)
response = self.client.patch(
f"/api/documents/{doc.id}/",
json.dumps(
{
"name": "test2",
"matching_algorithm": MatchingModel.MATCH_AUTO,
"owner": user1.id,
"set_permissions": {
"owner": user1.id,
"view": {
"users": [user2.id],
"groups": None,
"groups": [group1.id],
},
"change": {
"users": None,
@ -3950,12 +4035,15 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
content_type="application/json",
)
tag2 = Tag.objects.filter(name="test2").first()
self.assertEqual(response.status_code, status.HTTP_200_OK)
doc = Document.objects.get(pk=doc.id)
self.assertEqual(doc.owner, user1)
from guardian.core import ObjectPermissionChecker
checker = ObjectPermissionChecker(user2)
self.assertEqual(checker.has_perm("view_tag", tag2), True)
self.assertTrue(checker.has_perm("view_document", doc))
self.assertIn("view_document", get_perms(group1, doc))
def test_dynamic_permissions_fields(self):
user1 = User.objects.create_user(username="user1")

41
src/documents/tests/test_barcodes.py
View File

@ -906,6 +906,47 @@ class TestAsnBarcode(DirectoriesMixin, TestCase):
input_doc,
)
@override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR")
def test_scan_file_for_qrcode_without_upscale(self):
"""
GIVEN:
- A printed and scanned PDF document with a rather small QR code
WHEN:
- ASN barcode detection is run with default settings
- pyzbar is used for detection, as zxing would behave differently, and detect the QR code
THEN:
- ASN is not detected
"""
test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf"
with BarcodeReader(test_file, "application/pdf") as reader:
reader.detect()
self.assertEqual(len(reader.barcodes), 0)
@override_settings(CONSUMER_BARCODE_SCANNER="PYZBAR")
@override_settings(CONSUMER_BARCODE_DPI=600)
@override_settings(CONSUMER_BARCODE_UPSCALE=1.5)
def test_scan_file_for_qrcode_with_upscale(self):
"""
GIVEN:
- A printed and scanned PDF document with a rather small QR code
WHEN:
- ASN barcode detection is run with 600dpi and an upscale factor of 1.5 and pyzbar
- pyzbar is used for detection, as zxing would behave differently.
Upscaling is a workaround for detection problems with pyzbar,
when you cannot switch to zxing (aarch64 build problems of zxing)
THEN:
- ASN 123 is detected
"""
test_file = self.BARCODE_SAMPLE_DIR / "barcode-qr-asn-000123-upscale-dpi.pdf"
with BarcodeReader(test_file, "application/pdf") as reader:
reader.detect()
self.assertEqual(len(reader.barcodes), 1)
self.assertEqual(reader.asn, 123)
@pytest.mark.skipif(
not HAS_ZXING_LIB,

10
src/paperless/settings.py
View File

@ -781,6 +781,16 @@ CONSUMER_ASN_BARCODE_PREFIX: Final[str] = os.getenv(
)
CONSUMER_BARCODE_UPSCALE: Final[float] = float(
os.getenv("PAPERLESS_CONSUMER_BARCODE_UPSCALE", 0.0),
)
CONSUMER_BARCODE_DPI: Final[str] = int(
os.getenv("PAPERLESS_CONSUMER_BARCODE_DPI", 300),
)
OCR_PAGES = int(os.getenv("PAPERLESS_OCR_PAGES", 0))
# The default language that tesseract will attempt to use when parsing