paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-30 18:27:45 -05:00
Code Issues Packages Projects Releases Wiki Activity

Runs the pre-commit hooks over all the Python files

Browse Source
This commit is contained in:
Trenton Holmes
2022-03-11 10:55:51 -08:00
parent d3e9799279
commit 1771d18a21
94 changed files with 1638 additions and 991 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/paperless_tesseract/__init__.py
View File

@@ -1,2 +1,5 @@
# this is here so that django finds the checks.
from .checks import *
from .checks import check_default_language_available
from .checks import get_tesseract_langs
__all__ = ["get_tesseract_langs", "check_default_language_available"]

1
src/paperless_tesseract/apps.py
View File

@@ -1,5 +1,4 @@
from django.apps import AppConfig
from paperless_tesseract.signals import tesseract_consumer_declaration

12
src/paperless_tesseract/checks.py
View File

@@ -1,7 +1,9 @@
import subprocess
from django.conf import settings
from django.core.checks import Error, Warning, register
from django.core.checks import Error
from django.core.checks import register
from django.core.checks import Warning
def get_tesseract_langs():
@@ -19,8 +21,8 @@ def check_default_language_available(app_configs, **kwargs):
return [
Warning(
"No OCR language has been specified with PAPERLESS_OCR_LANGUAGE. "
"This means that tesseract will fallback to english."
)
"This means that tesseract will fallback to english.",
),
]
specified_langs = settings.OCR_LANGUAGE.split("+")
@@ -31,8 +33,8 @@ def check_default_language_available(app_configs, **kwargs):
Error(
f"The selected ocr language {lang} is "
f"not installed. Paperless cannot OCR your documents "
f"without it. Please fix PAPERLESS_OCR_LANGUAGE."
)
f"without it. Please fix PAPERLESS_OCR_LANGUAGE.",
),
]
return []

33
src/paperless_tesseract/parsers.py
View File

@@ -2,10 +2,11 @@ import json
import os
import re
from PIL import Image
from django.conf import settings
from documents.parsers import DocumentParser, ParseError, make_thumbnail_from_pdf
from documents.parsers import DocumentParser
from documents.parsers import make_thumbnail_from_pdf
from documents.parsers import ParseError
from PIL import Image
class NoTextFoundException(Exception):
@@ -42,7 +43,7 @@ class RasterisedDocumentParser(DocumentParser):
"prefix": meta.REVERSE_NS[m.group(1)],
"key": m.group(2),
"value": value,
}
},
)
except Exception as e:
self.log(
@@ -53,7 +54,9 @@ class RasterisedDocumentParser(DocumentParser):
def get_thumbnail(self, document_path, mime_type, file_name=None):
return make_thumbnail_from_pdf(
self.archive_path or document_path, self.tempdir, self.logging_group
self.archive_path or document_path,
self.tempdir,
self.logging_group,
)
def is_image(self, mime_type):
@@ -110,7 +113,6 @@ class RasterisedDocumentParser(DocumentParser):
return None
from pdfminer.high_level import extract_text as pdfminer_extract_text
from pdfminer.pdftypes import PDFException
try:
stripped = post_process_text(pdfminer_extract_text(pdf_file))
@@ -129,7 +131,12 @@ class RasterisedDocumentParser(DocumentParser):
return None
def construct_ocrmypdf_parameters(
self, input_file, mime_type, output_file, sidecar_file, safe_fallback=False
self,
input_file,
mime_type,
output_file,
sidecar_file,
safe_fallback=False,
):
ocrmypdf_args = {
"input_file": input_file,
@@ -167,7 +174,7 @@ class RasterisedDocumentParser(DocumentParser):
ocrmypdf_args["rotate_pages"] = True
ocrmypdf_args[
"rotate_pages_threshold"
] = settings.OCR_ROTATE_PAGES_THRESHOLD # NOQA: E501
] = settings.OCR_ROTATE_PAGES_THRESHOLD
if settings.OCR_PAGES > 0:
ocrmypdf_args["pages"] = f"1-{settings.OCR_PAGES}"
@@ -202,7 +209,7 @@ class RasterisedDocumentParser(DocumentParser):
raise ParseError(
f"Cannot produce archive PDF for image {input_file}, "
f"no DPI information is present in this image and "
f"OCR_IMAGE_DPI is not set."
f"OCR_IMAGE_DPI is not set.",
)
if settings.OCR_USER_ARGS and not safe_fallback:
@@ -241,7 +248,10 @@ class RasterisedDocumentParser(DocumentParser):
sidecar_file = os.path.join(self.tempdir, "sidecar.txt")
args = self.construct_ocrmypdf_parameters(
document_path, mime_type, archive_path, sidecar_file
document_path,
mime_type,
archive_path,
sidecar_file,
)
try:
@@ -289,7 +299,8 @@ class RasterisedDocumentParser(DocumentParser):
# is bigger and blurry due to --force-ocr.
self.text = self.extract_text(
sidecar_file_fallback, archive_path_fallback
sidecar_file_fallback,
archive_path_fallback,
)
except Exception as e:

8
src/paperless_tesseract/tests/test_checks.py
View File

@@ -1,8 +1,8 @@
from unittest import mock
from django.core.checks import ERROR
from django.test import TestCase, override_settings
from django.test import override_settings
from django.test import TestCase
from paperless_tesseract import check_default_language_available
@@ -16,8 +16,8 @@ class TestChecks(TestCase):
self.assertEqual(len(msgs), 1)
self.assertTrue(
msgs[0].msg.startswith(
"No OCR language has been specified with PAPERLESS_OCR_LANGUAGE"
)
"No OCR language has been specified with PAPERLESS_OCR_LANGUAGE",
),
)
@override_settings(OCR_LANGUAGE="ita")

109
src/paperless_tesseract/tests/test_parser.py
View File

@@ -3,11 +3,13 @@ import uuid
from typing import ContextManager
from unittest import mock
from django.test import TestCase, override_settings
from documents.parsers import ParseError, run_convert
from django.test import override_settings
from django.test import TestCase
from documents.parsers import ParseError
from documents.parsers import run_convert
from documents.tests.utils import DirectoriesMixin
from paperless_tesseract.parsers import RasterisedDocumentParser, post_process_text
from paperless_tesseract.parsers import post_process_text
from paperless_tesseract.parsers import RasterisedDocumentParser
image_to_string_calls = []
@@ -56,7 +58,9 @@ class TestParser(DirectoriesMixin, TestCase):
result,
actual_result,
"strip_exceess_whitespace({}) != '{}', but '{}'".format(
source, result, actual_result
source,
result,
actual_result,
),
)
@@ -65,7 +69,8 @@ class TestParser(DirectoriesMixin, TestCase):
def test_get_text_from_pdf(self):
parser = RasterisedDocumentParser(uuid.uuid4())
text = parser.extract_text(
None, os.path.join(self.SAMPLE_FILES, "simple-digital.pdf")
None,
os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"),
)
self.assertContainsStrings(text.strip(), ["This is a test document."])
@@ -73,7 +78,8 @@ class TestParser(DirectoriesMixin, TestCase):
def test_thumbnail(self):
parser = RasterisedDocumentParser(uuid.uuid4())
thumb = parser.get_thumbnail(
os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(thumb))
@@ -89,14 +95,16 @@ class TestParser(DirectoriesMixin, TestCase):
parser = RasterisedDocumentParser(uuid.uuid4())
thumb = parser.get_thumbnail(
os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(thumb))
def test_thumbnail_encrypted(self):
parser = RasterisedDocumentParser(uuid.uuid4())
thumb = parser.get_thumbnail(
os.path.join(self.SAMPLE_FILES, "encrypted.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "encrypted.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(thumb))
@@ -113,7 +121,8 @@ class TestParser(DirectoriesMixin, TestCase):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "simple-digital.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
@@ -124,7 +133,8 @@ class TestParser(DirectoriesMixin, TestCase):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "with-form.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
@@ -139,7 +149,8 @@ class TestParser(DirectoriesMixin, TestCase):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "with-form.pdf"),
"application/pdf",
)
self.assertIsNone(parser.archive_path)
@@ -168,7 +179,8 @@ class TestParser(DirectoriesMixin, TestCase):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "encrypted.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "encrypted.pdf"),
"application/pdf",
)
self.assertIsNone(parser.archive_path)
@@ -178,7 +190,8 @@ class TestParser(DirectoriesMixin, TestCase):
def test_with_form_error_notext(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "with-form.pdf"),
"application/pdf",
)
self.assertContainsStrings(
@@ -191,7 +204,8 @@ class TestParser(DirectoriesMixin, TestCase):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "with-form.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "with-form.pdf"),
"application/pdf",
)
self.assertContainsStrings(
@@ -221,7 +235,7 @@ class TestParser(DirectoriesMixin, TestCase):
parser = RasterisedDocumentParser(None)
dpi = parser.calculate_a4_dpi(
os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png")
os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"),
)
self.assertEqual(dpi, 62)
@@ -233,7 +247,8 @@ class TestParser(DirectoriesMixin, TestCase):
def f():
parser.parse(
os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"), "image/png"
os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"),
"image/png",
)
self.assertRaises(ParseError, f)
@@ -247,68 +262,80 @@ class TestParser(DirectoriesMixin, TestCase):
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
parser.get_text().lower(), ["this is a test document."]
parser.get_text().lower(),
["this is a test document."],
)
def test_multi_page(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
parser.get_text().lower(), ["page 1", "page 2", "page 3"]
parser.get_text().lower(),
["page 1", "page 2", "page 3"],
)
@override_settings(OCR_PAGES=2, OCR_MODE="skip")
def test_multi_page_pages_skip(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
parser.get_text().lower(), ["page 1", "page 2", "page 3"]
parser.get_text().lower(),
["page 1", "page 2", "page 3"],
)
@override_settings(OCR_PAGES=2, OCR_MODE="redo")
def test_multi_page_pages_redo(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
parser.get_text().lower(), ["page 1", "page 2", "page 3"]
parser.get_text().lower(),
["page 1", "page 2", "page 3"],
)
@override_settings(OCR_PAGES=2, OCR_MODE="force")
def test_multi_page_pages_force(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
parser.get_text().lower(), ["page 1", "page 2", "page 3"]
parser.get_text().lower(),
["page 1", "page 2", "page 3"],
)
@override_settings(OOCR_MODE="skip")
def test_multi_page_analog_pages_skip(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
parser.get_text().lower(), ["page 1", "page 2", "page 3"]
parser.get_text().lower(),
["page 1", "page 2", "page 3"],
)
@override_settings(OCR_PAGES=2, OCR_MODE="redo")
def test_multi_page_analog_pages_redo(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2"])
@@ -318,7 +345,8 @@ class TestParser(DirectoriesMixin, TestCase):
def test_multi_page_analog_pages_force(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(parser.get_text().lower(), ["page 1"])
@@ -329,29 +357,34 @@ class TestParser(DirectoriesMixin, TestCase):
def test_skip_noarchive_withtext(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-digital.pdf"),
"application/pdf",
)
self.assertIsNone(parser.archive_path)
self.assertContainsStrings(
parser.get_text().lower(), ["page 1", "page 2", "page 3"]
parser.get_text().lower(),
["page 1", "page 2", "page 3"],
)
@override_settings(OCR_MODE="skip_noarchive")
def test_skip_noarchive_notext(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-images.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
parser.get_text().lower(), ["page 1", "page 2", "page 3"]
parser.get_text().lower(),
["page 1", "page 2", "page 3"],
)
@override_settings(OCR_MODE="skip")
def test_multi_page_mixed(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-mixed.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-mixed.pdf"),
"application/pdf",
)
self.assertTrue(os.path.isfile(parser.archive_path))
self.assertContainsStrings(
@@ -368,11 +401,13 @@ class TestParser(DirectoriesMixin, TestCase):
def test_multi_page_mixed_no_archive(self):
parser = RasterisedDocumentParser(None)
parser.parse(
os.path.join(self.SAMPLE_FILES, "multi-page-mixed.pdf"), "application/pdf"
os.path.join(self.SAMPLE_FILES, "multi-page-mixed.pdf"),
"application/pdf",
)
self.assertIsNone(parser.archive_path)
self.assertContainsStrings(
parser.get_text().lower(), ["page 4", "page 5", "page 6"]
parser.get_text().lower(),
["page 4", "page 5", "page 6"],
)
@override_settings(OCR_MODE="skip", OCR_ROTATE_PAGES=True)