Feature: Allow setting backend configuration settings via the UI (#5126)

* Saving some start on this

* At least partially working for the tesseract parser

* Problems with migration testing need to figure out

* Work around that error

* Fixes max m_pixels

* Moving the settings to main paperless application

* Starting some consumer options

* More fixes and work

* Fixes these last tests

* Fix max_length on OcrSettings.mode field

* Fix all fields on Common & Ocr settings serializers

* Umbrellla config view

* Revert "Umbrellla config view"

This reverts commit fbaf9f4be30f89afeb509099180158a3406416a5.

* Updates to use a single configuration object for all settings

* Squashed commit of the following:

commit 8a0a49dd57
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 23:02:47 2023 -0800

    Fix formatting

commit 66b2d90c50
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 22:36:35 2023 -0800

    Refactor frontend data models

commit 5723bd8dd8
Author: Adam Bogdał <adam@bogdal.pl>
Date:   Wed Dec 20 01:17:43 2023 +0100

    Fix: speed up admin panel for installs with a large number of documents (#5052)

commit 9b08ce1761
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 15:18:51 2023 -0800

    Update PULL_REQUEST_TEMPLATE.md

commit a6248bec2d
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 15:02:05 2023 -0800

    Chore: Update Angular to v17 (#4980)

commit b1f6f52486
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 13:53:56 2023 -0800

    Fix: Dont allow null custom_fields property via API (#5063)

commit 638d9970fd
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 13:43:50 2023 -0800

    Enhancement: symmetric document links (#4907)

commit 5e8de4c1da
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 12:45:04 2023 -0800

    Enhancement: shared icon & shared by me filter (#4859)

commit 088bad9030
Author: Trenton H <797416+stumpylog@users.noreply.github.com>
Date:   Tue Dec 19 12:04:03 2023 -0800

    Bulk updates all the backend libraries (#5061)

* Saving some work on frontend config

* Very basic but dynamically-generated config form

* Saving work on slightly less ugly frontend config

* JSON validation for user_args field

* Fully dynamic config form

* Adds in some additional validators for a nicer error message

* Cleaning up the testing and coverage more

* Reverts unintentional change

* Adds documentation about the settings and the precedence

* Couple more commenting and style fixes

---------

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src/paperless_tesseract/parsers.py

@@ -1,9 +1,9 @@
-import json
 import os
 import re
 import subprocess
 import tempfile
 from pathlib import Path
+from typing import TYPE_CHECKING
 from typing import Optional
 
 from django.conf import settings
@@ -12,6 +12,10 @@ from PIL import Image
 from documents.parsers import DocumentParser
 from documents.parsers import ParseError
 from documents.parsers import make_thumbnail_from_pdf
+from paperless.config import OcrConfig
+from paperless.models import ArchiveFileChoices
+from paperless.models import CleanChoices
+from paperless.models import ModeChoices
 
 
 class NoTextFoundException(Exception):
@@ -30,6 +34,12 @@ class RasterisedDocumentParser(DocumentParser):
 
     logging_name = "paperless.parsing.tesseract"
 
+    def get_settings(self) -> OcrConfig:
+        """
+        This parser uses the OCR configuration settings to parse documents
+        """
+        return OcrConfig()
+
     def extract_metadata(self, document_path, mime_type):
         result = []
         if mime_type == "application/pdf":
@@ -66,7 +76,7 @@ class RasterisedDocumentParser(DocumentParser):
             self.logging_group,
         )
 
-    def is_image(self, mime_type):
+    def is_image(self, mime_type) -> bool:
         return mime_type in [
             "image/png",
             "image/jpeg",
@@ -76,7 +86,7 @@ class RasterisedDocumentParser(DocumentParser):
             "image/webp",
         ]
 
-    def has_alpha(self, image):
+    def has_alpha(self, image) -> bool:
         with Image.open(image) as im:
             return im.mode in ("RGBA", "LA")
 
@@ -91,7 +101,7 @@ class RasterisedDocumentParser(DocumentParser):
             ],
         )
 
-    def get_dpi(self, image):
+    def get_dpi(self, image) -> Optional[int]:
         try:
             with Image.open(image) as im:
                 x, y = im.info["dpi"]
@@ -100,7 +110,7 @@ class RasterisedDocumentParser(DocumentParser):
             self.log.warning(f"Error while getting DPI from image {image}: {e}")
             return None
 
-    def calculate_a4_dpi(self, image):
+    def calculate_a4_dpi(self, image) -> Optional[int]:
         try:
             with Image.open(image) as im:
                 width, height = im.size
@@ -113,13 +123,17 @@ class RasterisedDocumentParser(DocumentParser):
             self.log.warning(f"Error while calculating DPI for image {image}: {e}")
             return None
 
-    def extract_text(self, sidecar_file: Optional[Path], pdf_file: Path):
+    def extract_text(
+        self,
+        sidecar_file: Optional[Path],
+        pdf_file: Path,
+    ) -> Optional[str]:
         # When re-doing OCR, the sidecar contains ONLY the new text, not
         # the whole text, so do not utilize it in that case
         if (
             sidecar_file is not None
             and os.path.isfile(sidecar_file)
-            and settings.OCR_MODE != "redo"
+            and self.settings.mode != "redo"
         ):
             text = self.read_file_handle_unicode_errors(sidecar_file)
 
@@ -174,6 +188,8 @@ class RasterisedDocumentParser(DocumentParser):
         sidecar_file,
         safe_fallback=False,
     ):
+        if TYPE_CHECKING:
+            assert isinstance(self.settings, OcrConfig)
         ocrmypdf_args = {
             "input_file": input_file,
             "output_file": output_file,
@@ -181,46 +197,47 @@ class RasterisedDocumentParser(DocumentParser):
             # processes via the task library.
             "use_threads": True,
             "jobs": settings.THREADS_PER_WORKER,
-            "language": settings.OCR_LANGUAGE,
-            "output_type": settings.OCR_OUTPUT_TYPE,
+            "language": self.settings.language,
+            "output_type": self.settings.output_type,
             "progress_bar": False,
         }
 
         if "pdfa" in ocrmypdf_args["output_type"]:
             ocrmypdf_args[
                 "color_conversion_strategy"
-            ] = settings.OCR_COLOR_CONVERSION_STRATEGY
+            ] = self.settings.color_conversion_strategy
 
-        if settings.OCR_MODE == "force" or safe_fallback:
+        if self.settings.mode == ModeChoices.FORCE or safe_fallback:
             ocrmypdf_args["force_ocr"] = True
-        elif settings.OCR_MODE in ["skip", "skip_noarchive"]:
+        elif self.settings.mode in {
+            ModeChoices.SKIP,
+            ModeChoices.SKIP_NO_ARCHIVE,
+        }:
             ocrmypdf_args["skip_text"] = True
-        elif settings.OCR_MODE == "redo":
+        elif self.settings.mode == ModeChoices.REDO:
             ocrmypdf_args["redo_ocr"] = True
-        else:
-            raise ParseError(f"Invalid ocr mode: {settings.OCR_MODE}")
+        else:  # pragma: no cover
+            raise ParseError(f"Invalid ocr mode: {self.settings.mode}")
 
-        if settings.OCR_CLEAN == "clean":
+        if self.settings.clean == CleanChoices.CLEAN:
             ocrmypdf_args["clean"] = True
-        elif settings.OCR_CLEAN == "clean-final":
-            if settings.OCR_MODE == "redo":
+        elif self.settings.clean == CleanChoices.FINAL:
+            if self.settings.mode == ModeChoices.REDO:
                 ocrmypdf_args["clean"] = True
             else:
                 # --clean-final is not compatible with --redo-ocr
                 ocrmypdf_args["clean_final"] = True
 
-        if settings.OCR_DESKEW and settings.OCR_MODE != "redo":
+        if self.settings.deskew and self.settings.mode != ModeChoices.REDO:
             # --deskew is not compatible with --redo-ocr
             ocrmypdf_args["deskew"] = True
 
-        if settings.OCR_ROTATE_PAGES:
+        if self.settings.rotate:
             ocrmypdf_args["rotate_pages"] = True
-            ocrmypdf_args[
-                "rotate_pages_threshold"
-            ] = settings.OCR_ROTATE_PAGES_THRESHOLD
+            ocrmypdf_args["rotate_pages_threshold"] = self.settings.rotate_threshold
 
-        if settings.OCR_PAGES > 0:
-            ocrmypdf_args["pages"] = f"1-{settings.OCR_PAGES}"
+        if self.settings.pages is not None:
+            ocrmypdf_args["pages"] = f"1-{self.settings.pages}"
         else:
             # sidecar is incompatible with pages
             ocrmypdf_args["sidecar"] = sidecar_file
@@ -239,8 +256,8 @@ class RasterisedDocumentParser(DocumentParser):
             if dpi:
                 self.log.debug(f"Detected DPI for image {input_file}: {dpi}")
                 ocrmypdf_args["image_dpi"] = dpi
-            elif settings.OCR_IMAGE_DPI:
-                ocrmypdf_args["image_dpi"] = settings.OCR_IMAGE_DPI
+            elif self.settings.image_dpi is not None:
+                ocrmypdf_args["image_dpi"] = self.settings.image_dpi
             elif a4_dpi:
                 ocrmypdf_args["image_dpi"] = a4_dpi
             else:
@@ -254,19 +271,18 @@ class RasterisedDocumentParser(DocumentParser):
                     f"Image DPI of {ocrmypdf_args['image_dpi']} is low, OCR may fail",
                 )
 
-        if settings.OCR_USER_ARGS:
+        if self.settings.user_args is not None:
             try:
-                user_args = json.loads(settings.OCR_USER_ARGS)
-                ocrmypdf_args = {**ocrmypdf_args, **user_args}
+                ocrmypdf_args = {**ocrmypdf_args, **self.settings.user_args}
             except Exception as e:
                 self.log.warning(
                     f"There is an issue with PAPERLESS_OCR_USER_ARGS, so "
                     f"they will not be used. Error: {e}",
                 )
 
-        if settings.OCR_MAX_IMAGE_PIXELS is not None:
+        if self.settings.max_image_pixel is not None:
             # Convert pixels to mega-pixels and provide to ocrmypdf
-            max_pixels_mpixels = settings.OCR_MAX_IMAGE_PIXELS / 1_000_000.0
+            max_pixels_mpixels = self.settings.max_image_pixel / 1_000_000.0
             if max_pixels_mpixels > 0:
                 self.log.debug(
                     f"Calculated {max_pixels_mpixels} megapixels for OCR",
@@ -298,8 +314,12 @@ class RasterisedDocumentParser(DocumentParser):
         # If the original has text, and the user doesn't want an archive,
         # we're done here
         skip_archive_for_text = (
-            settings.OCR_MODE == "skip_noarchive"
-            or settings.OCR_SKIP_ARCHIVE_FILE in ["with_text", "always"]
+            self.settings.mode == ModeChoices.SKIP_NO_ARCHIVE
+            or self.settings.skip_archive_file
+            in {
+                ArchiveFileChoices.WITH_TEXT,
+                ArchiveFileChoices.ALWAYS,
+            }
         )
         if skip_archive_for_text and original_has_text:
             self.log.debug("Document has text, skipping OCRmyPDF entirely.")
@@ -329,7 +349,7 @@ class RasterisedDocumentParser(DocumentParser):
             self.log.debug(f"Calling OCRmyPDF with args: {args}")
             ocrmypdf.ocr(**args)
 
-            if settings.OCR_SKIP_ARCHIVE_FILE != "always":
+            if self.settings.skip_archive_file != ArchiveFileChoices.ALWAYS:
                 self.archive_path = archive_path
 
             self.text = self.extract_text(sidecar_file, archive_path)

src/paperless_tesseract/tests/test_parser.py

@@ -2,7 +2,6 @@ import os
 import shutil
 import tempfile
 import uuid
-from contextlib import AbstractContextManager
 from pathlib import Path
 from unittest import mock
 
@@ -17,28 +16,6 @@ from documents.tests.utils import FileSystemAssertsMixin
 from paperless_tesseract.parsers import RasterisedDocumentParser
 from paperless_tesseract.parsers import post_process_text
 
-image_to_string_calls = []
-
-
-def fake_convert(input_file, output_file, **kwargs):
-    with open(input_file) as f:
-        lines = f.readlines()
-
-    for i, line in enumerate(lines):
-        with open(output_file % i, "w") as f2:
-            f2.write(line.strip())
-
-
-class FakeImageFile(AbstractContextManager):
-    def __init__(self, fname):
-        self.fname = fname
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-    def __enter__(self):
-        return os.path.basename(self.fname)
-
 
 class TestParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
     SAMPLE_FILES = Path(__file__).resolve().parent / "samples"
@@ -769,43 +746,52 @@ class TestParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
         self.assertEqual(params["sidecar"], "sidecar.txt")
 
         with override_settings(OCR_CLEAN="none"):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertNotIn("clean", params)
             self.assertNotIn("clean_final", params)
 
         with override_settings(OCR_CLEAN="clean"):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertTrue(params["clean"])
             self.assertNotIn("clean_final", params)
 
         with override_settings(OCR_CLEAN="clean-final", OCR_MODE="skip"):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertTrue(params["clean_final"])
             self.assertNotIn("clean", params)
 
         with override_settings(OCR_CLEAN="clean-final", OCR_MODE="redo"):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertTrue(params["clean"])
             self.assertNotIn("clean_final", params)
 
         with override_settings(OCR_DESKEW=True, OCR_MODE="skip"):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertTrue(params["deskew"])
 
         with override_settings(OCR_DESKEW=True, OCR_MODE="redo"):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertNotIn("deskew", params)
 
         with override_settings(OCR_DESKEW=False, OCR_MODE="skip"):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertNotIn("deskew", params)
 
         with override_settings(OCR_MAX_IMAGE_PIXELS=1_000_001.0):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertIn("max_image_mpixels", params)
             self.assertAlmostEqual(params["max_image_mpixels"], 1, places=4)
 
         with override_settings(OCR_MAX_IMAGE_PIXELS=-1_000_001.0):
+            parser = RasterisedDocumentParser(None)
             params = parser.construct_ocrmypdf_parameters("", "", "", "")
             self.assertNotIn("max_image_mpixels", params)
 

src/paperless_tesseract/tests/test_parser_custom_settings.py

@@ -0,0 +1,232 @@
+import json
+
+from django.test import TestCase
+from django.test import override_settings
+
+from documents.tests.utils import DirectoriesMixin
+from documents.tests.utils import FileSystemAssertsMixin
+from paperless.models import ApplicationConfiguration
+from paperless.models import CleanChoices
+from paperless.models import ColorConvertChoices
+from paperless.models import ModeChoices
+from paperless.models import OutputTypeChoices
+from paperless_tesseract.parsers import RasterisedDocumentParser
+
+
+class TestParserSettingsFromDb(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
+    @staticmethod
+    def get_params():
+        """
+        Helper to get just the OCRMyPDF parameters from the parser
+        """
+        return RasterisedDocumentParser(None).construct_ocrmypdf_parameters(
+            input_file="input.pdf",
+            output_file="output.pdf",
+            sidecar_file="sidecar.txt",
+            mime_type="application/pdf",
+            safe_fallback=False,
+        )
+
+    def test_db_settings_ocr_pages(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_PAGES than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_PAGES=10):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.pages = 5
+            instance.save()
+
+            params = self.get_params()
+        self.assertEqual(params["pages"], "1-5")
+
+    def test_db_settings_ocr_language(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_LANGUAGE than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_LANGUAGE="eng+deu"):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.language = "fra+ita"
+            instance.save()
+
+            params = self.get_params()
+        self.assertEqual(params["language"], "fra+ita")
+
+    def test_db_settings_ocr_output_type(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_OUTPUT_TYPE than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_OUTPUT_TYPE="pdfa-3"):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.output_type = OutputTypeChoices.PDF_A
+            instance.save()
+
+            params = self.get_params()
+        self.assertEqual(params["output_type"], "pdfa")
+
+    def test_db_settings_ocr_mode(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_MODE than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_MODE="redo"):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.mode = ModeChoices.SKIP
+            instance.save()
+
+            params = self.get_params()
+        self.assertTrue(params["skip_text"])
+        self.assertNotIn("redo_ocr", params)
+        self.assertNotIn("force_ocr", params)
+
+    def test_db_settings_ocr_clean(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_CLEAN than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_CLEAN="clean-final"):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.unpaper_clean = CleanChoices.CLEAN
+            instance.save()
+
+            params = self.get_params()
+        self.assertTrue(params["clean"])
+        self.assertNotIn("clean_final", params)
+
+        with override_settings(OCR_CLEAN="clean-final"):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.unpaper_clean = CleanChoices.FINAL
+            instance.save()
+
+            params = self.get_params()
+        self.assertTrue(params["clean_final"])
+        self.assertNotIn("clean", params)
+
+    def test_db_settings_ocr_deskew(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_DESKEW than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_DESKEW=False):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.deskew = True
+            instance.save()
+
+            params = self.get_params()
+        self.assertTrue(params["deskew"])
+
+    def test_db_settings_ocr_rotate(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_ROTATE_PAGES
+              and OCR_ROTATE_PAGES_THRESHOLD than configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_ROTATE_PAGES=False, OCR_ROTATE_PAGES_THRESHOLD=30.0):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.rotate_pages = True
+            instance.rotate_pages_threshold = 15.0
+            instance.save()
+
+            params = self.get_params()
+        self.assertTrue(params["rotate_pages"])
+        self.assertAlmostEqual(params["rotate_pages_threshold"], 15.0)
+
+    def test_db_settings_ocr_max_pixels(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_MAX_IMAGE_PIXELS than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_MAX_IMAGE_PIXELS=2_000_000.0):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.max_image_pixels = 1_000_000.0
+            instance.save()
+
+            params = self.get_params()
+        self.assertAlmostEqual(params["max_image_mpixels"], 1.0)
+
+    def test_db_settings_ocr_color_convert(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_COLOR_CONVERSION_STRATEGY than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(OCR_COLOR_CONVERSION_STRATEGY="LeaveColorUnchanged"):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.color_conversion_strategy = ColorConvertChoices.INDEPENDENT
+            instance.save()
+
+            params = self.get_params()
+        self.assertEqual(
+            params["color_conversion_strategy"],
+            "UseDeviceIndependentColor",
+        )
+
+    def test_ocr_user_args(self):
+        """
+        GIVEN:
+            - Django settings defines different value for OCR_USER_ARGS than
+              configuration object
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Configuration from database is utilized
+        """
+        with override_settings(
+            OCR_USER_ARGS=json.dumps({"continue_on_soft_render_error": True}),
+        ):
+            instance = ApplicationConfiguration.objects.all().first()
+            instance.user_args = {"unpaper_args": "--pre-rotate 90"}
+            instance.save()
+
+            params = self.get_params()
+
+        self.assertIn("unpaper_args", params)
+        self.assertEqual(
+            params["unpaper_args"],
+            "--pre-rotate 90",
+        )