Bumps version to 1.13.0

Feature: allow disable warn on close saved view with changes

.codecov.yml

@@ -0,0 +1,19 @@
+# https://docs.codecov.com/docs/pull-request-comments
+# codecov will only comment if coverage changes
+comment:
+  require_changes: true
+coverage:
+  status:
+    project:
+      default:
+        # https://docs.codecov.com/docs/commit-status#threshold
+        threshold: 1%
+        # https://docs.codecov.com/docs/commit-status#only_pulls
+        only_pulls: true
+    patch:
+      default:
+        # For the changed lines only, target 75% covered, but
+        # allow as low as 50%
+        target: 75%
+        threshold: 25%
+        only_pulls: true

.github/scripts/cleanup-tags.py

@@ -7,6 +7,7 @@ import subprocess
 from argparse import ArgumentParser
 from typing import Dict
 from typing import Final
+from typing import Iterator
 from typing import List
 from typing import Optional
 
@@ -18,11 +19,14 @@ from github import GithubContainerRegistryApi
 logger = logging.getLogger("cleanup-tags")
 
 
-class DockerManifest2:
+class ImageProperties:
     """
-    Data class wrapping the Docker Image Manifest Version 2.
+    Data class wrapping the properties of an entry in the image index
+    manifests list.  It is NOT an actual image with layers, etc
 
-    See https://docs.docker.com/registry/spec/manifest-v2-2/
+    https://docs.docker.com/registry/spec/manifest-v2-2/
+    https://github.com/opencontainers/image-spec/blob/main/manifest.md
+    https://github.com/opencontainers/image-spec/blob/main/descriptor.md
     """
 
     def __init__(self, data: Dict) -> None:
@@ -39,6 +43,45 @@ class DockerManifest2:
         self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
 
 
+class ImageIndex:
+    """
+    Data class wrapping up logic for an OCI Image Index
+    JSON data.  Primary use is to access the manifests listing
+
+    See https://github.com/opencontainers/image-spec/blob/main/image-index.md
+    """
+
+    def __init__(self, package_url: str, tag: str) -> None:
+        self.qualified_name = f"{package_url}:{tag}"
+        logger.info(f"Getting image index for {self.qualified_name}")
+        try:
+            proc = subprocess.run(
+                [
+                    shutil.which("docker"),
+                    "buildx",
+                    "imagetools",
+                    "inspect",
+                    "--raw",
+                    self.qualified_name,
+                ],
+                capture_output=True,
+                check=True,
+            )
+
+            self._data = json.loads(proc.stdout)
+
+        except subprocess.CalledProcessError as e:
+            logger.error(
+                f"Failed to get image index for {self.qualified_name}: {e.stderr}",
+            )
+            raise e
+
+    @property
+    def image_pointers(self) -> Iterator[ImageProperties]:
+        for manifest_data in self._data["manifests"]:
+            yield ImageProperties(manifest_data)
+
+
 class RegistryTagsCleaner:
     """
     This is the base class for the image registry cleaning.  Given a package
@@ -85,7 +128,10 @@ class RegistryTagsCleaner:
 
     def clean(self):
         """
-        This method will delete image versions, based on the selected tags to delete
+        This method will delete image versions, based on the selected tags to delete.
+        It behaves more like an unlinking than actual deletion.  Removing the tag
+        simply removes a pointer to an image, but the actual image data remains accessible
+        if one has the sha256 digest of it.
         """
         for tag_to_delete in self.tags_to_delete:
             package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
@@ -149,27 +195,17 @@ class RegistryTagsCleaner:
 
             # Parse manifests to locate digests pointed to
             for tag in sorted(self.tags_to_keep):
-                full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
-                logger.info(f"Checking manifest for {full_name}")
                 try:
-                    proc = subprocess.run(
-                        [
-                            shutil.which("docker"),
-                            "manifest",
-                            "inspect",
-                            full_name,
-                        ],
-                        capture_output=True,
+                    image_index = ImageIndex(
+                        f"ghcr.io/{self.repo_owner}/{self.package_name}",
+                        tag,
                     )
-
-                    manifest_list = json.loads(proc.stdout)
-                    for manifest_data in manifest_list["manifests"]:
-                        manifest = DockerManifest2(manifest_data)
+                    for manifest in image_index.image_pointers:
 
                         if manifest.digest in untagged_versions:
                             logger.info(
                                 f"Skipping deletion of {manifest.digest},"
-                                f" referred to by {full_name}"
+                                f" referred to by {image_index.qualified_name}"
                                 f" for {manifest.platform}",
                             )
                             del untagged_versions[manifest.digest]
@@ -241,6 +277,55 @@ class RegistryTagsCleaner:
         # By default, keep anything which is tagged
         self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
 
+    def check_remaining_tags_valid(self):
+        """
+        Checks the non-deleted tags are still valid.  The assumption is if the
+        manifest is can be inspected and each image manifest if points to can be
+        inspected, the image will still pull.
+
+        https://github.com/opencontainers/image-spec/blob/main/image-index.md
+        """
+        logger.info("Beginning confirmation step")
+        a_tag_failed = False
+        for tag in sorted(self.tags_to_keep):
+
+            try:
+                image_index = ImageIndex(
+                    f"ghcr.io/{self.repo_owner}/{self.package_name}",
+                    tag,
+                )
+                for manifest in image_index.image_pointers:
+                    logger.info(f"Checking {manifest.digest} for {manifest.platform}")
+
+                    # This follows the pointer from the index to an actual image, layers and all
+                    # Note the format is @
+                    digest_name = f"ghcr.io/{self.repo_owner}/{self.package_name}@{manifest.digest}"
+
+                    try:
+
+                        subprocess.run(
+                            [
+                                shutil.which("docker"),
+                                "buildx",
+                                "imagetools",
+                                "inspect",
+                                "--raw",
+                                digest_name,
+                            ],
+                            capture_output=True,
+                            check=True,
+                        )
+                    except subprocess.CalledProcessError as e:
+                        logger.error(f"Failed to inspect digest: {e.stderr}")
+                        a_tag_failed = True
+            except subprocess.CalledProcessError as e:
+                a_tag_failed = True
+                logger.error(f"Failed to inspect: {e.stderr}")
+                continue
+
+        if a_tag_failed:
+            raise Exception("At least one image tag failed to inspect")
+
 
 class MainImageTagsCleaner(RegistryTagsCleaner):
     def decide_what_tags_to_keep(self):
@@ -301,7 +386,7 @@ class MainImageTagsCleaner(RegistryTagsCleaner):
 
 class LibraryTagsCleaner(RegistryTagsCleaner):
     """
-    Exists for the off change that someday, the installer library images
+    Exists for the off chance that someday, the installer library images
     will need their own logic
     """
 
@@ -397,6 +482,10 @@ def _main():
             # Clean images which are untagged
             cleaner.clean_untagged(args.is_manifest)
 
+            # Verify remaining tags still pull
+            if args.is_manifest:
+                cleaner.check_remaining_tags_valid()
+
 
 if __name__ == "__main__":
     _main()

.github/workflows/ci.yml

@@ -113,16 +113,12 @@ jobs:
       PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
       PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
       PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
-      # Skip Tests which require convert
-      PAPERLESS_TEST_SKIP_CONVERT: 1
       # Enable Gotenberg end to end testing
       GOTENBERG_LIVE: 1
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
       -
         name: Start containers
         run: |
@@ -145,6 +141,10 @@ jobs:
         run: |
           sudo apt-get update -qq
           sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
+      -
+        name: Configure ImageMagick
+        run: |
+          sudo cp docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
       -
         name: Install Python dependencies
         run: |
@@ -160,27 +160,14 @@ jobs:
           cd src/
           pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra
       -
-        name: Get changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v35
+        name: Upload coverage to Codecov
+        if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }}
+        uses: codecov/codecov-action@v3
         with:
-          files: |
-            src/**
-      -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == ${{ env.DEFAULT_PYTHON_VERSION }} && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv --python ${{ steps.setup-python.outputs.python-version }} run coveralls --service=github
+          # not required for public repos, but intermittently fails otherwise
+          token: ${{ secrets.CODECOV_TOKEN }}
+          # future expansion
+          flags: backend
       -
         name: Stop containers
         if: always()
@@ -212,12 +199,6 @@ jobs:
     name: Prepare Docker Pipeline Data
     if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-22.04
-    # If the push triggered the installer library workflow, wait for it to
-    # complete here.  This ensures the required versions for the final
-    # image have been built, while not waiting at all if the versions haven't changed
-    concurrency:
-      group: build-installer-library
-      cancel-in-progress: false
     needs:
       - documentation
       - tests-backend
@@ -353,7 +334,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: ./Dockerfile
@@ -448,21 +429,48 @@ jobs:
       -
         name: Move files
         run: |
-          mkdir dist
-          mkdir dist/paperless-ngx
-          mkdir dist/paperless-ngx/scripts
-          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
-          cp paperless.conf.example dist/paperless-ngx/paperless.conf
-          cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
-          cp -r docker/ dist/paperless-ngx/docker
-          cp scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
-          cp -r src/ dist/paperless-ngx/src
-          cp -r docs/_build/html/ dist/paperless-ngx/docs
-          mv static dist/paperless-ngx
+          echo "Making dist folders"
+          for directory in dist \
+                          dist/paperless-ngx \
+                          dist/paperless-ngx/scripts;
+          do
+            mkdir --verbose --parents ${directory}
+          done
+
+          echo "Copying basic files"
+          for file_name in .dockerignore \
+                          .env \
+                          Dockerfile \
+                          Pipfile \
+                          Pipfile.lock \
+                          requirements.txt \
+                          LICENSE \
+                          README.md \
+                          paperless.conf.example \
+                          gunicorn.conf.py
+          do
+            cp --verbose ${file_name} dist/paperless-ngx/
+          done
+          mv --verbose dist/paperless-ngx/paperless.conf.example paperless.conf
+
+          echo "Copying Docker related files"
+          cp --recursive docker/ dist/paperless-ngx/docker
+
+          echo "Copying startup scripts"
+          cp --verbose scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
+
+          echo "Copying source files"
+          cp --recursive src/ dist/paperless-ngx/src
+          echo "Copying documentation"
+          cp --recursive docs/_build/html/ dist/paperless-ngx/docs
+
+          mv --verbose static dist/paperless-ngx
       -
         name: Make release package
         run: |
+          echo "Creating release archive"
           cd dist
+          sudo chown -R 1000:1000 paperless-ngx/
           tar -cJf paperless-ngx.tar.xz paperless-ngx/
       -
         name: Upload release artifact
@@ -500,7 +508,7 @@ jobs:
       -
         name: Create Release and Changelog
         id: create-release
-        uses: paperless-ngx/release-drafter@master
+        uses: release-drafter/release-drafter@v5
         with:
           name: Paperless-ngx ${{ steps.get_version.outputs.version }}
           tag: ${{ steps.get_version.outputs.version }}

.github/workflows/cleanup-tags.yml

@@ -62,9 +62,9 @@ jobs:
         with:
           python-version: "3.10"
       -
-        name: Install httpx
+        name: Install Python libraries
         run: |
-          python -m pip install httpx
+          python -m pip install httpx docker
       #
       # Clean up primary package
       #
@@ -81,13 +81,3 @@ jobs:
         if: "${{ env.TOKEN != '' }}"
         run: |
           python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
-      #
-      # Verify tags which are left still pull
-      #
-      -
-        name: Check all tags still pull
-        run: |
-          ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
-          echo "Pulling all tags of ${ghcr_name}"
-          docker pull --quiet --all-tags ${ghcr_name}
-          docker image list

.github/workflows/installer-library.yml

@@ -169,3 +169,142 @@ jobs:
         PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
         PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
         LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}
+
+  commit-binary-files:
+    name: Store installers
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+      - build-jbig2enc
+      - build-psycopg2-wheel
+      - build-pikepdf-wheel
+    runs-on: ubuntu-22.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: binary-library
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends tree
+      -
+        name: Extract qpdf files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+
+          docker pull --quiet ${tag}
+          docker create --name qpdf-extract ${tag}
+
+          mkdir --parents qpdf/${version}/amd64
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/amd64 qpdf/${version}
+
+          mkdir --parents qpdf/${version}/arm64
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/arm64 qpdf/${version}
+
+          mkdir --parents qpdf/${version}/armv7
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/armv7 qpdf/${version}
+      -
+        name: Extract psycopg2 files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/amd64
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/amd64
+          mv psycopg2/${version}/amd64/wheels/* psycopg2/${version}/amd64
+          rm -r psycopg2/${version}/amd64/wheels/
+          docker rm psycopg2-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/arm64
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/arm64
+          mv psycopg2/${version}/arm64/wheels/* psycopg2/${version}/arm64
+          rm -r psycopg2/${version}/arm64/wheels/
+          docker rm psycopg2-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/armv7
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/armv7
+          mv psycopg2/${version}/armv7/wheels/* psycopg2/${version}/armv7
+          rm -r psycopg2/${version}/armv7/wheels/
+          docker rm psycopg2-extract
+      -
+        name: Extract pikepdf files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/amd64
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/amd64
+          mv pikepdf/${version}/amd64/wheels/* pikepdf/${version}/amd64
+          rm -r pikepdf/${version}/amd64/wheels/
+          docker rm pikepdf-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/arm64
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/arm64
+          mv pikepdf/${version}/arm64/wheels/* pikepdf/${version}/arm64
+          rm -r pikepdf/${version}/arm64/wheels/
+          docker rm pikepdf-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/armv7
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/armv7
+          mv pikepdf/${version}/armv7/wheels/* pikepdf/${version}/armv7
+          rm -r pikepdf/${version}/armv7/wheels/
+          docker rm pikepdf-extract
+      -
+        name: Extract jbig2enc files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/amd64
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/amd64/
+          mv jbig2enc/${version}/amd64/build/* jbig2enc/${version}/amd64/
+          docker rm jbig2enc-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/arm64
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/arm64
+          mv jbig2enc/${version}/arm64/build/* jbig2enc/${version}/arm64/
+          docker rm jbig2enc-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/armv7
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/armv7
+          mv jbig2enc/${version}/armv7/build/* jbig2enc/${version}/armv7/
+          docker rm jbig2enc-extract
+      -
+        name: Show file structure
+        run: |
+          tree .
+      -
+        name: Commit files
+        run: |
+          git config --global user.name "github-actions"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add pikepdf/ qpdf/ psycopg2/ jbig2enc/
+          git commit -m "Updating installer packages" || true
+          git push origin || true

.github/workflows/reusable-workflow-builder.yml

@@ -45,7 +45,7 @@ jobs:
         uses: docker/setup-qemu-action@v2
       -
         name: Build ${{ fromJSON(inputs.build-json).name }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: ${{ inputs.dockerfile }}

.python-version

@@ -1 +1 @@
-3.8.15
+3.8.16

Dockerfile

@@ -1,26 +1,12 @@
 # syntax=docker/dockerfile:1.4
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md
 
-# Pull the installer images from the library
-# These are all built previously
-# They provide either a .deb or .whl
-
-ARG JBIG2ENC_VERSION
-ARG QPDF_VERSION
-ARG PIKEPDF_VERSION
-ARG PSYCOPG2_VERSION
-
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
-
+# Stage: compile-frontend
+# Purpose: Compiles the frontend
+# Notes:
+#  - Does NPM stuff with Typescript and such
 FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
 
-# This stage compiles the frontend
-# This stage runs once for the native platform, as the outputs are not
-# dependent on target arch
-# Inputs: None
-
 COPY ./src-ui /src/src-ui
 
 WORKDIR /src/src-ui
@@ -30,15 +16,13 @@ RUN set -eux \
 RUN set -eux \
   && ./node_modules/.bin/ng build --configuration production
 
+# Stage: pipenv-base
+# Purpose: Generates a requirements.txt file for building
+# Comments:
+#  - pipenv dependencies are not left in the final image
+#  - pipenv can't touch the final image somehow
 FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base
 
-# This stage generates the requirements.txt file using pipenv
-# This stage runs once for the native platform, as the outputs are not
-# dependent on target arch
-# This way, pipenv dependencies are not left in the final image
-# nor can pipenv mess up the final image somehow
-# Inputs: None
-
 WORKDIR /usr/src/pipenv
 
 COPY Pipfile* ./
@@ -49,6 +33,10 @@ RUN set -eux \
   && echo "Generating requirement.txt" \
     && pipenv requirements > requirements.txt
 
+# Stage: main-app
+# Purpose: The final image
+# Comments:
+#  - Don't leave anything extra in here
 FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
@@ -58,30 +46,23 @@ LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-n
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
 ARG DEBIAN_FRONTEND=noninteractive
-# Buildx provided
+# Buildx provided, must be defined to use though
 ARG TARGETARCH
 ARG TARGETVARIANT
 
 # Workflow provided
+ARG JBIG2ENC_VERSION
 ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION
 
 #
 # Begin installation and configuration
 # Order the steps below from least often changed to most
 #
 
-# copy jbig2enc
-# Basically will never change again
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
-
 # Packages need for running
 ARG RUNTIME_PACKAGES="\
-  # Python
-  python3 \
-  python3-pip \
-  python3-setuptools \
   # General utils
   curl \
   # Docker specific
@@ -145,7 +126,7 @@ RUN set -eux \
     && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
     && rm -rf /var/lib/apt/lists/* \
   && echo "Installing supervisor" \
-    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.4
+    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.5
 
 # Copy gunicorn config
 # Changes very infrequently
@@ -154,7 +135,6 @@ WORKDIR /usr/src/paperless/
 COPY gunicorn.conf.py .
 
 # setup docker-specific things
-# Use mounts to avoid copying installer files into the image
 # These change sometimes, but rarely
 WORKDIR /usr/src/paperless/src/docker/
 
@@ -196,21 +176,31 @@ RUN set -eux \
     && ./install_management_commands.sh
 
 # Install the built packages from the installer library images
-# Use mounts to avoid copying installer files into the image
 # These change sometimes
-RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
-    --mount=type=bind,from=psycopg2-builder,target=/psycopg2 \
-    --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
-  set -eux \
+RUN set -eux \
+  && echo "Getting binaries" \
+    && mkdir paperless-ngx \
+    && curl --fail --silent --show-error --output paperless-ngx.tar.gz --location https://github.com/paperless-ngx/paperless-ngx/archive/41d6e7e407af09a0882736d50c89b6e015997bff.tar.gz \
+    && tar -xf paperless-ngx.tar.gz --directory paperless-ngx --strip-components=1 \
+    && cd paperless-ngx \
+    # Setting a specific revision ensures we know what this installed
+    # and ensures cache breaking on changes
+  && echo "Installing jbig2enc" \
+    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/jbig2 /usr/local/bin/ \
+    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/libjbig2enc* /usr/local/lib/ \
   && echo "Installing qpdf" \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
+    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
+    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
   && echo "Installing pikepdf and dependencies" \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/*.whl \
+    && python3 -m pip install --no-cache-dir ./pikepdf/${PIKEPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.whl \
     && python3 -m pip list \
   && echo "Installing psycopg2" \
-    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
-    && python3 -m pip list
+    && python3 -m pip install --no-cache-dir ./psycopg2/${PSYCOPG2_VERSION}/${TARGETARCH}${TARGETVARIANT}/psycopg2*.whl \
+    && python3 -m pip list \
+  && echo "Cleaning up image layer" \
+    && cd ../ \
+    && rm -rf paperless-ngx \
+    && rm paperless-ngx.tar.gz
 
 WORKDIR /usr/src/paperless/src/
 
@@ -254,11 +244,12 @@ COPY ./src ./
 COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
 
 # add users, setup scripts
+# Mount the compiled frontend to expected location
 RUN set -eux \
   && addgroup --gid 1000 paperless \
   && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
-  && chown -R paperless:paperless ../ \
-  && gosu paperless python3 manage.py collectstatic --clear --no-input \
+  && chown -R paperless:paperless /usr/src/paperless \
+  && gosu paperless python3 manage.py collectstatic --clear --no-input --link \
   && gosu paperless python3 manage.py compilemessages
 
 VOLUME ["/usr/src/paperless/data", \

Pipfile

@@ -12,6 +12,7 @@ name = "piwheels"
 dateparser = "~=1.1"
 django = "~=4.1"
 django-cors-headers = "*"
+django-compression-middleware = "*"
 django-extensions = "*"
 django-filter = "~=22.1"
 djangorestframework = "~=3.14"
@@ -53,21 +54,18 @@ nltk = "*"
 pdf2image = "*"
 flower = "*"
 bleach = "*"
-
 #
 # Packages locked due to issues (try to check if these are fixed in a release every so often)
 #
-
 # Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
 scipy = "==1.8.1"
-
 # Newer versions aren't builting yet (see https://www.piwheels.org/project/cryptography/)
 cryptography = "==38.0.1"
-
 # Locked version until https://github.com/django/channels_redis/issues/332
 # is resolved
 channels-redis = "==3.4.1"
 
+
 [dev-packages]
 coveralls = "*"
 factory-boy = "*"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "5d6da0ede3fc7dd05c9a1d836bf8786285b10b6134e763d06ee90d6e1ccb2be7"
+            "sha256": "d70848276d3ac35fa361c15ac2d634344cdb08618790502669eee209fc16fa00"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -117,6 +117,93 @@
             "index": "pypi",
             "version": "==5.0.1"
         },
+        "brotli": {
+            "hashes": [
+                "sha256:02177603aaca36e1fd21b091cb742bb3b305a569e2402f1ca38af471777fb019",
+                "sha256:11d3283d89af7033236fa4e73ec2cbe743d4f6a81d41bd234f24bf63dde979df",
+                "sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d",
+                "sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8",
+                "sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b",
+                "sha256:19598ecddd8a212aedb1ffa15763dd52a388518c4550e615aed88dc3753c0f0c",
+                "sha256:1c48472a6ba3b113452355b9af0a60da5c2ae60477f8feda8346f8fd48e3e87c",
+                "sha256:268fe94547ba25b58ebc724680609c8ee3e5a843202e9a381f6f9c5e8bdb5c70",
+                "sha256:269a5743a393c65db46a7bb982644c67ecba4b8d91b392403ad8a861ba6f495f",
+                "sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181",
+                "sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130",
+                "sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19",
+                "sha256:3148362937217b7072cf80a2dcc007f09bb5ecb96dae4617316638194113d5be",
+                "sha256:330e3f10cd01da535c70d09c4283ba2df5fb78e915bea0a28becad6e2ac010be",
+                "sha256:336b40348269f9b91268378de5ff44dc6fbaa2268194f85177b53463d313842a",
+                "sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa",
+                "sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429",
+                "sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126",
+                "sha256:3b8b09a16a1950b9ef495a0f8b9d0a87599a9d1f179e2d4ac014b2ec831f87e7",
+                "sha256:3c1306004d49b84bd0c4f90457c6f57ad109f5cc6067a9664e12b7b79a9948ad",
+                "sha256:3ffaadcaeafe9d30a7e4e1e97ad727e4f5610b9fa2f7551998471e3736738679",
+                "sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4",
+                "sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0",
+                "sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b",
+                "sha256:495ba7e49c2db22b046a53b469bbecea802efce200dffb69b93dd47397edc9b6",
+                "sha256:4d1b810aa0ed773f81dceda2cc7b403d01057458730e309856356d4ef4188438",
+                "sha256:503fa6af7da9f4b5780bb7e4cbe0c639b010f12be85d02c99452825dd0feef3f",
+                "sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389",
+                "sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6",
+                "sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26",
+                "sha256:5bf37a08493232fbb0f8229f1824b366c2fc1d02d64e7e918af40acd15f3e337",
+                "sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7",
+                "sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14",
+                "sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2",
+                "sha256:68715970f16b6e92c574c30747c95cf8cf62804569647386ff032195dc89a430",
+                "sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296",
+                "sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12",
+                "sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f",
+                "sha256:73fd30d4ce0ea48010564ccee1a26bfe39323fde05cb34b5863455629db61dc7",
+                "sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d",
+                "sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a",
+                "sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452",
+                "sha256:7ee83d3e3a024a9618e5be64648d6d11c37047ac48adff25f12fa4226cf23d1c",
+                "sha256:854c33dad5ba0fbd6ab69185fec8dab89e13cda6b7d191ba111987df74f38761",
+                "sha256:85f7912459c67eaab2fb854ed2bc1cc25772b300545fe7ed2dc03954da638649",
+                "sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b",
+                "sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea",
+                "sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c",
+                "sha256:8ed6a5b3d23ecc00ea02e1ed8e0ff9a08f4fc87a1f58a2530e71c0f48adf882f",
+                "sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a",
+                "sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031",
+                "sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267",
+                "sha256:97f715cf371b16ac88b8c19da00029804e20e25f30d80203417255d239f228b5",
+                "sha256:9bf919756d25e4114ace16a8ce91eb340eb57a08e2c6950c3cebcbe3dff2a5e7",
+                "sha256:9d12cf2851759b8de8ca5fde36a59c08210a97ffca0eb94c532ce7b17c6a3d1d",
+                "sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c",
+                "sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43",
+                "sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa",
+                "sha256:b1375b5d17d6145c798661b67e4ae9d5496920d9265e2f00f1c2c0b5ae91fbde",
+                "sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17",
+                "sha256:b3523f51818e8f16599613edddb1ff924eeb4b53ab7e7197f85cbc321cdca32f",
+                "sha256:b43775532a5904bc938f9c15b77c613cb6ad6fb30990f3b0afaea82797a402d8",
+                "sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb",
+                "sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb",
+                "sha256:ba72d37e2a924717990f4d7482e8ac88e2ef43fb95491eb6e0d124d77d2a150d",
+                "sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b",
+                "sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4",
+                "sha256:c8e521a0ce7cf690ca84b8cc2272ddaf9d8a50294fd086da67e517439614c755",
+                "sha256:cab1b5964b39607a66adbba01f1c12df2e55ac36c81ec6ed44f2fca44178bf1a",
+                "sha256:cb02ed34557afde2d2da68194d12f5719ee96cfb2eacc886352cb73e3808fc5d",
+                "sha256:cc0283a406774f465fb45ec7efb66857c09ffefbe49ec20b7882eff6d3c86d3a",
+                "sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3",
+                "sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7",
+                "sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1",
+                "sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb",
+                "sha256:e1abbeef02962596548382e393f56e4c94acd286bd0c5afba756cffc33670e8a",
+                "sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91",
+                "sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b",
+                "sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1",
+                "sha256:e4c4e92c14a57c9bd4cb4be678c25369bf7a092d55fd0866f759e425b9660806",
+                "sha256:ec1947eabbaf8e0531e8e899fc1d9876c179fc518989461f5d24e2223395a9e3",
+                "sha256:f909bbbc433048b499cb9db9e713b5d8d949e8c109a2a548502fb9aa8630f0b1"
+            ],
+            "version": "==1.0.9"
+        },
         "celery": {
             "extras": [
                 "redis"
@@ -353,6 +440,14 @@
             "index": "pypi",
             "version": "==2.4.0"
         },
+        "django-compression-middleware": {
+            "hashes": [
+                "sha256:5e089b299a603f73254a495bf60d82eb8a7d4d636ba23dbc0ee4e4e911eee37d",
+                "sha256:71d4bcd09546cf88783150ac6467eb4168599534774b09b806be3017139c295b"
+            ],
+            "index": "pypi",
+            "version": "==0.4.2"
+        },
         "django-cors-headers": {
             "hashes": [
                 "sha256:37e42883b5f1f2295df6b4bba96eb2417a14a03270cb24b2a07f021cd4487cf4",
@@ -2003,6 +2098,63 @@
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==5.5.2"
+        },
+        "zstandard": {
+            "hashes": [
+                "sha256:04c298d381a3b6274b0a8001f0da0ec7819d052ad9c3b0863fe8c7f154061f76",
+                "sha256:0fde1c56ec118940974e726c2a27e5b54e71e16c6f81d0b4722112b91d2d9009",
+                "sha256:126aa8433773efad0871f624339c7984a9c43913952f77d5abeee7f95a0c0860",
+                "sha256:1a4fb8b4ac6772e4d656103ccaf2e43e45bd16b5da324b963d58ef360d09eb73",
+                "sha256:2e4812720582d0803e84aefa2ac48ce1e1e6e200ca3ce1ae2be6d410c1d637ae",
+                "sha256:2f01b27d0b453f07cbcff01405cdd007e71f5d6410eb01303a16ba19213e58e4",
+                "sha256:31d12fcd942dd8dbf52ca5f6b1bbe287f44e5d551a081a983ff3ea2082867863",
+                "sha256:3c927b6aa682c6d96225e1c797f4a5d0b9f777b327dea912b23471aaf5385376",
+                "sha256:3d5bb598963ac1f1f5b72dd006adb46ca6203e4fb7269a5b6e1f99e85b07ad38",
+                "sha256:401508efe02341ae681752a87e8ac9ef76df85ef1a238a7a21786a489d2c983d",
+                "sha256:4514b19abe6dbd36d6c5d75c54faca24b1ceb3999193c5b1f4b685abeabde3d0",
+                "sha256:47dfa52bed3097c705451bafd56dac26535545a987b6759fa39da1602349d7ba",
+                "sha256:4fa496d2d674c6e9cffc561639d17009d29adee84a27cf1e12d3c9be14aa8feb",
+                "sha256:55a513ec67e85abd8b8b83af8813368036f03e2d29a50fc94033504918273980",
+                "sha256:55b3187e0bed004533149882ef8c24e954321f3be81f8a9ceffe35099b82a0d0",
+                "sha256:593f96718ad906e24d6534187fdade28b611f8ed06e27ba972ba48aecec45fc6",
+                "sha256:5e21032efe673b887464667d09406bab6e16d96b09ad87e80859e3a20b6745b6",
+                "sha256:60a86b7b2b1c300779167cf595e019e61afcc0e20c4838692983a921db9006ac",
+                "sha256:619f9bf37cdb4c3dc9d4120d2a1003f5db9446f3618a323219f408f6a9df6725",
+                "sha256:660b91eca10ee1b44c47843894abe3e6cfd80e50c90dee3123befbf7ca486bd3",
+                "sha256:67710d220af405f5ce22712fa741d85e8b3ada7a457ea419b038469ba379837c",
+                "sha256:6caed86cd47ae93915d9031dc04be5283c275e1a2af2ceff33932071f3eeff4d",
+                "sha256:6d2182e648e79213b3881998b30225b3f4b1f3e681f1c1eaf4cacf19bde1040d",
+                "sha256:72758c9f785831d9d744af282d54c3e0f9db34f7eae521c33798695464993da2",
+                "sha256:74c2637d12eaacb503b0b06efdf55199a11b1d7c580bd3dd9dfe84cac97ef2f6",
+                "sha256:755020d5aeb1b10bffd93d119e7709a2a7475b6ad79c8d5226cea3f76d152ce0",
+                "sha256:7ccc4727300f223184520a6064c161a90b5d0283accd72d1455bcd85ec44dd0d",
+                "sha256:81ab21d03e3b0351847a86a0b298b297fde1e152752614138021d6d16a476ea6",
+                "sha256:8371217dff635cfc0220db2720fc3ce728cd47e72bb7572cca035332823dbdfc",
+                "sha256:876567136b0359f6581ecd892bdb4ca03a0eead0265db73206c78cff03bcdb0f",
+                "sha256:879411d04068bd489db57dcf6b82ffad3c5fb2a1fdd30817c566d8b7bedee442",
+                "sha256:898500957ae5e7f31b7271ace4e6f3625b38c0ac84e8cedde8de3a77a7fdae5e",
+                "sha256:8c9ca56345b0c5574db47560603de9d05f63cce5dfeb3a456eb60f3fec737ff2",
+                "sha256:8ec2c146e10b59c376b6bc0369929647fcd95404a503a7aa0990f21c16462248",
+                "sha256:8f7c68de4f362c1b2f426395fe4e05028c56d0782b2ec3ae18a5416eaf775576",
+                "sha256:909bdd4e19ea437eb9b45d6695d722f6f0fd9d8f493e837d70f92062b9f39faf",
+                "sha256:9d97c713433087ba5cee61a3e8edb54029753d45a4288ad61a176fa4718033ce",
+                "sha256:a65e0119ad39e855427520f7829618f78eb2824aa05e63ff19b466080cd99210",
+                "sha256:aa9087571729c968cd853d54b3f6e9d0ec61e45cd2c31e0eb8a0d4bdbbe6da2f",
+                "sha256:aef0889417eda2db000d791f9739f5cecb9ccdd45c98f82c6be531bdc67ff0f2",
+                "sha256:b253d0c53c8ee12c3e53d181fb9ef6ce2cd9c41cbca1c56a535e4fc8ec41e241",
+                "sha256:b80f6f6478f9d4ca26daee6c61584499493bf97950cfaa1a02b16bb5c2c17e70",
+                "sha256:be6329b5ba18ec5d32dc26181e0148e423347ed936dda48bf49fb243895d1566",
+                "sha256:c7560f622e3849cc8f3e999791a915addd08fafe80b47fcf3ffbda5b5151047c",
+                "sha256:d1a7a716bb04b1c3c4a707e38e2dee46ac544fff931e66d7ae944f3019fc55b8",
+                "sha256:d63b04e16df8ea21dfcedbf5a60e11cbba9d835d44cb3cbff233cfd037a916d5",
+                "sha256:d777d239036815e9b3a093fa9208ad314c040c26d7246617e70e23025b60083a",
+                "sha256:e892d3177380ec080550b56a7ffeab680af25575d291766bdd875147ba246a91",
+                "sha256:e9c90a44470f2999779057aeaf33461cbd8bb59d8f15e983150d10bb260e16e0",
+                "sha256:f097dda5d4f9b9b01b3c9fa2069f9c02929365f48f341feddf3d6b32510a2f93",
+                "sha256:f4ebfe03cbae821ef994b2e58e4df6a087470cc522aca502614e82a143365d45"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.19.0"
         }
     },
     "develop": {

README.md

@@ -1,7 +1,7 @@
 [![ci](https://github.com/paperless-ngx/paperless-ngx/workflows/ci/badge.svg)](https://github.com/paperless-ngx/paperless-ngx/actions)
 [![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx)
 [![Documentation Status](https://img.shields.io/github/deployments/paperless-ngx/paperless-ngx/github-pages?label=docs)](https://docs.paperless-ngx.com)
-[![Coverage Status](https://coveralls.io/repos/github/paperless-ngx/paperless-ngx/badge.svg?branch=master)](https://coveralls.io/github/paperless-ngx/paperless-ngx?branch=master)
+[![codecov](https://codecov.io/gh/paperless-ngx/paperless-ngx/branch/main/graph/badge.svg?token=VK6OUPJ3TY)](https://codecov.io/gh/paperless-ngx/paperless-ngx)
 [![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org)
 [![demo](https://cronitor.io/badges/ve7ItY/production/W5E_B9jkelG9ZbDiNHUPQEVH3MY.svg)](https://demo.paperless-ngx.com)
 

docker-builders/Dockerfile.jbig2enc

@@ -29,7 +29,20 @@ RUN set -eux \
     && ./autogen.sh \
     && ./configure \
     && make \
+  && echo "Gathering package data" \
+    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./pkg-list.txt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+  && echo "Moving files around" \
+    && mkdir build \
+    # Unlink a symlink that causes problems
+    && unlink ./src/.libs/libjbig2enc.la \
+    # Move what the link pointed to
+    && mv ./src/libjbig2enc.la ./build/ \
+    # Move the shared library .so files
+    && mv ./src/.libs/libjbig2enc* ./build/ \
+    # And move the cli binary
+    && mv ./src/jbig2 ./build/ \
+    && mv ./pkg-list.txt ./build/

docker-builders/Dockerfile.pikepdf

@@ -7,12 +7,17 @@
 # Default to pulling from the main repo registry when manually building
 ARG REPO="paperless-ngx/paperless-ngx"
 
-ARG QPDF_VERSION
-FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
-
 # This does nothing, except provide a name for a copy below
+ARG QPDF_VERSION
+FROM --platform=$BUILDPLATFORM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
 
-FROM python:3.9-slim-bullseye as main
+#
+# Stage: builder
+# Purpose:
+#  - Build the pikepdf wheel
+#  - Build any dependent wheels which can't be found
+#
+FROM python:3.9-slim-bullseye as builder
 
 LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
 
@@ -100,3 +105,14 @@ RUN set -eux \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
     && rm -rf /var/lib/apt/lists/*
+
+#
+# Stage: package
+# Purpose: Holds the compiled .whl files in a tiny image to pull
+#
+FROM alpine:3.17 as package
+
+WORKDIR /usr/src/wheels/
+
+COPY --from=builder /usr/src/wheels/*.whl ./
+COPY --from=builder /usr/src/wheels/pkg-list.txt ./

docker-builders/Dockerfile.psycopg2

@@ -2,7 +2,12 @@
 # Inputs:
 #    - PSYCOPG2_VERSION - Version to build
 
-FROM python:3.9-slim-bullseye as main
+#
+# Stage: builder
+# Purpose:
+#  - Build the psycopg2 wheel
+#
+FROM python:3.9-slim-bullseye as builder
 
 LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
 
@@ -48,3 +53,14 @@ RUN set -eux \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
     && rm -rf /var/lib/apt/lists/*
+
+#
+# Stage: package
+# Purpose: Holds the compiled .whl files in a tiny image to pull
+#
+FROM alpine:3.17 as package
+
+WORKDIR /usr/src/wheels/
+
+COPY --from=builder /usr/src/wheels/*.whl ./
+COPY --from=builder /usr/src/wheels/pkg-list.txt ./

docker-builders/README.md

@@ -0,0 +1,57 @@
+# Installer Library
+
+This folder contains the Dockerfiles for building certain installers or libraries, which are then pulled into the main image.
+
+## [jbig2enc](https://github.com/agl/jbig2enc)
+
+### Why
+
+JBIG is an image coding which can achieve better compression of images for PDFs.
+
+### What
+
+The Docker image builds a shared library file and utility, which is copied into the correct location in the final image.
+
+### Updating
+
+1. Ensure the given qpdf version is present in [Debian bookworm](https://packages.debian.org/bookworm/qpdf)
+2. Update `.build-config.json` to the given version
+3. If the Debian specific version has incremented, update `Dockerfile.qpdf`
+
+See Also:
+
+- [OCRMyPDF Documentation](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html)
+
+## [psycopg2](https://www.psycopg.org/)
+
+### Why
+
+The pre-built wheels of psycopg2 are built on Debian 9, which provides a quite old version of libpq-dev. This causes issue with authentication methods.
+
+### What
+
+The image builds psycopg2 wheels on Debian 10 and places the produced wheels into `/usr/src/wheels/`.
+
+See Also:
+
+- [Issue 266](https://github.com/paperless-ngx/paperless-ngx/issues/266)
+
+## [qpdf](https://qpdf.readthedocs.io/en/stable/index.html)
+
+### Why
+
+qpdf and it's library provide tools to read, manipulate and fix up PDFs. Version 11 is also required by `pikepdf` 6+ and Debian 9 does not provide above version 10.
+
+### What
+
+The Docker image cross compiles .deb installers for each supported architecture of the main image. The installers are placed in `/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/`
+
+## [pikepdf](https://pikepdf.readthedocs.io/en/latest/)
+
+### Why
+
+Required by OCRMyPdf, this is a general purpose library for PDF manipulation in Python via the qpdf libraries.
+
+### What
+
+The built wheels are placed into `/usr/src/wheels/`

docker/compose/docker-compose.ci-test.yml

@@ -6,7 +6,7 @@
 version: "3.7"
 services:
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     hostname: gotenberg
     container_name: gotenberg
     network_mode: host

docker/compose/docker-compose.mariadb-tika.yml

@@ -59,7 +59,7 @@ services:
       - gotenberg
       - tika
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000"]
       interval: 30s
@@ -83,7 +83,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     restart: unless-stopped
     # The gotenberg chromium route is used to convert .eml files. We do not
     # want to allow external content like tracking pixels or even javascript.

docker/compose/docker-compose.mariadb.yml

@@ -53,7 +53,7 @@ services:
       - db
       - broker
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000"]
       interval: 30s

docker/compose/docker-compose.portainer.yml

@@ -53,7 +53,7 @@ services:
       - db
       - broker
     ports:
-      - 8010:8000
+      - "8010:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s

docker/compose/docker-compose.postgres-tika.yml

@@ -57,7 +57,7 @@ services:
       - gotenberg
       - tika
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
@@ -77,7 +77,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     restart: unless-stopped
 
     # The gotenberg chromium route is used to convert .eml files. We do not

docker/compose/docker-compose.postgres.yml

@@ -51,7 +51,7 @@ services:
       - db
       - broker
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s

docker/compose/docker-compose.sqlite-tika.yml

@@ -46,7 +46,7 @@ services:
       - gotenberg
       - tika
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
@@ -65,7 +65,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     restart: unless-stopped
 
     # The gotenberg chromium route is used to convert .eml files. We do not

docker/compose/docker-compose.sqlite.yml

@@ -37,7 +37,7 @@ services:
     depends_on:
       - broker
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s

docker/docker-prepare.sh

@@ -80,7 +80,7 @@ django_checks() {
 
 search_index() {
 
-	local -r index_version=1
+	local -r index_version=3
 	local -r index_version_file=${DATA_DIR}/.index_version
 
 	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then

docker/flower-conditional.sh

@@ -3,5 +3,10 @@
 echo "Checking if we should start flower..."
 
 if [[ -n  "${PAPERLESS_ENABLE_FLOWER}" ]]; then
-	celery --app paperless flower
+	# Small delay to allow celery to be up first
+	echo "Starting flower in 5s"
+	sleep 5
+	celery --app paperless flower --conf=/usr/src/paperless/src/paperless/flowerconfig.py
+else
+	echo "Not starting flower"
 fi

docs/advanced_usage.md

@@ -121,7 +121,17 @@ Executed after the consumer sees a new document in the consumption
 folder, but before any processing of the document is performed. This
 script can access the following relevant environment variables set:
 
-- `DOCUMENT_SOURCE_PATH`
+| Environment Variable    | Description                                                  |
+| ----------------------- | ------------------------------------------------------------ |
+| `DOCUMENT_SOURCE_PATH`  | Original path of the consumed document                       |
+| `DOCUMENT_WORKING_PATH` | Path to a copy of the original that consumption will work on |
+
+!!! note
+
+    Pre-consume scripts which modify the document should only change
+    the `DOCUMENT_WORKING_PATH` file or a second consume task may
+    be triggered, leading to failures as two tasks work on the
+    same document path
 
 A simple but common example for this would be creating a simple script
 like this:
@@ -130,7 +140,7 @@ like this:
 
 ```bash
 #!/usr/bin/env bash
-pdf2pdfocr.py -i ${DOCUMENT_SOURCE_PATH}
+pdf2pdfocr.py -i ${DOCUMENT_WORKING_PATH}
 ```
 
 `/etc/paperless.conf`
@@ -157,26 +167,36 @@ Executed after the consumer has successfully processed a document and
 has moved it into paperless. It receives the following environment
 variables:
 
-- `DOCUMENT_ID`
-- `DOCUMENT_FILE_NAME`
-- `DOCUMENT_CREATED`
-- `DOCUMENT_MODIFIED`
-- `DOCUMENT_ADDED`
-- `DOCUMENT_SOURCE_PATH`
-- `DOCUMENT_ARCHIVE_PATH`
-- `DOCUMENT_THUMBNAIL_PATH`
-- `DOCUMENT_DOWNLOAD_URL`
-- `DOCUMENT_THUMBNAIL_URL`
-- `DOCUMENT_CORRESPONDENT`
-- `DOCUMENT_TAGS`
-- `DOCUMENT_ORIGINAL_FILENAME`
+| Environment Variable         | Description                                   |
+| ---------------------------- | --------------------------------------------- |
+| `DOCUMENT_ID`                | Database primary key of the document          |
+| `DOCUMENT_FILE_NAME`         | Formatted filename, not including paths       |
+| `DOCUMENT_CREATED`           | Date & time when document created             |
+| `DOCUMENT_MODIFIED`          | Date & time when document was last modified   |
+| `DOCUMENT_ADDED`             | Date & time when document was added           |
+| `DOCUMENT_SOURCE_PATH`       | Path to the original document file            |
+| `DOCUMENT_ARCHIVE_PATH`      | Path to the generate archive file (if any)    |
+| `DOCUMENT_THUMBNAIL_PATH`    | Path to the generated thumbnail               |
+| `DOCUMENT_DOWNLOAD_URL`      | URL for document download                     |
+| `DOCUMENT_THUMBNAIL_URL`     | URL for the document thumbnail                |
+| `DOCUMENT_CORRESPONDENT`     | Assigned correspondent (if any)               |
+| `DOCUMENT_TAGS`              | Comma separated list of tags applied (if any) |
+| `DOCUMENT_ORIGINAL_FILENAME` | Filename of original document                 |
 
-The script can be in any language, but for a simple shell script
-example, you can take a look at
-[post-consumption-example.sh](https://github.com/paperless-ngx/paperless-ngx/blob/main/scripts/post-consumption-example.sh)
-in this project.
+The script can be in any language, A simple shell script example:
 
-The post consumption script cannot cancel the consumption process.
+```bash title="post-consumption-example"
+--8<-- "./scripts/post-consumption-example.sh"
+```
+
+!!! note
+
+    The post consumption script cannot cancel the consumption process.
+
+!!! warning
+
+    The post consumption script should not modify the document files
+    directly
 
 The script's stdout and stderr will be logged line by line to the
 webserver log, along with the exit code of the script.
@@ -481,3 +501,9 @@ You can also set the default for new tables (this does NOT affect
 existing tables) with:
 
 `ALTER DATABASE <db_name> CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`
+
+!!! warning
+
+    Using mariadb version 10.4+ is recommended. Using the `utf8mb3` character set on
+    an older system may fix issues that can arise while setting up Paperless-ngx but
+    `utf8mb3` can cause issues with consumption (where `utf8mb4` does not).

docs/changelog.md

@@ -1,5 +1,154 @@
 # Changelog
 
+## paperless-ngx 1.12.2
+
+_Note: Version 1.12.x introduced searching of comments which will work for comments added after the upgrade but a reindex of the search index is required in order to be able to search
+older comments. The Docker image will automatically perform this reindex, bare metal installations will have to perform this manually, see [the docs](https://docs.paperless-ngx.com/administration/#index)._
+
+### Bug Fixes
+
+- Bugfix: Allow pre-consume scripts to modify incoming file [@stumpylog](https://github.com/stumpylog) ([#2547](https://github.com/paperless-ngx/paperless-ngx/pull/2547))
+- Bugfix: Return to page based barcode scanning [@stumpylog](https://github.com/stumpylog) ([#2544](https://github.com/paperless-ngx/paperless-ngx/pull/2544))
+- Fix: Try to prevent title debounce overwriting [@shamoon](https://github.com/shamoon) ([#2543](https://github.com/paperless-ngx/paperless-ngx/pull/2543))
+- Fix comment search highlight + multi-word search [@shamoon](https://github.com/shamoon) ([#2542](https://github.com/paperless-ngx/paperless-ngx/pull/2542))
+- Bugfix: Request PDF/A format from Gotenberg [@stumpylog](https://github.com/stumpylog) ([#2530](https://github.com/paperless-ngx/paperless-ngx/pull/2530))
+- Fix: Trigger reindex for pre-existing comments [@shamoon](https://github.com/shamoon) ([#2519](https://github.com/paperless-ngx/paperless-ngx/pull/2519))
+
+### Documentation
+
+- Bugfix: Allow pre-consume scripts to modify incoming file [@stumpylog](https://github.com/stumpylog) ([#2547](https://github.com/paperless-ngx/paperless-ngx/pull/2547))
+- Fix: Trigger reindex for pre-existing comments [@shamoon](https://github.com/shamoon) ([#2519](https://github.com/paperless-ngx/paperless-ngx/pull/2519))
+- Minor updates to development documentation [@clemensrieder](https://github.com/clemensrieder) ([#2474](https://github.com/paperless-ngx/paperless-ngx/pull/2474))
+- [Documentation] Add v1.12.1 changelog [@github-actions](https://github.com/github-actions) ([#2515](https://github.com/paperless-ngx/paperless-ngx/pull/2515))
+
+### Maintenance
+
+- Chore: Fix tag cleaner to work with attestations [@stumpylog](https://github.com/stumpylog) ([#2532](https://github.com/paperless-ngx/paperless-ngx/pull/2532))
+- Chore: Make installers statically versioned [@stumpylog](https://github.com/stumpylog) ([#2517](https://github.com/paperless-ngx/paperless-ngx/pull/2517))
+
+### All App Changes
+
+- Bugfix: Allow pre-consume scripts to modify incoming file [@stumpylog](https://github.com/stumpylog) ([#2547](https://github.com/paperless-ngx/paperless-ngx/pull/2547))
+- Bugfix: Return to page based barcode scanning [@stumpylog](https://github.com/stumpylog) ([#2544](https://github.com/paperless-ngx/paperless-ngx/pull/2544))
+- Fix: Try to prevent title debounce overwriting [@shamoon](https://github.com/shamoon) ([#2543](https://github.com/paperless-ngx/paperless-ngx/pull/2543))
+- Fix comment search highlight + multi-word search [@shamoon](https://github.com/shamoon) ([#2542](https://github.com/paperless-ngx/paperless-ngx/pull/2542))
+- Bugfix: Request PDF/A format from Gotenberg [@stumpylog](https://github.com/stumpylog) ([#2530](https://github.com/paperless-ngx/paperless-ngx/pull/2530))
+
+## paperless-ngx 1.12.1
+
+### Bug Fixes
+
+- Fix: comments not showing in search until after manual reindex in v1.12 [@shamoon](https://github.com/shamoon) ([#2513](https://github.com/paperless-ngx/paperless-ngx/pull/2513))
+- Fix: date range search broken in 1.12 [@shamoon](https://github.com/shamoon) ([#2509](https://github.com/paperless-ngx/paperless-ngx/pull/2509))
+
+### Documentation
+
+- [Documentation] Add v1.12.0 changelog [@github-actions](https://github.com/github-actions) ([#2507](https://github.com/paperless-ngx/paperless-ngx/pull/2507))
+
+### Maintenance
+
+- Moves back to the main release-drafter now that it does what we wanted [@stumpylog](https://github.com/stumpylog) ([#2503](https://github.com/paperless-ngx/paperless-ngx/pull/2503))
+
+### All App Changes
+
+- Fix: comments not showing in search until after manual reindex in v1.12 [@shamoon](https://github.com/shamoon) ([#2513](https://github.com/paperless-ngx/paperless-ngx/pull/2513))
+- Fix: date range search broken in 1.12 [@shamoon](https://github.com/shamoon) ([#2509](https://github.com/paperless-ngx/paperless-ngx/pull/2509))
+
+## paperless-ngx 1.12.0
+
+### Features
+
+- New document_exporter options [@mhelleboid](https://github.com/mhelleboid) ([#2448](https://github.com/paperless-ngx/paperless-ngx/pull/2448))
+- Read ASN from barcode on page [@peterkappelt](https://github.com/peterkappelt) ([#2437](https://github.com/paperless-ngx/paperless-ngx/pull/2437))
+- Add AppleMail color tag support [@clemensrieder](https://github.com/clemensrieder) ([#2407](https://github.com/paperless-ngx/paperless-ngx/pull/2407))
+- Feature: Retain original filename on upload [@stumpylog](https://github.com/stumpylog) ([#2404](https://github.com/paperless-ngx/paperless-ngx/pull/2404))
+- Feature: Control scheduled tasks via cron expressions [@stumpylog](https://github.com/stumpylog) ([#2403](https://github.com/paperless-ngx/paperless-ngx/pull/2403))
+- Simplify json parsing in build scripts [@tribut](https://github.com/tribut) ([#2370](https://github.com/paperless-ngx/paperless-ngx/pull/2370))
+- Feature: include comments in advanced search [@shamoon](https://github.com/shamoon) ([#2351](https://github.com/paperless-ngx/paperless-ngx/pull/2351))
+
+### Bug Fixes
+
+- Fix: limit asn integer size [@shamoon](https://github.com/shamoon) ([#2498](https://github.com/paperless-ngx/paperless-ngx/pull/2498))
+- Bugfix: Rescales images for better barcode locating [@stumpylog](https://github.com/stumpylog) ([#2468](https://github.com/paperless-ngx/paperless-ngx/pull/2468))
+- Fix: fix downgrade migration [@shamoon](https://github.com/shamoon) ([#2494](https://github.com/paperless-ngx/paperless-ngx/pull/2494))
+- Fix: Allow setting mailrule order from frontend [@shamoon](https://github.com/shamoon) ([#2459](https://github.com/paperless-ngx/paperless-ngx/pull/2459))
+- Fix: tag color ordering [@shamoon](https://github.com/shamoon) ([#2456](https://github.com/paperless-ngx/paperless-ngx/pull/2456))
+- Fix: Better Handle arbitrary ISO 8601 strings after celery serializing [@shamoon](https://github.com/shamoon) ([#2441](https://github.com/paperless-ngx/paperless-ngx/pull/2441))
+- Use correct canonical path for nltk_data [@amo13](https://github.com/amo13) ([#2429](https://github.com/paperless-ngx/paperless-ngx/pull/2429))
+- Fix: Include optional socket file in release [@stumpylog](https://github.com/stumpylog) ([#2409](https://github.com/paperless-ngx/paperless-ngx/pull/2409))
+- Fix: display rtl content in correct direction [@shamoon](https://github.com/shamoon) ([#2302](https://github.com/paperless-ngx/paperless-ngx/pull/2302))
+- Fixed endpoint count in Docs The REST API [@PascalSenn](https://github.com/PascalSenn) ([#2386](https://github.com/paperless-ngx/paperless-ngx/pull/2386))
+- Fix subpath for websockets [@tribut](https://github.com/tribut) ([#2371](https://github.com/paperless-ngx/paperless-ngx/pull/2371))
+- Fix: Make missing environment from file files informational only [@stumpylog](https://github.com/stumpylog) ([#2368](https://github.com/paperless-ngx/paperless-ngx/pull/2368))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Fix: preview content remains hidden on mobile [@shamoon](https://github.com/shamoon) ([#2346](https://github.com/paperless-ngx/paperless-ngx/pull/2346))
+- Bugfix: Removal of alpha channel truncates multipage TIFFs [@stumpylog](https://github.com/stumpylog) ([#2335](https://github.com/paperless-ngx/paperless-ngx/pull/2335))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+
+### Documentation
+
+- Docs: Fix typo - docker-compose.yml file name in setup doc [@muli](https://github.com/muli) ([#2477](https://github.com/paperless-ngx/paperless-ngx/pull/2477))
+- document existence of document_thumbnails [@frrad](https://github.com/frrad) ([#2470](https://github.com/paperless-ngx/paperless-ngx/pull/2470))
+- Add optional sudo command to bare metal docs [@shamoon](https://github.com/shamoon) ([#2464](https://github.com/paperless-ngx/paperless-ngx/pull/2464))
+- Fix link [@edenhaus](https://github.com/edenhaus) ([#2458](https://github.com/paperless-ngx/paperless-ngx/pull/2458))
+- Documentation: Fix comment re bare metal runserver command [@shamoon](https://github.com/shamoon) ([#2420](https://github.com/paperless-ngx/paperless-ngx/pull/2420))
+- Fix formatting of config variable in docs [@peterkappelt](https://github.com/peterkappelt) ([#2445](https://github.com/paperless-ngx/paperless-ngx/pull/2445))
+- Update docs nginx reverse proxy example [@Sprinterfreak](https://github.com/Sprinterfreak) ([#2443](https://github.com/paperless-ngx/paperless-ngx/pull/2443))
+- [Documentation] Add note re for dev server [@shamoon](https://github.com/shamoon) ([#2387](https://github.com/paperless-ngx/paperless-ngx/pull/2387))
+- Fixed endpoint count in Docs The REST API [@PascalSenn](https://github.com/PascalSenn) ([#2386](https://github.com/paperless-ngx/paperless-ngx/pull/2386))
+- [ Docs] Update bare metal setup instructions [@natrius](https://github.com/natrius) ([#2281](https://github.com/paperless-ngx/paperless-ngx/pull/2281))
+- [Docs] Add Paperless Mobile app to docs [@astubenbord](https://github.com/astubenbord) ([#2378](https://github.com/paperless-ngx/paperless-ngx/pull/2378))
+- Tiny spelling change [@veverkap](https://github.com/veverkap) ([#2369](https://github.com/paperless-ngx/paperless-ngx/pull/2369))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+- [Documentation] Add note that PAPERLESS_URL cant contain a path [@shamoon](https://github.com/shamoon) ([#2319](https://github.com/paperless-ngx/paperless-ngx/pull/2319))
+- [Documentation] Add v1.11.3 changelog [@github-actions](https://github.com/github-actions) ([#2311](https://github.com/paperless-ngx/paperless-ngx/pull/2311))
+
+### Maintenance
+
+- Fix: Include optional socket file in release [@stumpylog](https://github.com/stumpylog) ([#2409](https://github.com/paperless-ngx/paperless-ngx/pull/2409))
+- Chore: remove helm chart code [@shamoon](https://github.com/shamoon) ([#2388](https://github.com/paperless-ngx/paperless-ngx/pull/2388))
+- Simplify json parsing in build scripts [@tribut](https://github.com/tribut) ([#2370](https://github.com/paperless-ngx/paperless-ngx/pull/2370))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Bump tj-actions/changed-files from 34 to 35 [@dependabot](https://github.com/dependabot) ([#2303](https://github.com/paperless-ngx/paperless-ngx/pull/2303))
+
+### Dependencies
+
+<details>
+<summary>4 changes</summary>
+
+- Chore: Backend library updates [@stumpylog](https://github.com/stumpylog) ([#2401](https://github.com/paperless-ngx/paperless-ngx/pull/2401))
+- Bump tj-actions/changed-files from 34 to 35 [@dependabot](https://github.com/dependabot) ([#2303](https://github.com/paperless-ngx/paperless-ngx/pull/2303))
+- Bump [@<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2306](https://github.com/paperless-ngx/paperless-ngx/pull/2306))
+- Bump [@<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2308](https://github.com/paperless-ngx/paperless-ngx/pull/2308))
+</details>
+
+### All App Changes
+
+- New document_exporter options [@mhelleboid](https://github.com/mhelleboid) ([#2448](https://github.com/paperless-ngx/paperless-ngx/pull/2448))
+- Fix: limit asn integer size [@shamoon](https://github.com/shamoon) ([#2498](https://github.com/paperless-ngx/paperless-ngx/pull/2498))
+- Fix: fix downgrade migration [@shamoon](https://github.com/shamoon) ([#2494](https://github.com/paperless-ngx/paperless-ngx/pull/2494))
+- Read ASN from barcode on page [@peterkappelt](https://github.com/peterkappelt) ([#2437](https://github.com/paperless-ngx/paperless-ngx/pull/2437))
+- Fix: Allow setting mailrule order from frontend [@shamoon](https://github.com/shamoon) ([#2459](https://github.com/paperless-ngx/paperless-ngx/pull/2459))
+- Chore: Update to Angular 15 \& associated frontend deps [@shamoon](https://github.com/shamoon) ([#2411](https://github.com/paperless-ngx/paperless-ngx/pull/2411))
+- Fix: tag color ordering [@shamoon](https://github.com/shamoon) ([#2456](https://github.com/paperless-ngx/paperless-ngx/pull/2456))
+- Fix: Better Handle arbitrary ISO 8601 strings after celery serializing [@shamoon](https://github.com/shamoon) ([#2441](https://github.com/paperless-ngx/paperless-ngx/pull/2441))
+- Use correct canonical path for nltk_data [@amo13](https://github.com/amo13) ([#2429](https://github.com/paperless-ngx/paperless-ngx/pull/2429))
+- Add AppleMail color tag support [@clemensrieder](https://github.com/clemensrieder) ([#2407](https://github.com/paperless-ngx/paperless-ngx/pull/2407))
+- Chore: Convert document exporter to use pathlib [@stumpylog](https://github.com/stumpylog) ([#2416](https://github.com/paperless-ngx/paperless-ngx/pull/2416))
+- Feature: Retain original filename on upload [@stumpylog](https://github.com/stumpylog) ([#2404](https://github.com/paperless-ngx/paperless-ngx/pull/2404))
+- Feature: Control scheduled tasks via cron expressions [@stumpylog](https://github.com/stumpylog) ([#2403](https://github.com/paperless-ngx/paperless-ngx/pull/2403))
+- Fix: display rtl content in correct direction [@shamoon](https://github.com/shamoon) ([#2302](https://github.com/paperless-ngx/paperless-ngx/pull/2302))
+- Fix subpath for websockets [@tribut](https://github.com/tribut) ([#2371](https://github.com/paperless-ngx/paperless-ngx/pull/2371))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Feature: include comments in advanced search [@shamoon](https://github.com/shamoon) ([#2351](https://github.com/paperless-ngx/paperless-ngx/pull/2351))
+- Chore: More frontend tests [@shamoon](https://github.com/shamoon) ([#2352](https://github.com/paperless-ngx/paperless-ngx/pull/2352))
+- Chore: Fixing up some minor annoyances [@stumpylog](https://github.com/stumpylog) ([#2348](https://github.com/paperless-ngx/paperless-ngx/pull/2348))
+- Bugfix: Removal of alpha channel truncates multipage TIFFs [@stumpylog](https://github.com/stumpylog) ([#2335](https://github.com/paperless-ngx/paperless-ngx/pull/2335))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+- Add Arabic language to frontend [@KhaledEmad7](https://github.com/KhaledEmad7) ([#2313](https://github.com/paperless-ngx/paperless-ngx/pull/2313))
+- Bump [@<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2306](https://github.com/paperless-ngx/paperless-ngx/pull/2306))
+- Bump [@<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2308](https://github.com/paperless-ngx/paperless-ngx/pull/2308))
+
 ## paperless-ngx 1.11.3
 
 ### Breaking Changes

docs/configuration.md

@@ -141,7 +141,8 @@ directory.
 files created using "collectstatic" manager command are stored.
 
     Unless you're doing something fancy, there is no need to override
-    this.
+    this.  If this is changed, you may need to run
+    `collectstatic` again.
 
     Defaults to "../static/", relative to the "src" directory.
 
@@ -625,7 +626,7 @@ services:
     # ...
 
     gotenberg:
-      image: gotenberg/gotenberg:7.6
+      image: gotenberg/gotenberg:7.8
       restart: unless-stopped
       # The gotenberg chromium route is used to convert .eml files. We do not
       # want to allow external content like tracking pixels or even javascript.
@@ -757,6 +758,18 @@ should be a valid crontab(5) expression describing when to run.
 
     Defaults to `30 0 * * sun` or Sunday at 30 minutes past midnight.
 
+`PAPERLESS_ENABLE_COMPRESSION=<bool>`
+
+: Enables compression of the responses from the webserver.
+
+: Defaults to 1, enabling compression.
+
+    !!! note
+
+        If you are using a proxy such as nginx, it is likely more efficient
+        to enable compression in your proxy configuration rather than
+        the webserver
+
 ## Polling {#polling}
 
 `PAPERLESS_CONSUMER_POLLING=<num>`
@@ -986,13 +999,20 @@ within your documents.
 `PAPERLESS_CONSUMER_IGNORE_PATTERNS=<json>`
 
 : By default, paperless ignores certain files and folders in the
-consumption directory, such as system files created by the Mac OS.
+consumption directory, such as system files created by the Mac OS
+or hidden folders some tools use to store data.
 
     This can be adjusted by configuring a custom json array with
     patterns to exclude.
 
+    For example, `.DS_STORE/*` will ignore any files found in a folder
+    named `.DS_STORE`, including `.DS_STORE/bar.pdf` and `foo/.DS_STORE/bar.pdf`
+
+    A pattern like `._*` will ignore anything starting with `._`, including:
+    `._foo.pdf` and `._bar/foo.pdf`
+
     Defaults to
-    `[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]`.
+    `[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]`.
 
 ## Binaries
 

docs/development.md

@@ -1,9 +1,9 @@
 # Development
 
 This section describes the steps you need to take to start development
-on paperless-ngx.
+on Paperless-ngx.
 
-Check out the source from github. The repository is organized in the
+Check out the source from GitHub. The repository is organized in the
 following way:
 
 - `main` always represents the latest release and will only see
@@ -12,7 +12,7 @@ following way:
 - `feature-X` contain bigger changes that will be in some release, but
   not necessarily the next one.
 
-When making functional changes to paperless, _always_ make your changes
+When making functional changes to Paperless-ngx, _always_ make your changes
 on the `dev` branch.
 
 Apart from that, the folder structure is as follows:
@@ -24,9 +24,9 @@ Apart from that, the folder structure is as follows:
   development.
 - `docker/` - Files required to build the docker image.
 
-## Contributing to Paperless
+## Contributing to Paperless-ngx
 
-Maybe you've been using Paperless for a while and want to add a feature
+Maybe you've been using Paperless-ngx for a while and want to add a feature
 or two, or maybe you've come across a bug that you have some ideas how
 to solve. The beauty of open source software is that you can see what's
 wrong and help to get it fixed for everyone!
@@ -36,13 +36,13 @@ conduct](https://github.com/paperless-ngx/paperless-ngx/blob/main/CODE_OF_CONDUC
 and other important information in the [contributing
 guidelines](https://github.com/paperless-ngx/paperless-ngx/blob/main/CONTRIBUTING.md).
 
-## Code formatting with pre-commit Hooks
+## Code formatting with pre-commit hooks
 
 To ensure a consistent style and formatting across the project source,
-the project utilizes a Git [`pre-commit`](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks)
-hook to perform some formatting and linting before a commit is allowed.
+the project utilizes Git [`pre-commit`](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks)
+hooks to perform some formatting and linting before a commit is allowed.
 That way, everyone uses the same style and some common issues can be caught
-early on. See below for installation instructions.
+early on.
 
 Once installed, hooks will run when you commit. If the formatting isn't
 quite right or a linter catches something, the commit will be rejected.
@@ -51,129 +51,110 @@ as the Python formatting tool `black`, will format failing
 files, so all you need to do is `git add` those files again
 and retry your commit.
 
-## Initial setup and first start
+## General setup
 
-After you forked and cloned the code from github you need to perform a
-first-time setup. To do the setup you need to perform the steps from the
-following chapters in a certain order:
+After you forked and cloned the code from GitHub you need to perform a
+first-time setup.
+
+!!! note
+
+    Every command is executed directly from the root folder of the project unless specified otherwise.
 
 1.  Install prerequisites + pipenv as mentioned in
-    [Bare metal route](/setup#bare_metal)
+    [Bare metal route](/setup#bare_metal).
 
 2.  Copy `paperless.conf.example` to `paperless.conf` and enable debug
-    mode.
+    mode within the file via `PAPERLESS_DEBUG=true`.
 
-3.  Install the Angular CLI interface:
+3.  Create `consume` and `media` directories:
 
-    ```shell-session
-    $ npm install -g @angular/cli
+    ```bash
+    $ mkdir -p consume media
     ```
 
-4.  Install pre-commit hooks
+4.  Install the Python dependencies:
 
-    ```shell-session
-    pre-commit install
+    ```bash
+    $ pipenv install --dev
     ```
 
-5.  Create `consume` and `media` folders in the cloned root folder.
+    !!! note
 
-    ```shell-session
-    mkdir -p consume media
+        Using a virtual environment is highly recommended. You can spawn one via `pipenv shell`.
+        Make sure you're using Python 3.10.x or lower. Otherwise you might
+        get issues with building dependencies. You can use
+        [pyenv](https://github.com/pyenv/pyenv) to install a specific
+        Python version.
+
+5.  Install pre-commit hooks:
+
+    ```bash
+    $ pre-commit install
     ```
 
-6.  You can now either ...
+6.  Apply migrations and create a superuser for your development instance:
+
+    ```bash
+    # src/
+
+    $ python3 manage.py migrate
+    $ python3 manage.py createsuperuser
+    ```
+
+7.  You can now either ...
 
     - install redis or
 
-    - use the included scripts/start-services.sh to use docker to fire
+    - use the included `scripts/start_services.sh` to use docker to fire
       up a redis instance (and some other services such as tika,
       gotenberg and a database server) or
 
     - spin up a bare redis container
 
-      ```shell-session
-      docker run -d -p 6379:6379 --restart unless-stopped redis:latest
+      ```
+      $ docker run -d -p 6379:6379 --restart unless-stopped redis:latest
       ```
 
-7.  Install the python dependencies by performing in the src/ directory.
-
-    ```shell-session
-    pipenv install --dev
-    ```
-
-!!! note
-
-    Make sure you're using python 3.10.x or lower. Otherwise you might
-    get issues with building dependencies. You can use
-    [pyenv](https://github.com/pyenv/pyenv) to install a specific
-    python version.
-
-8.  Generate the static UI so you can perform a login to get session
-    that is required for frontend development (this needs to be done one
-    time only). From src-ui directory:
-
-    ```shell-session
-    npm install .
-    ./node_modules/.bin/ng build --configuration production
-    ```
-
-9.  Apply migrations and create a superuser for your dev instance:
-
-    ```shell-session
-    python3 manage.py migrate
-    python3 manage.py createsuperuser
-    ```
-
-10. Now spin up the dev backend. Depending on which part of paperless
-    you're developing for, you need to have some or all of them
-    running.
-
-    ```shell-session
-    python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
-    ```
-
-11. Login with the superuser credentials provided in step 8 at
-    `http://localhost:8000` to create a session that enables you to use
-    the backend.
-
-Backend development environment is now ready, to start Frontend
-development go to `/src-ui` and run `ng serve`. From there you can use
-`http://localhost:4200` for a preview.
+8.  Continue with either back-end or front-end development – or both :-).
 
 ## Back end development
 
-The backend is a [Django](https://www.djangoproject.com/) application. PyCharm works well for development,
-but you can use whatever you want.
+The back end is a [Django](https://www.djangoproject.com/) application. [PyCharm](https://www.jetbrains.com/de-de/pycharm/) as well as [Visual Studio Code](https://code.visualstudio.com) work well for development, but you can use whatever you want.
 
-Configure the IDE to use the src/ folder as the base source folder.
+Configure the IDE to use the `src/`-folder as the base source folder.
 Configure the following launch configurations in your IDE:
 
 - `python3 manage.py runserver`
-- `celery --app paperless worker`
 - `python3 manage.py document_consumer`
+- `celery --app paperless worker -l DEBUG` (or any other log level)
 
 To start them all:
 
-```shell-session
-python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
+```bash
+# src/
+
+$ python3 manage.py runserver & \
+  python3 manage.py document_consumer & \
+  celery --app paperless worker -l DEBUG
 ```
 
-Testing and code style:
+You might need the front end to test your back end code. This assumes that you have AngularJS installed on your system. Go to the [Front end development](#front-end-development) section for further details. To build the front end once use this commmand:
+
+```bash
+# src-ui/
+
+$ npm install
+$ ng build --configuration production
+```
+
+### Testing
 
 - Run `pytest` in the `src/` directory to execute all tests. This also
-  generates a HTML coverage report. When runnings test, paperless.conf
-  is loaded as well. However: the tests rely on the default
+  generates a HTML coverage report. When runnings test, `paperless.conf`
+  is loaded as well. However, the tests rely on the default
   configuration. This is not ideal. But for now, make sure no settings
   except for DEBUG are overridden when testing.
 
-- Coding style is enforced by the Git pre-commit hooks. These will
-  ensure your code is formatted and do some linting when you do a `git commit`.
-
-- You can also run `black` manually to format your code
-
-- The `pre-commit` hooks will modify files and interact with each other.
-  It may take a couple of `git add`, `git commit` cycle to satisfy them.
-
 !!! note
 
       The line length rule E501 is generally useful for getting multiple
@@ -184,98 +165,98 @@ Testing and code style:
 
 ## Front end development
 
-The front end is built using Angular. In order to get started, you need
-`npm`. Install the Angular CLI interface with
+The front end is built using AngularJS. In order to get started, you need Node.js (version 14.15+) and
+`npm`.
 
-```shell-session
-$ npm install -g @angular/cli
-```
+!!! note
 
-and make sure that it's on your path. Next, in the src-ui/ directory,
-install the required dependencies of the project.
+    The following commands are all performed in the `src-ui`-directory. You will need a running back end (including an active session) to connect to the back end API. To spin it up refer to the commands under the section [above](#back-end-development).
 
-```shell-session
-$ npm install
-```
+1. Install the Angular CLI. You might need sudo privileges
+   to perform this command:
 
-You can launch a development server by running
+   ```bash
+   $ npm install -g @angular/cli
+   ```
 
-```shell-session
-$ ng serve
-```
+2. Make sure that it's on your path.
 
-This will automatically update whenever you save. However, in-place
-compilation might fail on syntax errors, in which case you need to
-restart it.
+3. Install all neccessary modules:
 
-By default, the development server is available on
-`http://localhost:4200/` and is configured to access the API at
-`http://localhost:8000/api/`, which is the default of the backend. If
-you enabled DEBUG on the back end, several security overrides for
-allowed hosts, CORS and X-Frame-Options are in place so that the front
-end behaves exactly as in production. This also relies on you being
-logged into the back end. Without a valid session, The front end will
-simply not work.
+   ```bash
+   $ npm install
+   ```
 
-Testing and code style:
+4. You can launch a development server by running:
 
-- The frontend code (.ts, .html, .scss) use `prettier` for code
+   ```bash
+   $ ng serve
+   ```
+
+   This will automatically update whenever you save. However, in-place
+   compilation might fail on syntax errors, in which case you need to
+   restart it.
+
+   By default, the development server is available on `http://localhost:4200/` and is configured to access the API at
+   `http://localhost:8000/api/`, which is the default of the backend. If you enabled `DEBUG` on the back end, several security overrides for allowed hosts, CORS and X-Frame-Options are in place so that the front end behaves exactly as in production.
+
+### Testing and code style
+
+- The front end code (.ts, .html, .scss) use `prettier` for code
   formatting via the Git `pre-commit` hooks which run automatically on
-  commit. See
-  [above](#code-formatting-with-pre-commit-hooks) for installation. You can also run this via cli with a
+  commit. See [above](#code-formatting-with-pre-commit-hooks) for installation instructions. You can also run this via the CLI with a
   command such as
 
-  ```shell-session
+  ```bash
   $ git ls-files -- '*.ts' | xargs pre-commit run prettier --files
   ```
 
-- Frontend testing uses jest and cypress. There is currently a need
-  for significantly more frontend tests. Unit tests and e2e tests,
+- Front end testing uses jest and cypress. There is currently a need
+  for significantly more front end tests. Unit tests and e2e tests,
   respectively, can be run non-interactively with:
 
-  ```shell-session
+  ```bash
   $ ng test
   $ npm run e2e:ci
   ```
 
-  Cypress also includes a UI which can be run from within the `src-ui`
-  directory with
+  - Cypress also includes a UI which can be run with:
 
-  ```shell-session
-  $ ./node_modules/.bin/cypress open
+    ```bash
+    $ ./node_modules/.bin/cypress open
+    ```
+
+- In order to build the front end and serve it as part of Django, execute:
+
+  ```bash
+  $ ng build --configuration production
   ```
 
-In order to build the front end and serve it as part of django, execute
-
-```shell-session
-$ ng build --configuration production
-```
-
-This will build the front end and put it in a location from which the
-Django server will serve it as static content. This way, you can verify
-that authentication is working.
+  This will build the front end and put it in a location from which the
+  Django server will serve it as static content. This way, you can verify
+  that authentication is working.
 
 ## Localization
 
-Paperless is available in many different languages. Since paperless
-consists both of a django application and an Angular front end, both
+Paperless-ngx is available in many different languages. Since Paperless-ngx
+consists both of a Django application and an AngularJS front end, both
 these parts have to be translated separately.
 
 ### Front end localization
 
-- The Angular front end does localization according to the [Angular
+- The AngularJS front end does localization according to the [Angular
   documentation](https://angular.io/guide/i18n).
 - The source language of the project is "en_US".
-- The source strings end up in the file "src-ui/messages.xlf".
+- The source strings end up in the file `src-ui/messages.xlf`.
 - The translated strings need to be placed in the
-  "src-ui/src/locale/" folder.
+  `src-ui/src/locale/` folder.
 - In order to extract added or changed strings from the source files,
   call `ng xi18n --ivy`.
 
 Adding new languages requires adding the translated files in the
-"src-ui/src/locale/" folder and adjusting a couple files.
+`src-ui/src/locale/` folder and adjusting a couple files.
 
-1.  Adjust "src-ui/angular.json":
+1.  Adjust `src-ui/angular.json`:
 
     ```json
     "i18n": {
@@ -292,7 +273,7 @@ Adding new languages requires adding the translated files in the
     ```
 
 2.  Add the language to the available options in
-    "src-ui/src/app/services/settings.service.ts":
+    `src-ui/src/app/services/settings.service.ts`:
 
     ```typescript
     getLanguageOptions(): LanguageOption[] {
@@ -313,7 +294,7 @@ Adding new languages requires adding the translated files in the
     and "yyyy".
 
 3.  Import and register the Angular data for this locale in
-    "src-ui/src/app/app.module.ts":
+    `src-ui/src/app/app.module.ts`:
 
     ```typescript
     import localeDe from '@angular/common/locales/de'
@@ -326,10 +307,10 @@ A majority of the strings that appear in the back end appear only when
 the admin is used. However, some of these are still shown on the front
 end (such as error messages).
 
-- The django application does localization according to the [django
+- The django application does localization according to the [Django
   documentation](https://docs.djangoproject.com/en/3.1/topics/i18n/translation/).
 - The source language of the project is "en_US".
-- Localization files end up in the folder "src/locale/".
+- Localization files end up in the folder `src/locale/`.
 - In order to extract strings from the application, call
   `python3 manage.py makemessages -l en_US`. This is important after
   making changes to translatable strings.
@@ -340,8 +321,8 @@ end (such as error messages).
   command.
 
 Adding new languages requires adding the translated files in the
-"src/locale/" folder and adjusting the file
-"src/paperless/settings.py" to include the new language:
+`src/locale/`-folder and adjusting the file
+`src/paperless/settings.py` to include the new language:
 
 ```python
 LANGUAGES = [
@@ -360,18 +341,27 @@ LANGUAGES = [
 The documentation is built using material-mkdocs, see their [documentation](https://squidfunk.github.io/mkdocs-material/reference/).
 If you want to build the documentation locally, this is how you do it:
 
-1.  Install python dependencies.
+1.  Have an active pipenv shell (`pipenv shell`) and install Python dependencies:
 
-    ```shell-session
-    $ cd /path/to/paperless
+    ```bash
     $ pipenv install --dev
     ```
 
 2.  Build the documentation
 
-    ```shell-session
-    $ cd /path/to/paperless
-    $ pipenv mkdocs build --config-file mkdocs.yml
+    ```bash
+    $ mkdocs build --config-file mkdocs.yml
+    ```
+
+    _alternatively..._
+
+3.  Serve the documentation. This will spin up a
+    copy of the documentation at http://127.0.0.1:8000
+    that will automatically refresh everytime you change
+    something.
+
+    ```bash
+    $ mkdocs serve
     ```
 
 ## Building the Docker image
@@ -384,35 +374,35 @@ helper script `build-docker-image.sh`.
 
 Building the docker image from source:
 
-```shell-session
+```bash
 ./build-docker-image.sh Dockerfile -t <your-tag>
 ```
 
-## Extending Paperless
+## Extending Paperless-ngx
 
-Paperless does not have any fancy plugin systems and will probably never
+Paperless-ngx does not have any fancy plugin systems and will probably never
 have. However, some parts of the application have been designed to allow
 easy integration of additional features without any modification to the
 base code.
 
 ### Making custom parsers
 
-Paperless uses parsers to add documents to paperless. A parser is
+Paperless-ngx uses parsers to add documents. A parser is
 responsible for:
 
-- Retrieve the content from the original
-- Create a thumbnail
-- Optional: Retrieve a created date from the original
-- Optional: Create an archived document from the original
+- Retrieving the content from the original
+- Creating a thumbnail
+- _optional:_ Retrieving a created date from the original
+- _optional:_ Creainge an archived document from the original
 
-Custom parsers can be added to paperless to support more file types. In
+Custom parsers can be added to Paperless-ngx to support more file types. In
 order to do that, you need to write the parser itself and announce its
-existence to paperless.
+existence to Paperless-ngx.
 
 The parser itself must extend `documents.parsers.DocumentParser` and
 must implement the methods `parse` and `get_thumbnail`. You can provide
 your own implementation to `get_date` if you don't want to rely on
-paperless' default date guessing mechanisms.
+Paperless-ngx' default date guessing mechanisms.
 
 ```python
 class MyCustomParser(DocumentParser):
@@ -444,7 +434,7 @@ to be empty and removed after consumption finished. You can use that
 directory to store any intermediate files and also use it to store the
 thumbnail / archived document.
 
-After that, you need to announce your parser to paperless. You need to
+After that, you need to announce your parser to Paperless-ngx. You need to
 connect a handler to the `document_consumer_declaration` signal. Have a
 look in the file `src/paperless_tesseract/apps.py` on how that's done.
 The handler is a method that returns information about your parser:
@@ -464,11 +454,11 @@ def myparser_consumer_declaration(sender, **kwargs):
 - `parser` is a reference to a class that extends `DocumentParser`.
 - `weight` is used whenever two or more parsers are able to parse a
   file: The parser with the higher weight wins. This can be used to
-  override the parsers provided by paperless.
+  override the parsers provided by Paperless-ngx.
 - `mime_types` is a dictionary. The keys are the mime types your
   parser supports and the value is the default file extension that
-  paperless should use when storing files and serving them for
+  Paperless-ngx should use when storing files and serving them for
   download. We could guess that from the file extensions, but some
-  mime types have many extensions associated with them and the python
+  mime types have many extensions associated with them and the Python
   methods responsible for guessing the extension do not always return
   the same value.

docs/setup.md

@@ -388,12 +388,7 @@ supported.
     ```
 
 8.  Install python requirements from the `requirements.txt` file. It is
-    up to you if you wish to use a virtual environment or not. First you
-    should update your pip, so it gets the actual packages.
-
-    ```shell-session
-    sudo -Hu paperless pip3 install --upgrade pip
-    ```
+    up to you if you wish to use a virtual environment or not. First you should update your pip, so it gets the actual packages.
 
     ```shell-session
     sudo -Hu paperless pip3 install -r requirements.txt
@@ -713,6 +708,12 @@ below use PostgreSQL, but are applicable to MySQL/MariaDB with the
     MySQL also enforces limits on maximum lengths, but does so differently than
     PostgreSQL.  It may not be possible to migrate to MySQL due to this.
 
+!!! warning
+
+    Using mariadb version 10.4+ is recommended. Using the `utf8mb3` character set on
+    an older system may fix issues that can arise while setting up Paperless-ngx but
+    `utf8mb3` can cause issues with consumption (where `utf8mb4` does not).
+
 1.  Stop paperless, if it is running.
 
 2.  Tell paperless to use PostgreSQL:

install-paperless-ngx.sh

@@ -321,7 +321,7 @@ fi
 wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/docker-compose.$DOCKER_COMPOSE_VERSION.yml" -O docker-compose.yml
 wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/.env" -O .env
 
-SECRET_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1)
+SECRET_KEY=$(tr --delete --complement 'a-zA-Z0-9' < /dev/urandom 2>/dev/null | head --bytes 64)
 
 DEFAULT_LANGUAGES=("deu eng fra ita spa")
 
@@ -346,7 +346,7 @@ read -r -a OCR_LANGUAGES_ARRAY <<< "${_split_langs}"
 	fi
 } > docker-compose.env
 
-sed -i "s/- 8000:8000/- $PORT:8000/g" docker-compose.yml
+sed -i "s/- \"8000:8000\"/- \"$PORT:8000\"/g" docker-compose.yml
 
 sed -i "s#- \./consume:/usr/src/paperless/consume#- $CONSUME_FOLDER:/usr/src/paperless/consume#g" docker-compose.yml
 

mkdocs.yml

@@ -41,6 +41,7 @@ markdown_extensions:
       anchor_linenums: true
   - pymdownx.superfences
   - pymdownx.inlinehilite
+  - pymdownx.snippets
 strict: true
 nav:
     - index.md
@@ -54,7 +55,7 @@ nav:
     - 'FAQs': faq.md
     - troubleshooting.md
     - changelog.md
-copyright: Copyright © 2016 - 2022 Daniel Quinn, Jonas Winkler, and the Paperless-ngx team
+copyright: Copyright © 2016 - 2023 Daniel Quinn, Jonas Winkler, and the Paperless-ngx team
 extra:
   social:
     - icon: fontawesome/brands/github

scripts/start_services.sh

@@ -2,5 +2,5 @@
 
 docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
-docker run -p 3000:3000 -d gotenberg/gotenberg:7.6 gotenberg --chromium-disable-javascript=true --chromium-allow-list="file:///tmp/.*"
+docker run -p 3000:3000 -d gotenberg/gotenberg:7.8 gotenberg --chromium-disable-javascript=true --chromium-allow-list="file:///tmp/.*"
 docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/angular.json

@@ -192,7 +192,8 @@
   "cli": {
     "schematicCollections": [
       "@angular-eslint/schematics"
-    ]
+    ],
+    "analytics": false
   },
   "schematics": {
     "@angular-eslint/schematics:application": {

src-ui/package.json

@@ -13,14 +13,14 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/common": "~15.1.0",
-    "@angular/compiler": "~15.1.0",
-    "@angular/core": "~15.1.0",
-    "@angular/forms": "~15.1.0",
-    "@angular/localize": "~15.1.0",
-    "@angular/platform-browser": "~15.1.0",
-    "@angular/platform-browser-dynamic": "~15.1.0",
-    "@angular/router": "~15.1.0",
+    "@angular/common": "~15.1.2",
+    "@angular/compiler": "~15.1.2",
+    "@angular/core": "~15.1.2",
+    "@angular/forms": "~15.1.2",
+    "@angular/localize": "~15.1.2",
+    "@angular/platform-browser": "~15.1.2",
+    "@angular/platform-browser-dynamic": "~15.1.2",
+    "@angular/router": "~15.1.2",
     "@ng-bootstrap/ng-bootstrap": "^14.0.1",
     "@ng-select/ng-select": "^10.0.1",
     "@ngneat/dirty-check-forms": "^3.0.3",
@@ -39,18 +39,18 @@
   },
   "devDependencies": {
     "@angular-builders/jest": "15.0.0",
-    "@angular-devkit/build-angular": "~15.1.0",
-    "@angular-eslint/builder": "15.1.0",
-    "@angular-eslint/eslint-plugin": "15.1.0",
-    "@angular-eslint/eslint-plugin-template": "15.1.0",
-    "@angular-eslint/schematics": "15.1.0",
-    "@angular-eslint/template-parser": "15.1.0",
-    "@angular/cli": "~15.1.0",
-    "@angular/compiler-cli": "~15.1.0",
+    "@angular-devkit/build-angular": "~15.1.4",
+    "@angular-eslint/builder": "15.2.0",
+    "@angular-eslint/eslint-plugin": "15.2.0",
+    "@angular-eslint/eslint-plugin-template": "15.2.0",
+    "@angular-eslint/schematics": "15.2.0",
+    "@angular-eslint/template-parser": "15.2.0",
+    "@angular/cli": "~15.1.4",
+    "@angular/compiler-cli": "~15.1.2",
     "@types/jest": "28.1.6",
     "@types/node": "^18.7.23",
     "@typescript-eslint/eslint-plugin": "^5.43.0",
-    "@typescript-eslint/parser": "^5.43.0",
+    "@typescript-eslint/parser": "^5.50.0",
     "concurrently": "7.4.0",
     "eslint": "^8.31.0",
     "jest": "28.1.3",

src-ui/src/app/components/common/input/tags/tags.component.html

@@ -17,7 +17,7 @@
       (blur)="onBlur()">
 
       <ng-template ng-label-tmp let-item="item">
-        <span class="tag-wrap tag-wrap-delete" (click)="removeTag(item.id)">
+        <span class="tag-wrap tag-wrap-delete" (mousedown)="removeTag($event, item.id)">
           <svg width="1.2em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
             <use xlink:href="assets/bootstrap-icons.svg#x"/>
           </svg>

src-ui/src/app/components/common/input/tags/tags.component.ts

@@ -65,7 +65,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
 
   private _lastSearchTerm: string
 
-  getTag(id) {
+  getTag(id: number) {
     if (this.tags) {
       return this.tags.find((tag) => tag.id == id)
     } else {
@@ -73,7 +73,10 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
     }
   }
 
-  removeTag(id) {
+  removeTag(event: PointerEvent, id: number) {
+    // prevent opening dropdown
+    event.stopImmediatePropagation()
+
     let index = this.value.indexOf(id)
     if (index > -1) {
       let oldValue = this.value

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -63,7 +63,7 @@
 
 
 <div class="row">
-    <div class="col mb-4">
+    <div class="col-md-6 col-xl-4 mb-4">
 
         <form [formGroup]='documentForm' (ngSubmit)="save()">
 

src-ui/src/app/components/document-detail/document-detail.component.scss

@@ -22,6 +22,15 @@
   --page-margin: 1px 0 20px;
 }
 
+::ng-deep .ng-select-taggable {
+  max-width: calc(100% - 46px); // fudge factor for ng-select button width
+}
+
+.btn-group .dropdown-toggle-split {
+  border-top-right-radius: inherit;
+  border-bottom-right-radius: inherit;
+}
+
 .password-prompt {
   position: absolute;
   top: 30%;

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -204,6 +204,10 @@ export class DocumentDetailComponent
             )
             .subscribe({
               next: (titleValue) => {
+                // In the rare case when the field changed just after debounced event was fired.
+                // We dont want to overwrite whats actually in the text field, so just return
+                if (titleValue !== this.titleInput.value) return
+
                 this.title = titleValue
                 this.documentForm.patchValue({ title: titleValue })
               },

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html

@@ -26,11 +26,11 @@
         </div>
         <p class="card-text">
           <span *ngIf="document.__search_hit__ && document.__search_hit__.highlights" [innerHtml]="document.__search_hit__.highlights"></span>
-          <span *ngIf="document.__search_hit__ && document.__search_hit__.comment_highlights">
+          <span *ngFor="let highlight of searchCommentHighlights" class="d-block">
             <svg width="1em" height="1em" fill="currentColor" class="me-2">
               <use xlink:href="assets/bootstrap-icons.svg#chat-left-text"/>
             </svg>
-            <span [innerHtml]="document.__search_hit__.comment_highlights"></span>
+            <span [innerHtml]="highlight"></span>
           </span>
           <span *ngIf="!document.__search_hit__" class="result-content">{{contentTrimmed}}</span>
         </p>

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts

@@ -70,6 +70,22 @@ export class DocumentCardLargeComponent {
     }
   }
 
+  get searchCommentHighlights() {
+    let highlights = []
+    if (
+      this.document['__search_hit__'] &&
+      this.document['__search_hit__'].comment_highlights
+    ) {
+      // only show comments with a match
+      highlights = (
+        this.document['__search_hit__'].comment_highlights as string
+      )
+        .split(',')
+        .filter((higlight) => higlight.includes('<span'))
+    }
+    return highlights
+  }
+
   getIsThumbInverted() {
     return this.settingsService.get(SETTINGS_KEYS.DARK_MODE_THUMB_INVERTED)
   }

src-ui/src/app/components/manage/management-list/management-list.component.html

@@ -10,7 +10,7 @@
     </div>
   </div>
 
-  <ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
+  <ngb-pagination class="col-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" [maxSize]="5" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
 </div>
 
 <table class="table table-striped align-middle border shadow-sm">
@@ -72,5 +72,5 @@
 
 <div class="d-flex">
   <div i18n *ngIf="collectionSize > 0">{collectionSize, plural, =1 {One {{typeName}}} other {{{collectionSize || 0}} total {{typeNamePlural}}}}</div>
-  <ngb-pagination *ngIf="collectionSize > 20" class="ms-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
+  <ngb-pagination *ngIf="collectionSize > 20" class="ms-auto" [pageSize]="25" [collectionSize]="collectionSize" [(page)]="page" [maxSize]="5" (pageChange)="reloadData()" aria-label="Default pagination"></ngb-pagination>
 </div>

src-ui/src/app/components/manage/settings/settings.component.html

@@ -143,7 +143,7 @@
             <p i18n>
               <em>No tracking data is collected by the app in any way.</em>
             </p>
-            <app-input-check i18n-title title="Enable update checking" formControlName="updateCheckingEnabled" i18n-hint hint="Note that for users of thirdy-party containers e.g. linuxserver.io this notification may be 'ahead' of the current third-party release."></app-input-check>
+            <app-input-check i18n-title title="Enable update checking" formControlName="updateCheckingEnabled" i18n-hint hint="Note that for users of third-party containers e.g. linuxserver.io this notification may be 'ahead' of the current third-party release."></app-input-check>
           </div>
         </div>
 
@@ -189,6 +189,14 @@
       <a ngbNavLink i18n>Saved views</a>
       <ng-template ngbNavContent>
 
+        <h4 i18n>Settings</h4>
+        <div class="row mb-3">
+          <div class="offset-md-3 col">
+            <app-input-check i18n-title title="Show warning when closing saved views with unsaved changes" formControlName="savedViewsWarnOnUnsavedChange"></app-input-check>
+          </div>
+        </div>
+
+        <h4 i18n>Views</h4>
         <div formGroupName="savedViews">
 
             <div *ngFor="let view of savedViews" [formGroupName]="view.id" class="row">

src-ui/src/app/components/manage/settings/settings.component.ts

@@ -84,6 +84,7 @@ export class SettingsComponent
     notificationsConsumerFailed: new FormControl(null),
     notificationsConsumerSuppressOnDashboard: new FormControl(null),
 
+    savedViewsWarnOnUnsavedChange: new FormControl(null),
     savedViews: this.savedViewGroup,
 
     mailAccounts: this.mailAccountGroup,
@@ -194,6 +195,9 @@ export class SettingsComponent
       notificationsConsumerSuppressOnDashboard: this.settings.get(
         SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD
       ),
+      savedViewsWarnOnUnsavedChange: this.settings.get(
+        SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE
+      ),
       savedViews: {},
       mailAccounts: {},
       mailRules: {},
@@ -462,6 +466,10 @@ export class SettingsComponent
       SETTINGS_KEYS.UPDATE_CHECKING_ENABLED,
       this.settingsForm.value.updateCheckingEnabled
     )
+    this.settings.set(
+      SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE,
+      this.settingsForm.value.savedViewsWarnOnUnsavedChange
+    )
     this.settings.setLanguage(this.settingsForm.value.displayLanguage)
     this.settings
       .storeSettings()

src-ui/src/app/data/paperless-uisettings.ts

@@ -41,6 +41,8 @@ export const SETTINGS_KEYS = {
   UPDATE_CHECKING_ENABLED: 'general-settings:update-checking:enabled',
   UPDATE_CHECKING_BACKEND_SETTING:
     'general-settings:update-checking:backend-setting',
+  SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE:
+    'general-settings:saved-views:warn-on-unsaved-change',
 }
 
 export const SETTINGS: PaperlessUiSetting[] = [
@@ -139,4 +141,9 @@ export const SETTINGS: PaperlessUiSetting[] = [
     type: 'string',
     default: '',
   },
+  {
+    key: SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE,
+    type: 'boolean',
+    default: true,
+  },
 ]

src-ui/src/app/guards/dirty-saved-view.guard.ts

@@ -4,17 +4,25 @@ import { first, Observable, Subject } from 'rxjs'
 import { DocumentListComponent } from '../components/document-list/document-list.component'
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap'
 import { ConfirmDialogComponent } from '../components/common/confirm-dialog/confirm-dialog.component'
+import { SettingsService } from '../services/settings.service'
+import { SETTINGS_KEYS } from '../data/paperless-uisettings'
 
 @Injectable()
 export class DirtySavedViewGuard
   implements CanDeactivate<DocumentListComponent>
 {
-  constructor(private modalService: NgbModal) {}
+  constructor(
+    private modalService: NgbModal,
+    private settings: SettingsService
+  ) {}
 
   canDeactivate(
     component: DocumentListComponent
   ): boolean | Observable<boolean> {
-    return component.savedViewIsModified ? this.warn(component) : true
+    return component.savedViewIsModified &&
+      this.settings.get(SETTINGS_KEYS.SAVED_VIEWS_WARN_ON_UNSAVED_CHANGE)
+      ? this.warn(component)
+      : true
   }
 
   warn(component: DocumentListComponent) {

src-ui/src/environments/environment.prod.ts

@@ -5,7 +5,7 @@ export const environment = {
   apiBaseUrl: document.baseURI + 'api/',
   apiVersion: '2',
   appTitle: 'Paperless-ngx',
-  version: '1.12.0',
+  version: '1.13.0',
   webSocketHost: window.location.host,
   webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
   webSocketBaseUrl: base_url.pathname + 'ws/',

src/documents/barcodes.py

@@ -4,18 +4,17 @@ import shutil
 import tempfile
 from dataclasses import dataclass
 from functools import lru_cache
-from math import ceil
 from pathlib import Path
+from typing import Dict
 from typing import List
 from typing import Optional
 
 import magic
 from django.conf import settings
 from pdf2image import convert_from_path
+from pdf2image.exceptions import PDFPageCountError
 from pikepdf import Page
-from pikepdf import PasswordError
 from pikepdf import Pdf
-from pikepdf import PdfImage
 from PIL import Image
 from PIL import ImageSequence
 from pyzbar import pyzbar
@@ -154,52 +153,15 @@ def scan_file_for_barcodes(
     (page_number, barcode_text) tuples
     """
 
-    def _pikepdf_barcode_scan(pdf_filepath: str) -> List[Barcode]:
-        detected_barcodes = []
-        with Pdf.open(pdf_filepath) as pdf:
-            for page_num, page in enumerate(pdf.pages):
-                for image_key in page.images:
-                    pdfimage = PdfImage(page.images[image_key])
-
-                    # This type is known to have issues:
-                    # https://github.com/pikepdf/pikepdf/issues/401
-                    if "/CCITTFaxDecode" in pdfimage.filters:
-                        raise BarcodeImageFormatError(
-                            "Unable to decode CCITTFaxDecode images",
-                        )
-
-                    # Not all images can be transcoded to a PIL image, which
-                    # is what pyzbar expects to receive, so this may
-                    # raise an exception, triggering fallback
-                    pillow_img = pdfimage.as_pil_image()
-
-                    # Scale the image down
-                    # See: https://github.com/paperless-ngx/paperless-ngx/issues/2385
-                    # TLDR: zbar has issues with larger images
-                    width, height = pillow_img.size
-                    if width > 1024:
-                        scaler = ceil(width / 1024)
-                        new_width = int(width / scaler)
-                        new_height = int(height / scaler)
-                        pillow_img = pillow_img.resize((new_width, new_height))
-
-                    width, height = pillow_img.size
-                    if height > 2048:
-                        scaler = ceil(height / 2048)
-                        new_width = int(width / scaler)
-                        new_height = int(height / scaler)
-                        pillow_img = pillow_img.resize((new_width, new_height))
-
-                    for barcode_value in barcode_reader(pillow_img):
-                        detected_barcodes.append(Barcode(page_num, barcode_value))
-
-        return detected_barcodes
-
     def _pdf2image_barcode_scan(pdf_filepath: str) -> List[Barcode]:
         detected_barcodes = []
         # use a temporary directory in case the file is too big to handle in memory
         with tempfile.TemporaryDirectory() as path:
-            pages_from_path = convert_from_path(pdf_filepath, output_folder=path)
+            pages_from_path = convert_from_path(
+                pdf_filepath,
+                dpi=300,
+                output_folder=path,
+            )
             for current_page_number, page in enumerate(pages_from_path):
                 for barcode_value in barcode_reader(page):
                     detected_barcodes.append(
@@ -219,27 +181,19 @@ def scan_file_for_barcodes(
         # Always try pikepdf first, it's usually fine, faster and
         # uses less memory
         try:
-            barcodes = _pikepdf_barcode_scan(pdf_filepath)
+            barcodes = _pdf2image_barcode_scan(pdf_filepath)
         # Password protected files can't be checked
-        except PasswordError as e:
+        # This is the exception raised for those
+        except PDFPageCountError as e:
             logger.warning(
                 f"File is likely password protected, not checking for barcodes: {e}",
             )
-        # Handle pikepdf related image decoding issues with a fallback to page
-        # by page conversion to images in a temporary directory
-        except Exception as e:
+        # This file is really borked, allow the consumption to continue
+        # but it may fail further on
+        except Exception as e:  # pragma: no cover
             logger.warning(
-                f"Falling back to pdf2image because: {e}",
+                f"Exception during barcode scanning: {e}",
             )
-            try:
-                barcodes = _pdf2image_barcode_scan(pdf_filepath)
-            # This file is really borked, allow the consumption to continue
-            # but it may fail further on
-            except Exception as e:  # pragma: no cover
-                logger.warning(
-                    f"Exception during barcode scanning: {e}",
-                )
-
     else:
         logger.warning(
             f"Unsupported file format for barcode reader: {str(mime_type)}",
@@ -248,16 +202,25 @@ def scan_file_for_barcodes(
     return DocumentBarcodeInfo(pdf_filepath, barcodes)
 
 
-def get_separating_barcodes(barcodes: List[Barcode]) -> List[int]:
+def get_separating_barcodes(barcodes: List[Barcode]) -> Dict[int, bool]:
     """
     Search the parsed barcodes for separators
-    and returns a list of page numbers, which
-    separate the file into new files.
+    and returns a dict of page numbers, which
+    separate the file into new files, together
+    with the information whether to keep the page.
     """
     # filter all barcodes for the separator string
     # get the page numbers of the separating barcodes
+    separator_pages = {bc.page: False for bc in barcodes if bc.is_separator}
+    if not settings.CONSUMER_ENABLE_ASN_BARCODE:
+        return separator_pages
 
-    return list({bc.page for bc in barcodes if bc.is_separator})
+    # add the page numbers of the ASN barcodes
+    # (except for first page, that might lead to infinite loops).
+    return {
+        **separator_pages,
+        **{bc.page: True for bc in barcodes if bc.is_asn and bc.page != 0},
+    }
 
 
 def get_asn_from_barcodes(barcodes: List[Barcode]) -> Optional[int]:
@@ -289,10 +252,11 @@ def get_asn_from_barcodes(barcodes: List[Barcode]) -> Optional[int]:
     return asn
 
 
-def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]:
+def separate_pages(filepath: str, pages_to_split_on: Dict[int, bool]) -> List[str]:
     """
     Separate the provided pdf file on the pages_to_split_on.
-    The pages which are defined by page_numbers will be removed.
+    The pages which are defined by the keys in page_numbers
+    will be removed if the corresponding value is false.
     Returns a list of (temporary) filepaths to consume.
     These will need to be deleted later.
     """
@@ -308,26 +272,28 @@ def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]:
     fname = os.path.splitext(os.path.basename(filepath))[0]
     pdf = Pdf.open(filepath)
 
+    # Start with an empty document
+    current_document: List[Page] = []
     # A list of documents, ie a list of lists of pages
-    documents: List[List[Page]] = []
-    # A single document, ie a list of pages
-    document: List[Page] = []
+    documents: List[List[Page]] = [current_document]
 
     for idx, page in enumerate(pdf.pages):
         # Keep building the new PDF as long as it is not a
         # separator index
         if idx not in pages_to_split_on:
-            document.append(page)
-            # Make sure to append the very last document to the documents
-            if idx == (len(pdf.pages) - 1):
-                documents.append(document)
-                document = []
-        else:
-            # This is a split index, save the current PDF pages, and restart
-            # a new destination page listing
-            logger.debug(f"Starting new document at idx {idx}")
-            documents.append(document)
-            document = []
+            current_document.append(page)
+            continue
+
+        # This is a split index
+        # Start a new destination page listing
+        logger.debug(f"Starting new document at idx {idx}")
+        current_document = []
+        documents.append(current_document)
+        keep_page = pages_to_split_on[idx]
+        if keep_page:
+            # Keep the page
+            # (new document is started by asn barcode)
+            current_document.append(page)
 
     documents = [x for x in documents if len(x)]
 
@@ -359,11 +325,10 @@ def save_to_dir(
     Optionally rename the file.
     """
     if os.path.isfile(filepath) and os.path.isdir(target_dir):
-        dst = shutil.copy(filepath, target_dir)
-        logging.debug(f"saved {str(filepath)} to {str(dst)}")
-        if newname:
-            dst_new = os.path.join(target_dir, newname)
-            logger.debug(f"moving {str(dst)} to {str(dst_new)}")
-            os.rename(dst, dst_new)
+        dest = target_dir
+        if newname is not None:
+            dest = os.path.join(dest, newname)
+        shutil.copy(filepath, dest)
+        logging.debug(f"saved {str(filepath)} to {str(dest)}")
     else:
         logger.warning(f"{str(filepath)} or {str(target_dir)} don't exist.")

src/documents/consumer.py

@@ -1,7 +1,10 @@
 import datetime
 import hashlib
 import os
+import shutil
+import tempfile
 import uuid
+from pathlib import Path
 from subprocess import CompletedProcess
 from subprocess import run
 from typing import Optional
@@ -94,7 +97,8 @@ class Consumer(LoggingMixin):
 
     def __init__(self):
         super().__init__()
-        self.path = None
+        self.path: Optional[Path] = None
+        self.original_path: Optional[Path] = None
         self.filename = None
         self.override_title = None
         self.override_correspondent_id = None
@@ -142,11 +146,16 @@ class Consumer(LoggingMixin):
             return
         # Validate the range is above zero and less than uint32_t max
         # otherwise, Whoosh can't handle it in the index
-        if self.override_asn < 0 or self.override_asn > 0xFF_FF_FF_FF:
+        if (
+            self.override_asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
+            or self.override_asn > Document.ARCHIVE_SERIAL_NUMBER_MAX
+        ):
             self._fail(
                 MESSAGE_ASN_RANGE,
                 f"Not consuming {self.filename}: "
-                f"Given ASN {self.override_asn} is out of range [0, 4,294,967,295]",
+                f"Given ASN {self.override_asn} is out of range "
+                f"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, "
+                f"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}]",
             )
         if Document.objects.filter(archive_serial_number=self.override_asn).exists():
             self._fail(
@@ -167,16 +176,18 @@ class Consumer(LoggingMixin):
 
         self.log("info", f"Executing pre-consume script {settings.PRE_CONSUME_SCRIPT}")
 
-        filepath_arg = os.path.normpath(self.path)
+        working_file_path = str(self.path)
+        original_file_path = str(self.original_path)
 
         script_env = os.environ.copy()
-        script_env["DOCUMENT_SOURCE_PATH"] = filepath_arg
+        script_env["DOCUMENT_SOURCE_PATH"] = original_file_path
+        script_env["DOCUMENT_WORKING_PATH"] = working_file_path
 
         try:
             completed_proc = run(
                 args=[
                     settings.PRE_CONSUME_SCRIPT,
-                    filepath_arg,
+                    original_file_path,
                 ],
                 env=script_env,
                 capture_output=True,
@@ -195,7 +206,7 @@ class Consumer(LoggingMixin):
                 exception=e,
             )
 
-    def run_post_consume_script(self, document):
+    def run_post_consume_script(self, document: Document):
         if not settings.POST_CONSUME_SCRIPT:
             return
 
@@ -285,8 +296,8 @@ class Consumer(LoggingMixin):
         Return the document object if it was successfully created.
         """
 
-        self.path = path
-        self.filename = override_filename or os.path.basename(path)
+        self.path = Path(path).resolve()
+        self.filename = override_filename or self.path.name
         self.override_title = override_title
         self.override_correspondent_id = override_correspondent_id
         self.override_document_type_id = override_document_type_id
@@ -311,6 +322,15 @@ class Consumer(LoggingMixin):
 
         self.log("info", f"Consuming {self.filename}")
 
+        # For the actual work, copy the file into a tempdir
+        self.original_path = self.path
+        tempdir = tempfile.TemporaryDirectory(
+            prefix="paperless-ngx",
+            dir=settings.SCRATCH_DIR,
+        )
+        self.path = Path(tempdir.name) / Path(self.filename)
+        shutil.copy(self.original_path, self.path)
+
         # Determine the parser class.
 
         mime_type = magic.from_file(self.path, mime=True)
@@ -322,6 +342,7 @@ class Consumer(LoggingMixin):
             mime_type,
         )
         if not parser_class:
+            tempdir.cleanup()
             self._fail(MESSAGE_UNSUPPORTED_TYPE, f"Unsupported mime type {mime_type}")
 
         # Notify all listeners that we're going to do some work.
@@ -380,6 +401,7 @@ class Consumer(LoggingMixin):
 
         except ParseError as e:
             document_parser.cleanup()
+            tempdir.cleanup()
             self._fail(
                 str(e),
                 f"Error while consuming document {self.filename}: {e}",
@@ -453,11 +475,12 @@ class Consumer(LoggingMixin):
                 # Delete the file only if it was successfully consumed
                 self.log("debug", f"Deleting file {self.path}")
                 os.unlink(self.path)
+                self.original_path.unlink()
 
                 # https://github.com/jonaswinkler/paperless-ng/discussions/1037
                 shadow_file = os.path.join(
-                    os.path.dirname(self.path),
-                    "._" + os.path.basename(self.path),
+                    os.path.dirname(self.original_path),
+                    "._" + os.path.basename(self.original_path),
                 )
 
                 if os.path.isfile(shadow_file):
@@ -474,6 +497,7 @@ class Consumer(LoggingMixin):
             )
         finally:
             document_parser.cleanup()
+            tempdir.cleanup()
 
         self.run_post_consume_script(document)
 

src/documents/index.py

@@ -5,6 +5,7 @@ from contextlib import contextmanager
 
 from dateutil.parser import isoparse
 from django.conf import settings
+from django.utils import timezone
 from documents.models import Comment
 from documents.models import Document
 from whoosh import classify
@@ -89,10 +90,22 @@ def open_index_searcher():
         searcher.close()
 
 
-def update_document(writer, doc):
+def update_document(writer: AsyncWriter, doc: Document):
     tags = ",".join([t.name for t in doc.tags.all()])
     tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
     comments = ",".join([str(c.comment) for c in Comment.objects.filter(document=doc)])
+    asn = doc.archive_serial_number
+    if asn is not None and (
+        asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
+        or asn > Document.ARCHIVE_SERIAL_NUMBER_MAX
+    ):
+        logger.error(
+            f"Not indexing Archive Serial Number {asn} of document {doc.pk}. "
+            f"ASN is out of range "
+            f"[{Document.ARCHIVE_SERIAL_NUMBER_MIN:,}, "
+            f"{Document.ARCHIVE_SERIAL_NUMBER_MAX:,}.",
+        )
+        asn = 0
     writer.update_document(
         id=doc.pk,
         title=doc.title,
@@ -108,7 +121,7 @@ def update_document(writer, doc):
         has_type=doc.document_type is not None,
         created=doc.created,
         added=doc.added,
-        asn=doc.archive_serial_number,
+        asn=asn,
         modified=doc.modified,
         path=doc.storage_path.name if doc.storage_path else None,
         path_id=doc.storage_path.id if doc.storage_path else None,
@@ -262,7 +275,7 @@ class DelayedFullTextQuery(DelayedQuery):
             ["content", "title", "correspondent", "tag", "type", "comments"],
             self.searcher.ixreader.schema,
         )
-        qp.add_plugin(DateParserPlugin())
+        qp.add_plugin(DateParserPlugin(basedate=timezone.now()))
         q = qp.parse(q_str)
 
         corrected = self.searcher.correct_query(q, q_str)

src/documents/management/commands/document_consumer.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from fnmatch import filter
 from pathlib import Path
 from pathlib import PurePath
 from threading import Event
@@ -7,6 +8,7 @@ from threading import Thread
 from time import monotonic
 from time import sleep
 from typing import Final
+from typing import Set
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
@@ -25,15 +27,15 @@ except ImportError:  # pragma: nocover
 logger = logging.getLogger("paperless.management.consumer")
 
 
-def _tags_from_path(filepath):
-    """Walk up the directory tree from filepath to CONSUMPTION_DIR
-    and get or create Tag IDs for every directory.
+def _tags_from_path(filepath) -> Set[Tag]:
+    """
+    Walk up the directory tree from filepath to CONSUMPTION_DIR
+    and get or create Tag IDs for every directory.
+
+    Returns set of Tag models
     """
-    normalized_consumption_dir = os.path.abspath(
-        os.path.normpath(settings.CONSUMPTION_DIR),
-    )
     tag_ids = set()
-    path_parts = Path(filepath).relative_to(normalized_consumption_dir).parent.parts
+    path_parts = Path(filepath).relative_to(settings.CONSUMPTION_DIR).parent.parts
     for part in path_parts:
         tag_ids.add(
             Tag.objects.get_or_create(name__iexact=part, defaults={"name": part})[0].pk,
@@ -43,14 +45,41 @@ def _tags_from_path(filepath):
 
 
 def _is_ignored(filepath: str) -> bool:
-    normalized_consumption_dir = os.path.abspath(
-        os.path.normpath(settings.CONSUMPTION_DIR),
+    """
+    Checks if the given file should be ignored, based on configured
+    patterns.
+
+    Returns True if the file is ignored, False otherwise
+    """
+    filepath = os.path.abspath(
+        os.path.normpath(filepath),
     )
-    filepath_relative = PurePath(filepath).relative_to(normalized_consumption_dir)
-    return any(filepath_relative.match(p) for p in settings.CONSUMER_IGNORE_PATTERNS)
+
+    # Trim out the consume directory, leaving only filename and it's
+    # path relative to the consume directory
+    filepath_relative = PurePath(filepath).relative_to(settings.CONSUMPTION_DIR)
+
+    # March through the components of the path, including directories and the filename
+    # looking for anything matching
+    # foo/bar/baz/file.pdf -> (foo, bar, baz, file.pdf)
+    parts = []
+    for part in filepath_relative.parts:
+        # If the part is not the name (ie, it's a dir)
+        # Need to append the trailing slash or fnmatch doesn't match
+        # fnmatch("dir", "dir/*") == False
+        # fnmatch("dir/", "dir/*") == True
+        if part != filepath_relative.name:
+            part = part + "/"
+        parts.append(part)
+
+    for pattern in settings.CONSUMER_IGNORE_PATTERNS:
+        if len(filter(parts, pattern)):
+            return True
+
+    return False
 
 
-def _consume(filepath):
+def _consume(filepath: str) -> None:
     if os.path.isdir(filepath) or _is_ignored(filepath):
         return
 
@@ -103,7 +132,13 @@ def _consume(filepath):
         logger.exception("Error while consuming document")
 
 
-def _consume_wait_unmodified(file):
+def _consume_wait_unmodified(file: str) -> None:
+    """
+    Waits for the given file to appear unmodified based on file size
+    and modification time.  Will wait a configured number of seconds
+    and retry a configured number of times before either consuming or
+    giving up
+    """
     if _is_ignored(file):
         return
 

src/documents/management/commands/document_exporter.py

@@ -311,8 +311,8 @@ class Command(BaseCommand):
                 archive_target = None
 
             # 3.4. write files to target folder
-            t = int(time.mktime(document.created.timetuple()))
             if document.storage_type == Document.STORAGE_TYPE_GPG:
+                t = int(time.mktime(document.created.timetuple()))
 
                 original_target.parent.mkdir(parents=True, exist_ok=True)
                 with document.source_file as out_file:

src/documents/migrations/1030_alter_paperlesstask_task_file_name.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-02-03 21:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("documents", "1029_alter_document_archive_serial_number"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="paperlesstask",
+            name="task_file_name",
+            field=models.CharField(
+                help_text="Name of the file which the Task was run for",
+                max_length=255,
+                null=True,
+                verbose_name="Task Filename",
+            ),
+        ),
+    ]

src/documents/models.py

@@ -3,6 +3,7 @@ import logging
 import os
 import re
 from collections import OrderedDict
+from typing import Final
 from typing import Optional
 
 import dateutil.parser
@@ -229,6 +230,9 @@ class Document(models.Model):
         help_text=_("The original name of the file when it was uploaded"),
     )
 
+    ARCHIVE_SERIAL_NUMBER_MIN: Final[int] = 0
+    ARCHIVE_SERIAL_NUMBER_MAX: Final[int] = 0xFF_FF_FF_FF
+
     archive_serial_number = models.PositiveIntegerField(
         _("archive serial number"),
         blank=True,
@@ -236,8 +240,8 @@ class Document(models.Model):
         unique=True,
         db_index=True,
         validators=[
-            MaxValueValidator(0xFF_FF_FF_FF),
-            MinValueValidator(0),
+            MaxValueValidator(ARCHIVE_SERIAL_NUMBER_MAX),
+            MinValueValidator(ARCHIVE_SERIAL_NUMBER_MIN),
         ],
         help_text=_(
             "The position of this document in your physical document " "archive.",
@@ -555,7 +559,7 @@ class PaperlessTask(models.Model):
     task_file_name = models.CharField(
         null=True,
         max_length=255,
-        verbose_name=_("Task Name"),
+        verbose_name=_("Task Filename"),
         help_text=_("Name of the file which the Task was run for"),
     )
 

src/documents/serialisers.py

@@ -599,11 +599,17 @@ class StoragePathSerializer(MatchingModelSerializer):
                 document_type="document_type",
                 created="created",
                 created_year="created_year",
+                created_year_short="created_year_short",
                 created_month="created_month",
+                created_month_name="created_month_name",
+                created_month_name_short="created_month_name_short",
                 created_day="created_day",
                 added="added",
                 added_year="added_year",
+                added_year_short="added_year_short",
                 added_month="added_month",
+                added_month_name="added_month_name",
+                added_month_name_short="added_month_name_short",
                 added_day="added_day",
                 asn="asn",
                 tags="tags",

src/documents/tasks.py

@@ -128,6 +128,18 @@ def consume_file(
                 )
 
                 if document_list:
+
+                    # If the file is an upload, it's in the scratch directory
+                    # Move it to consume directory to be picked up
+                    # Otherwise, use the current parent to keep possible tags
+                    # from subdirectories
+                    try:
+                        # is_relative_to would be nicer, but new in 3.9
+                        _ = path.relative_to(settings.SCRATCH_DIR)
+                        save_to_dir = settings.CONSUMPTION_DIR
+                    except ValueError:
+                        save_to_dir = path.parent
+
                     for n, document in enumerate(document_list):
                         # save to consumption dir
                         # rename it to the original filename  with number prefix
@@ -136,23 +148,18 @@ def consume_file(
                         else:
                             newname = None
 
-                        # If the file is an upload, it's in the scratch directory
-                        # Move it to consume directory to be picked up
-                        # Otherwise, use the current parent to keep possible tags
-                        # from subdirectories
-                        try:
-                            # is_relative_to would be nicer, but new in 3.9
-                            _ = path.relative_to(settings.SCRATCH_DIR)
-                            save_to_dir = settings.CONSUMPTION_DIR
-                        except ValueError:
-                            save_to_dir = path.parent
-
                         barcodes.save_to_dir(
                             document,
                             newname=newname,
                             target_dir=save_to_dir,
                         )
 
+                        # Split file has been copied safely, remove it
+                        os.remove(document)
+
+                    # And clean up the directory as well, now it's empty
+                    shutil.rmtree(os.path.dirname(document_list[0]))
+
                     # Delete the PDF file which was split
                     os.remove(doc_barcode_info.pdf_path)
 
@@ -164,7 +171,7 @@ def consume_file(
                     # notify the sender, otherwise the progress bar
                     # in the UI stays stuck
                     payload = {
-                        "filename": override_filename,
+                        "filename": override_filename or path.name,
                         "task_id": task_id,
                         "current_progress": 100,
                         "max_progress": 100,

src/documents/tests/test_api.py

@@ -7,6 +7,7 @@ import tempfile
 import urllib.request
 import uuid
 import zipfile
+from datetime import timedelta
 from pathlib import Path
 from unittest import mock
 from unittest.mock import MagicMock
@@ -23,6 +24,7 @@ from django.conf import settings
 from django.contrib.auth.models import User
 from django.test import override_settings
 from django.utils import timezone
+from dateutil.relativedelta import relativedelta
 from documents import bulk_edit
 from documents import index
 from documents.models import Correspondent
@@ -119,28 +121,28 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         response = self.client.get("/api/documents/", format="json")
         self.assertEqual(response.status_code, 200)
         results_full = response.data["results"]
-        self.assertTrue("content" in results_full[0])
-        self.assertTrue("id" in results_full[0])
+        self.assertIn("content", results_full[0])
+        self.assertIn("id", results_full[0])
 
         response = self.client.get("/api/documents/?fields=id", format="json")
         self.assertEqual(response.status_code, 200)
         results = response.data["results"]
         self.assertFalse("content" in results[0])
-        self.assertTrue("id" in results[0])
+        self.assertIn("id", results[0])
         self.assertEqual(len(results[0]), 1)
 
         response = self.client.get("/api/documents/?fields=content", format="json")
         self.assertEqual(response.status_code, 200)
         results = response.data["results"]
-        self.assertTrue("content" in results[0])
+        self.assertIn("content", results[0])
         self.assertFalse("id" in results[0])
         self.assertEqual(len(results[0]), 1)
 
         response = self.client.get("/api/documents/?fields=id,content", format="json")
         self.assertEqual(response.status_code, 200)
         results = response.data["results"]
-        self.assertTrue("content" in results[0])
-        self.assertTrue("id" in results[0])
+        self.assertIn("content", results[0])
+        self.assertIn("id", results[0])
         self.assertEqual(len(results[0]), 2)
 
         response = self.client.get(
@@ -150,7 +152,7 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, 200)
         results = response.data["results"]
         self.assertFalse("content" in results[0])
-        self.assertTrue("id" in results[0])
+        self.assertIn("id", results[0])
         self.assertEqual(len(results[0]), 1)
 
         response = self.client.get("/api/documents/?fields=", format="json")
@@ -505,6 +507,270 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         response = self.client.get("/api/documents/?query=content&page=3&page_size=10")
         self.assertEqual(response.status_code, 404)
 
+    @override_settings(
+        TIME_ZONE="UTC",
+    )
+    def test_search_added_in_last_week(self):
+        """
+        GIVEN:
+            - Three documents added right now
+            - The timezone is UTC time
+        WHEN:
+            - Query for documents added in the last 7 days
+        THEN:
+            - All three recent documents are returned
+        """
+        d1 = Document.objects.create(
+            title="invoice",
+            content="the thing i bought at a shop and paid with bank account",
+            checksum="A",
+            pk=1,
+        )
+        d2 = Document.objects.create(
+            title="bank statement 1",
+            content="things i paid for in august",
+            pk=2,
+            checksum="B",
+        )
+        d3 = Document.objects.create(
+            title="bank statement 3",
+            content="things i paid for in september",
+            pk=3,
+            checksum="C",
+        )
+        with index.open_index_writer() as writer:
+            index.update_document(writer, d1)
+            index.update_document(writer, d2)
+            index.update_document(writer, d3)
+
+        response = self.client.get("/api/documents/?query=added:[-1 week to now]")
+        results = response.data["results"]
+        # Expect 3 documents returned
+        self.assertEqual(len(results), 3)
+
+        for idx, subset in enumerate(
+            [
+                {"id": 1, "title": "invoice"},
+                {"id": 2, "title": "bank statement 1"},
+                {"id": 3, "title": "bank statement 3"},
+            ],
+        ):
+            result = results[idx]
+            # Assert subset in results
+            self.assertDictEqual(result, {**result, **subset})
+
+    @override_settings(
+        TIME_ZONE="America/Chicago",
+    )
+    def test_search_added_in_last_week_with_timezone_behind(self):
+        """
+        GIVEN:
+            - Two documents added right now
+            - One document added over a week ago
+            - The timezone is behind UTC time (-6)
+        WHEN:
+            - Query for documents added in the last 7 days
+        THEN:
+            - The two recent documents are returned
+        """
+        d1 = Document.objects.create(
+            title="invoice",
+            content="the thing i bought at a shop and paid with bank account",
+            checksum="A",
+            pk=1,
+        )
+        d2 = Document.objects.create(
+            title="bank statement 1",
+            content="things i paid for in august",
+            pk=2,
+            checksum="B",
+        )
+        d3 = Document.objects.create(
+            title="bank statement 3",
+            content="things i paid for in september",
+            pk=3,
+            checksum="C",
+            # 7 days, 1 hour and 1 minute ago
+            added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
+        )
+        with index.open_index_writer() as writer:
+            index.update_document(writer, d1)
+            index.update_document(writer, d2)
+            index.update_document(writer, d3)
+
+        response = self.client.get("/api/documents/?query=added:[-1 week to now]")
+        results = response.data["results"]
+
+        # Expect 2 documents returned
+        self.assertEqual(len(results), 2)
+
+        for idx, subset in enumerate(
+            [{"id": 1, "title": "invoice"}, {"id": 2, "title": "bank statement 1"}],
+        ):
+            result = results[idx]
+            # Assert subset in results
+            self.assertDictEqual(result, {**result, **subset})
+
+    @override_settings(
+        TIME_ZONE="Europe/Sofia",
+    )
+    def test_search_added_in_last_week_with_timezone_ahead(self):
+        """
+        GIVEN:
+            - Two documents added right now
+            - One document added over a week ago
+            - The timezone is behind UTC time (+2)
+        WHEN:
+            - Query for documents added in the last 7 days
+        THEN:
+            - The two recent documents are returned
+        """
+        d1 = Document.objects.create(
+            title="invoice",
+            content="the thing i bought at a shop and paid with bank account",
+            checksum="A",
+            pk=1,
+        )
+        d2 = Document.objects.create(
+            title="bank statement 1",
+            content="things i paid for in august",
+            pk=2,
+            checksum="B",
+        )
+        d3 = Document.objects.create(
+            title="bank statement 3",
+            content="things i paid for in september",
+            pk=3,
+            checksum="C",
+            # 7 days, 1 hour and 1 minute ago
+            added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
+        )
+        with index.open_index_writer() as writer:
+            index.update_document(writer, d1)
+            index.update_document(writer, d2)
+            index.update_document(writer, d3)
+
+        response = self.client.get("/api/documents/?query=added:[-1 week to now]")
+        results = response.data["results"]
+
+        # Expect 2 documents returned
+        self.assertEqual(len(results), 2)
+
+        for idx, subset in enumerate(
+            [{"id": 1, "title": "invoice"}, {"id": 2, "title": "bank statement 1"}],
+        ):
+            result = results[idx]
+            # Assert subset in results
+            self.assertDictEqual(result, {**result, **subset})
+
+    def test_search_added_in_last_month(self):
+        """
+        GIVEN:
+            - One document added right now
+            - One documents added about a week ago
+            - One document added over 1 month
+        WHEN:
+            - Query for documents added in the last month
+        THEN:
+            - The two recent documents are returned
+        """
+        d1 = Document.objects.create(
+            title="invoice",
+            content="the thing i bought at a shop and paid with bank account",
+            checksum="A",
+            pk=1,
+        )
+        d2 = Document.objects.create(
+            title="bank statement 1",
+            content="things i paid for in august",
+            pk=2,
+            checksum="B",
+            # 1 month, 1 day ago
+            added=timezone.now() - relativedelta(months=1, days=1),
+        )
+        d3 = Document.objects.create(
+            title="bank statement 3",
+            content="things i paid for in september",
+            pk=3,
+            checksum="C",
+            # 7 days, 1 hour and 1 minute ago
+            added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
+        )
+
+        with index.open_index_writer() as writer:
+            index.update_document(writer, d1)
+            index.update_document(writer, d2)
+            index.update_document(writer, d3)
+
+        response = self.client.get("/api/documents/?query=added:[-1 month to now]")
+        results = response.data["results"]
+
+        # Expect 2 documents returned
+        self.assertEqual(len(results), 2)
+
+        for idx, subset in enumerate(
+            [{"id": 1, "title": "invoice"}, {"id": 3, "title": "bank statement 3"}],
+        ):
+            result = results[idx]
+            # Assert subset in results
+            self.assertDictEqual(result, {**result, **subset})
+
+    @override_settings(
+        TIME_ZONE="America/Denver",
+    )
+    def test_search_added_in_last_month_timezone_behind(self):
+        """
+        GIVEN:
+            - One document added right now
+            - One documents added about a week ago
+            - One document added over 1 month
+            - The timezone is behind UTC time (-6 or -7)
+        WHEN:
+            - Query for documents added in the last month
+        THEN:
+            - The two recent documents are returned
+        """
+        d1 = Document.objects.create(
+            title="invoice",
+            content="the thing i bought at a shop and paid with bank account",
+            checksum="A",
+            pk=1,
+        )
+        d2 = Document.objects.create(
+            title="bank statement 1",
+            content="things i paid for in august",
+            pk=2,
+            checksum="B",
+            # 1 month, 1 day ago
+            added=timezone.now() - relativedelta(months=1, days=1),
+        )
+        d3 = Document.objects.create(
+            title="bank statement 3",
+            content="things i paid for in september",
+            pk=3,
+            checksum="C",
+            # 7 days, 1 hour and 1 minute ago
+            added=timezone.now() - timedelta(days=7, hours=1, minutes=1),
+        )
+
+        with index.open_index_writer() as writer:
+            index.update_document(writer, d1)
+            index.update_document(writer, d2)
+            index.update_document(writer, d3)
+
+        response = self.client.get("/api/documents/?query=added:[-1 month to now]")
+        results = response.data["results"]
+
+        # Expect 2 documents returned
+        self.assertEqual(len(results), 2)
+
+        for idx, subset in enumerate(
+            [{"id": 1, "title": "invoice"}, {"id": 3, "title": "bank statement 3"}],
+        ):
+            result = results[idx]
+            # Assert subset in results
+            self.assertDictEqual(result, {**result, **subset})
+
     @mock.patch("documents.index.autocomplete")
     def test_search_autocomplete(self, m):
         m.side_effect = lambda ix, term, limit: [term for _ in range(limit)]
@@ -2933,8 +3199,32 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, 400)
         self.assertEqual(StoragePath.objects.count(), 1)
 
+    def test_api_storage_path_placeholders(self):
+        """
+        GIVEN:
+            - API request to create a storage path with placeholders
+            - Storage path is valid
+        WHEN:
+            - API is called
+        THEN:
+            - Correct HTTP response
+            - New storage path is created
+        """
+        response = self.client.post(
+            self.ENDPOINT,
+            json.dumps(
+                {
+                    "name": "Storage path with placeholders",
+                    "path": "{title}/{correspondent}/{document_type}/{created}/{created_year}/{created_year_short}/{created_month}/{created_month_name}/{created_month_name_short}/{created_day}/{added}/{added_year}/{added_year_short}/{added_month}/{added_month_name}/{added_month_name_short}/{added_day}/{asn}/{tags}/{tag_list}/",
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(StoragePath.objects.count(), 2)
 
-class TestTasks(APITestCase):
+
+class TestTasks(DirectoriesMixin, APITestCase):
     ENDPOINT = "/api/tasks/"
     ENDPOINT_ACKNOWLEDGE = "/api/acknowledge_tasks/"
 

src/documents/tests/test_consumer.py

@@ -833,7 +833,8 @@ class PreConsumeTestCase(TestCase):
         with tempfile.NamedTemporaryFile() as script:
             with override_settings(PRE_CONSUME_SCRIPT=script.name):
                 c = Consumer()
-                c.path = "path-to-file"
+                c.original_path = "path-to-file"
+                c.path = "/tmp/somewhere/path-to-file"
                 c.run_pre_consume_script()
 
                 m.assert_called_once()
@@ -841,10 +842,17 @@ class PreConsumeTestCase(TestCase):
                 args, kwargs = m.call_args
 
                 command = kwargs["args"]
+                environment = kwargs["env"]
 
                 self.assertEqual(command[0], script.name)
                 self.assertEqual(command[1], "path-to-file")
 
+                subset = {
+                    "DOCUMENT_SOURCE_PATH": c.original_path,
+                    "DOCUMENT_WORKING_PATH": c.path,
+                }
+                self.assertDictEqual(environment, {**environment, **subset})
+
     @mock.patch("documents.consumer.Consumer.log")
     def test_script_with_output(self, mocked_log):
         """
@@ -961,9 +969,10 @@ class PostConsumeTestCase(TestCase):
 
                 m.assert_called_once()
 
-                args, kwargs = m.call_args
+                _, kwargs = m.call_args
 
                 command = kwargs["args"]
+                environment = kwargs["env"]
 
                 self.assertEqual(command[0], script.name)
                 self.assertEqual(command[1], str(doc.pk))
@@ -972,6 +981,16 @@ class PostConsumeTestCase(TestCase):
                 self.assertEqual(command[7], "my_bank")
                 self.assertCountEqual(command[8].split(","), ["a", "b"])
 
+                subset = {
+                    "DOCUMENT_ID": str(doc.pk),
+                    "DOCUMENT_DOWNLOAD_URL": f"/api/documents/{doc.pk}/download/",
+                    "DOCUMENT_THUMBNAIL_URL": f"/api/documents/{doc.pk}/thumb/",
+                    "DOCUMENT_CORRESPONDENT": "my_bank",
+                    "DOCUMENT_TAGS": "a,b",
+                }
+
+                self.assertDictEqual(environment, {**environment, **subset})
+
     def test_script_exit_non_zero(self):
         """
         GIVEN:

src/documents/tests/test_importer.py

@@ -25,7 +25,7 @@ class TestImporter(TestCase):
         cmd.manifest = [{"model": "documents.document"}]
         with self.assertRaises(CommandError) as cm:
             cmd._check_manifest()
-        self.assertTrue("The manifest file contains a record" in str(cm.exception))
+        self.assertIn("The manifest file contains a record", str(cm.exception))
 
         cmd.manifest = [
             {"model": "documents.document", EXPORTER_FILE_NAME: "noexist.pdf"},
@@ -33,6 +33,7 @@ class TestImporter(TestCase):
         # self.assertRaises(CommandError, cmd._check_manifest)
         with self.assertRaises(CommandError) as cm:
             cmd._check_manifest()
-        self.assertTrue(
-            'The manifest file refers to "noexist.pdf"' in str(cm.exception),
+        self.assertIn(
+            'The manifest file refers to "noexist.pdf"',
+            str(cm.exception),
         )

src/documents/tests/test_index.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 from django.test import TestCase
 from documents import index
 from documents.models import Document
@@ -31,3 +33,60 @@ class TestAutoComplete(DirectoriesMixin, TestCase):
         )
         self.assertListEqual(index.autocomplete(ix, "tes", limit=1), [b"test3"])
         self.assertListEqual(index.autocomplete(ix, "tes", limit=0), [])
+
+    def test_archive_serial_number_ranging(self):
+        """
+        GIVEN:
+            - Document with an archive serial number above schema allowed size
+        WHEN:
+            - Document is provided to the index
+        THEN:
+            - Error is logged
+            - Document ASN is reset to 0 for the index
+        """
+        doc1 = Document.objects.create(
+            title="doc1",
+            checksum="A",
+            content="test test2 test3",
+            # yes, this is allowed, unless full_clean is run
+            # DRF does call the validators, this test won't
+            archive_serial_number=Document.ARCHIVE_SERIAL_NUMBER_MAX + 1,
+        )
+        with self.assertLogs("paperless.index", level="ERROR") as cm:
+            with mock.patch(
+                "documents.index.AsyncWriter.update_document",
+            ) as mocked_update_doc:
+                index.add_or_update_document(doc1)
+
+                mocked_update_doc.assert_called_once()
+                _, kwargs = mocked_update_doc.call_args
+
+                self.assertEqual(kwargs["asn"], 0)
+
+                error_str = cm.output[0]
+                expected_str = "ERROR:paperless.index:Not indexing Archive Serial Number 4294967296 of document 1"
+                self.assertIn(expected_str, error_str)
+
+    def test_archive_serial_number_is_none(self):
+        """
+        GIVEN:
+            - Document with no archive serial number
+        WHEN:
+            - Document is provided to the index
+        THEN:
+            - ASN isn't touched
+        """
+        doc1 = Document.objects.create(
+            title="doc1",
+            checksum="A",
+            content="test test2 test3",
+        )
+        with mock.patch(
+            "documents.index.AsyncWriter.update_document",
+        ) as mocked_update_doc:
+            index.add_or_update_document(doc1)
+
+            mocked_update_doc.assert_called_once()
+            _, kwargs = mocked_update_doc.call_args
+
+            self.assertIsNone(kwargs["asn"])

src/documents/tests/test_management_consumer.py

@@ -247,22 +247,85 @@ class TestConsumer(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
 
     def test_is_ignored(self):
         test_paths = [
-            (os.path.join(self.dirs.consumption_dir, "foo.pdf"), False),
-            (os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"), False),
-            (os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"), True),
-            (
-                os.path.join(self.dirs.consumption_dir, "foo", ".DS_STORE", "bar.pdf"),
-                True,
-            ),
-            (os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"), True),
-            (os.path.join(self.dirs.consumption_dir, "._foo.pdf"), True),
-            (os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"), False),
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "foo.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "foo", "bar.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    "foo",
+                    ".DS_STORE",
+                    "bar.pdf",
+                ),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    ".DS_STORE",
+                    "foo",
+                    "bar.pdf",
+                ),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".stfolder", "foo.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".stfolder.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    ".stversions",
+                    "foo.pdf",
+                ),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, ".stversions.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "._foo.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "my_foo.pdf"),
+                "ignore": False,
+            },
+            {
+                "path": os.path.join(self.dirs.consumption_dir, "._foo", "bar.pdf"),
+                "ignore": True,
+            },
+            {
+                "path": os.path.join(
+                    self.dirs.consumption_dir,
+                    "@eaDir",
+                    "SYNO@.fileindexdb",
+                    "_1jk.fnm",
+                ),
+                "ignore": True,
+            },
         ]
-        for file_path, expected_ignored in test_paths:
+        for test_setup in test_paths:
+            filepath = test_setup["path"]
+            expected_ignored_result = test_setup["ignore"]
             self.assertEqual(
-                expected_ignored,
-                document_consumer._is_ignored(file_path),
-                f'_is_ignored("{file_path}") != {expected_ignored}',
+                expected_ignored_result,
+                document_consumer._is_ignored(filepath),
+                f'_is_ignored("{filepath}") != {expected_ignored_result}',
             )
 
     @mock.patch("documents.management.commands.document_consumer.open")

src/documents/tests/test_parsers.py

@@ -1,6 +1,8 @@
 from tempfile import TemporaryDirectory
 from unittest import mock
 
+from django.apps import apps
+from django.test import override_settings
 from django.test import TestCase
 from documents.parsers import get_default_file_extension
 from documents.parsers import get_parser_class_for_mime_type
@@ -8,6 +10,7 @@ from documents.parsers import get_supported_file_extensions
 from documents.parsers import is_file_ext_supported
 from paperless_tesseract.parsers import RasterisedDocumentParser
 from paperless_text.parsers import TextDocumentParser
+from paperless_tika.parsers import TikaDocumentParser
 
 
 class TestParserDiscovery(TestCase):
@@ -124,14 +127,43 @@ class TestParserDiscovery(TestCase):
 
 
 class TestParserAvailability(TestCase):
-    def test_file_extensions(self):
-
+    def test_tesseract_parser(self):
+        """
+        GIVEN:
+            - Various mime types
+        WHEN:
+            - The parser class is instantiated
+        THEN:
+            - The Tesseract based parser is return
+        """
         supported_mimes_and_exts = [
             ("application/pdf", ".pdf"),
             ("image/png", ".png"),
             ("image/jpeg", ".jpg"),
             ("image/tiff", ".tif"),
             ("image/webp", ".webp"),
+        ]
+
+        supported_exts = get_supported_file_extensions()
+
+        for mime_type, ext in supported_mimes_and_exts:
+            self.assertIn(ext, supported_exts)
+            self.assertEqual(get_default_file_extension(mime_type), ext)
+            self.assertIsInstance(
+                get_parser_class_for_mime_type(mime_type)(logging_group=None),
+                RasterisedDocumentParser,
+            )
+
+    def test_text_parser(self):
+        """
+        GIVEN:
+            - Various mime types of a text form
+        WHEN:
+            - The parser class is instantiated
+        THEN:
+            - The text based parser is return
+        """
+        supported_mimes_and_exts = [
             ("text/plain", ".txt"),
             ("text/csv", ".csv"),
         ]
@@ -141,23 +173,55 @@ class TestParserAvailability(TestCase):
         for mime_type, ext in supported_mimes_and_exts:
             self.assertIn(ext, supported_exts)
             self.assertEqual(get_default_file_extension(mime_type), ext)
+            self.assertIsInstance(
+                get_parser_class_for_mime_type(mime_type)(logging_group=None),
+                TextDocumentParser,
+            )
 
+    def test_tika_parser(self):
+        """
+        GIVEN:
+            - Various mime types of a office document form
+        WHEN:
+            - The parser class is instantiated
+        THEN:
+            - The Tika/Gotenberg based parser is return
+        """
+        supported_mimes_and_exts = [
+            ("application/vnd.oasis.opendocument.text", ".odt"),
+            ("text/rtf", ".rtf"),
+            ("application/msword", ".doc"),
+            (
+                "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                ".docx",
+            ),
+        ]
+
+        # Force the app ready to notice the settings override
+        with override_settings(TIKA_ENABLED=True, INSTALLED_APPS=["paperless_tika"]):
+            app = apps.get_app_config("paperless_tika")
+            app.ready()
+            supported_exts = get_supported_file_extensions()
+
+        for mime_type, ext in supported_mimes_and_exts:
+            self.assertIn(ext, supported_exts)
+            self.assertEqual(get_default_file_extension(mime_type), ext)
+            self.assertIsInstance(
+                get_parser_class_for_mime_type(mime_type)(logging_group=None),
+                TikaDocumentParser,
+            )
+
+    def test_no_parser_for_mime(self):
+        self.assertIsNone(get_parser_class_for_mime_type("text/sdgsdf"))
+
+    def test_default_extension(self):
         # Test no parser declared still returns a an extension
         self.assertEqual(get_default_file_extension("application/zip"), ".zip")
 
         # Test invalid mimetype returns no extension
         self.assertEqual(get_default_file_extension("aasdasd/dgfgf"), "")
 
-        self.assertIsInstance(
-            get_parser_class_for_mime_type("application/pdf")(logging_group=None),
-            RasterisedDocumentParser,
-        )
-        self.assertIsInstance(
-            get_parser_class_for_mime_type("text/plain")(logging_group=None),
-            TextDocumentParser,
-        )
-        self.assertIsNone(get_parser_class_for_mime_type("text/sdgsdf"))
-
+    def test_file_extension_support(self):
         self.assertTrue(is_file_ext_supported(".pdf"))
         self.assertFalse(is_file_ext_supported(".hsdfh"))
         self.assertFalse(is_file_ext_supported(""))

src/documents/tests/utils.py

@@ -3,6 +3,7 @@ import shutil
 import tempfile
 from collections import namedtuple
 from contextlib import contextmanager
+from unittest import mock
 
 from django.apps import apps
 from django.db import connection
@@ -86,6 +87,30 @@ class DirectoriesMixin:
         remove_dirs(self.dirs)
 
 
+class ConsumerProgressMixin:
+    def setUp(self) -> None:
+        self.send_progress_patcher = mock.patch(
+            "documents.consumer.Consumer._send_progress",
+        )
+        self.send_progress_mock = self.send_progress_patcher.start()
+        super().setUp()
+
+    def tearDown(self) -> None:
+        super().tearDown()
+        self.send_progress_patcher.stop()
+
+
+class DocumentConsumeDelayMixin:
+    def setUp(self) -> None:
+        self.consume_file_patcher = mock.patch("documents.tasks.consume_file.delay")
+        self.consume_file_mock = self.consume_file_patcher.start()
+        super().setUp()
+
+    def tearDown(self) -> None:
+        super().tearDown()
+        self.consume_file_patcher.stop()
+
+
 class TestMigrations(TransactionTestCase):
     @property
     def app(self):

src/documents/views.py

@@ -445,6 +445,10 @@ class DocumentViewSet(
                 )
                 c.save()
 
+                from documents import index
+
+                index.add_or_update_document(self.get_object())
+
                 return Response(self.getComments(doc))
             except Exception as e:
                 logger.warning(f"An error occurred saving comment: {str(e)}")
@@ -456,6 +460,11 @@ class DocumentViewSet(
         elif request.method == "DELETE":
             comment = Comment.objects.get(id=int(request.GET.get("id")))
             comment.delete()
+
+            from documents import index
+
+            index.add_or_update_document(self.get_object())
+
             return Response(self.getComments(doc))
 
         return Response(
@@ -468,19 +477,14 @@ class DocumentViewSet(
 class SearchResultSerializer(DocumentSerializer):
     def to_representation(self, instance):
         doc = Document.objects.get(id=instance["id"])
-        commentTerm = instance.results.q.subqueries[0]
         comments = ",".join(
-            [
-                str(c.comment)
-                for c in Comment.objects.filter(document=instance["id"])
-                if commentTerm.text in c.comment
-            ],
+            [str(c.comment) for c in Comment.objects.filter(document=instance["id"])],
         )
         r = super().to_representation(doc)
         r["__search_hit__"] = {
             "score": instance.score,
             "highlights": instance.highlights("content", text=doc.content),
-            "comment_highlights": instance.highlights("content", text=comments)
+            "comment_highlights": instance.highlights("comments", text=comments)
             if doc
             else None,
             "rank": instance.rank,

src/locale/nl_NL/LC_MESSAGES/django.po

@@ -3,7 +3,7 @@ msgstr ""
 "Project-Id-Version: paperless-ngx\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2022-11-09 21:50+0000\n"
-"PO-Revision-Date: 2023-01-23 12:37\n"
+"PO-Revision-Date: 2023-01-27 19:22\n"
 "Last-Translator: \n"
 "Language-Team: Dutch\n"
 "Language: nl_NL\n"
@@ -368,15 +368,15 @@ msgstr "heeft tags in"
 
 #: documents/models.py:410
 msgid "ASN greater than"
-msgstr ""
+msgstr "ASN groter dan"
 
 #: documents/models.py:411
 msgid "ASN less than"
-msgstr ""
+msgstr "ASN kleiner dan"
 
 #: documents/models.py:412
 msgid "storage path is"
-msgstr ""
+msgstr "opslagpad is"
 
 #: documents/models.py:422
 msgid "rule type"
@@ -396,99 +396,99 @@ msgstr "filterregels"
 
 #: documents/models.py:536
 msgid "Task ID"
-msgstr ""
+msgstr "Taak ID"
 
 #: documents/models.py:537
 msgid "Celery ID for the Task that was run"
-msgstr ""
+msgstr "Celery ID voor de taak die werd uitgevoerd"
 
 #: documents/models.py:542
 msgid "Acknowledged"
-msgstr ""
+msgstr "Bevestigd"
 
 #: documents/models.py:543
 msgid "If the task is acknowledged via the frontend or API"
-msgstr ""
+msgstr "Of de taak is bevestigd via de frontend of de API"
 
 #: documents/models.py:549 documents/models.py:556
 msgid "Task Name"
-msgstr ""
+msgstr "Taaknaam"
 
 #: documents/models.py:550
 msgid "Name of the file which the Task was run for"
-msgstr ""
+msgstr "Naam van het bestand waarvoor de taak werd uitgevoerd"
 
 #: documents/models.py:557
 msgid "Name of the Task which was run"
-msgstr ""
+msgstr "Naam van de uitgevoerde taak"
 
 #: documents/models.py:562
 msgid "Task Positional Arguments"
-msgstr ""
+msgstr "Positionele argumenten voor taak"
 
 #: documents/models.py:564
 msgid "JSON representation of the positional arguments used with the task"
-msgstr ""
+msgstr "JSON weergave van de positionele argumenten die gebruikt worden voor de taak"
 
 #: documents/models.py:569
 msgid "Task Named Arguments"
-msgstr ""
+msgstr "Argumenten met naam voor taak"
 
 #: documents/models.py:571
 msgid "JSON representation of the named arguments used with the task"
-msgstr ""
+msgstr "JSON weergave van de argumenten met naam die gebruikt worden voor de taak"
 
 #: documents/models.py:578
 msgid "Task State"
-msgstr ""
+msgstr "Taakstatus"
 
 #: documents/models.py:579
 msgid "Current state of the task being run"
-msgstr ""
+msgstr "Huidige status van de taak die wordt uitgevoerd"
 
 #: documents/models.py:584
 msgid "Created DateTime"
-msgstr ""
+msgstr "Aangemaakt DateTime"
 
 #: documents/models.py:585
 msgid "Datetime field when the task result was created in UTC"
-msgstr ""
+msgstr "Datetime veld wanneer het resultaat van de taak werd aangemaakt in UTC"
 
 #: documents/models.py:590
 msgid "Started DateTime"
-msgstr ""
+msgstr "Gestart DateTime"
 
 #: documents/models.py:591
 msgid "Datetime field when the task was started in UTC"
-msgstr ""
+msgstr "Datetime veld wanneer de taak werd gestart in UTC"
 
 #: documents/models.py:596
 msgid "Completed DateTime"
-msgstr ""
+msgstr "Voltooid DateTime"
 
 #: documents/models.py:597
 msgid "Datetime field when the task was completed in UTC"
-msgstr ""
+msgstr "Datetime veld wanneer de taak werd voltooid in UTC"
 
 #: documents/models.py:602
 msgid "Result Data"
-msgstr ""
+msgstr "Resultaatgegevens"
 
 #: documents/models.py:604
 msgid "The data returned by the task"
-msgstr ""
+msgstr "Gegevens geretourneerd door de taak"
 
 #: documents/models.py:613
 msgid "Comment for the document"
-msgstr ""
+msgstr "Commentaar op het document"
 
 #: documents/models.py:642
 msgid "comment"
-msgstr ""
+msgstr "opmerking"
 
 #: documents/models.py:643
 msgid "comments"
-msgstr ""
+msgstr "opmerkingen"
 
 #: documents/serialisers.py:72
 #, python-format

src/paperless/settings.py

@@ -109,6 +109,16 @@ def _parse_redis_url(env_redis: Optional[str]) -> Tuple[str]:
 
 
 def _parse_beat_schedule() -> Dict:
+    """
+    Configures the scheduled tasks, according to default or
+    environment variables.  Task expiration is configured so the task will
+    expire (and not run), shortly before the default frequency will put another
+    of the same task into the queue
+
+
+    https://docs.celeryq.dev/en/stable/userguide/periodic-tasks.html#beat-entries
+    https://docs.celeryq.dev/en/latest/userguide/calling.html#expiration
+    """
     schedule = {}
     tasks = [
         {
@@ -117,6 +127,11 @@ def _parse_beat_schedule() -> Dict:
             # Default every ten minutes
             "env_default": "*/10 * * * *",
             "task": "paperless_mail.tasks.process_mail_accounts",
+            "options": {
+                # 1 minute before default schedule sends again
+                "expires": 9.0
+                * 60.0,
+            },
         },
         {
             "name": "Train the classifier",
@@ -124,6 +139,11 @@ def _parse_beat_schedule() -> Dict:
             # Default hourly at 5 minutes past the hour
             "env_default": "5 */1 * * *",
             "task": "documents.tasks.train_classifier",
+            "options": {
+                # 1 minute before default schedule sends again
+                "expires": 59.0
+                * 60.0,
+            },
         },
         {
             "name": "Optimize the index",
@@ -131,6 +151,12 @@ def _parse_beat_schedule() -> Dict:
             # Default daily at midnight
             "env_default": "0 0 * * *",
             "task": "documents.tasks.index_optimize",
+            "options": {
+                # 1 hour before default schedule sends again
+                "expires": 23.0
+                * 60.0
+                * 60.0,
+            },
         },
         {
             "name": "Perform sanity check",
@@ -138,6 +164,12 @@ def _parse_beat_schedule() -> Dict:
             # Default Sunday at 00:30
             "env_default": "30 0 * * sun",
             "task": "documents.tasks.sanity_check",
+            "options": {
+                # 1 hour before default schedule sends again
+                "expires": ((7.0 * 24.0) - 1.0)
+                * 60.0
+                * 60.0,
+            },
         },
     ]
     for task in tasks:
@@ -151,9 +183,11 @@ def _parse_beat_schedule() -> Dict:
         #   - five time-and-date fields
         #   - separated by at least one blank
         minute, hour, day_month, month, day_week = value.split(" ")
+
         schedule[task["name"]] = {
             "task": task["task"],
             "schedule": crontab(minute, hour, day_week, day_month, month),
+            "options": task["options"],
         }
 
     return schedule
@@ -263,6 +297,10 @@ MIDDLEWARE = [
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
+# Optional to enable compression
+if __get_boolean("PAPERLESS_ENABLE_COMPRESSION", "yes"):  # pragma: nocover
+    MIDDLEWARE.insert(0, "compression_middleware.middleware.CompressionMiddleware")
+
 ROOT_URLCONF = "paperless.urls"
 
 FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME")
@@ -280,7 +318,6 @@ _CELERY_REDIS_URL, _CHANNELS_REDIS_URL = _parse_redis_url(
     os.getenv("PAPERLESS_REDIS", None),
 )
 
-# TODO: what is this used for?
 TEMPLATES = [
     {
         "BACKEND": "django.template.backends.django.DjangoTemplates",
@@ -561,22 +598,21 @@ LOGGING = {
 # Task queue                                                                  #
 ###############################################################################
 
-TASK_WORKERS = __get_int("PAPERLESS_TASK_WORKERS", 1)
-
-WORKER_TIMEOUT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800)
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html
 
 CELERY_BROKER_URL = _CELERY_REDIS_URL
 CELERY_TIMEZONE = TIME_ZONE
 
 CELERY_WORKER_HIJACK_ROOT_LOGGER = False
-CELERY_WORKER_CONCURRENCY = TASK_WORKERS
+CELERY_WORKER_CONCURRENCY: Final[int] = __get_int("PAPERLESS_TASK_WORKERS", 1)
+TASK_WORKERS = CELERY_WORKER_CONCURRENCY
 CELERY_WORKER_MAX_TASKS_PER_CHILD = 1
 CELERY_WORKER_SEND_TASK_EVENTS = True
-
+CELERY_TASK_SEND_SENT_EVENT = True
 CELERY_SEND_TASK_SENT_EVENT = True
 
 CELERY_TASK_TRACK_STARTED = True
-CELERY_TASK_TIME_LIMIT = WORKER_TIMEOUT
+CELERY_TASK_TIME_LIMIT: Final[int] = __get_int("PAPERLESS_WORKER_TIMEOUT", 1800)
 
 CELERY_RESULT_EXTENDED = True
 CELERY_RESULT_BACKEND = "django-db"
@@ -608,7 +644,7 @@ def default_threads_per_worker(task_workers) -> int:
 
 THREADS_PER_WORKER = os.getenv(
     "PAPERLESS_THREADS_PER_WORKER",
-    default_threads_per_worker(TASK_WORKERS),
+    default_threads_per_worker(CELERY_WORKER_CONCURRENCY),
 )
 
 ###############################################################################
@@ -637,7 +673,7 @@ CONSUMER_IGNORE_PATTERNS = list(
     json.loads(
         os.getenv(
             "PAPERLESS_CONSUMER_IGNORE_PATTERNS",
-            '[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]',  # noqa: E501
+            '[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]',  # noqa: E501
         ),
     ),
 )

src/paperless/tests/test_settings.py

@@ -149,6 +149,11 @@ class TestRedisSocketConversion(TestCase):
 
 
 class TestCeleryScheduleParsing(TestCase):
+    MAIL_EXPIRE_TIME = 9.0 * 60.0
+    CLASSIFIER_EXPIRE_TIME = 59.0 * 60.0
+    INDEX_EXPIRE_TIME = 23.0 * 60.0 * 60.0
+    SANITY_EXPIRE_TIME = ((7.0 * 24.0) - 1.0) * 60.0 * 60.0
+
     def test_schedule_configuration_default(self):
         """
         GIVEN:
@@ -165,18 +170,22 @@ class TestCeleryScheduleParsing(TestCase):
                 "Check all e-mail accounts": {
                     "task": "paperless_mail.tasks.process_mail_accounts",
                     "schedule": crontab(minute="*/10"),
+                    "options": {"expires": self.MAIL_EXPIRE_TIME},
                 },
                 "Train the classifier": {
                     "task": "documents.tasks.train_classifier",
                     "schedule": crontab(minute="5", hour="*/1"),
+                    "options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
                 },
                 "Optimize the index": {
                     "task": "documents.tasks.index_optimize",
                     "schedule": crontab(minute=0, hour=0),
+                    "options": {"expires": self.INDEX_EXPIRE_TIME},
                 },
                 "Perform sanity check": {
                     "task": "documents.tasks.sanity_check",
                     "schedule": crontab(minute=30, hour=0, day_of_week="sun"),
+                    "options": {"expires": self.SANITY_EXPIRE_TIME},
                 },
             },
             schedule,
@@ -203,18 +212,22 @@ class TestCeleryScheduleParsing(TestCase):
                 "Check all e-mail accounts": {
                     "task": "paperless_mail.tasks.process_mail_accounts",
                     "schedule": crontab(minute="*/50", day_of_week="mon"),
+                    "options": {"expires": self.MAIL_EXPIRE_TIME},
                 },
                 "Train the classifier": {
                     "task": "documents.tasks.train_classifier",
                     "schedule": crontab(minute="5", hour="*/1"),
+                    "options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
                 },
                 "Optimize the index": {
                     "task": "documents.tasks.index_optimize",
                     "schedule": crontab(minute=0, hour=0),
+                    "options": {"expires": self.INDEX_EXPIRE_TIME},
                 },
                 "Perform sanity check": {
                     "task": "documents.tasks.sanity_check",
                     "schedule": crontab(minute=30, hour=0, day_of_week="sun"),
+                    "options": {"expires": self.SANITY_EXPIRE_TIME},
                 },
             },
             schedule,
@@ -238,14 +251,17 @@ class TestCeleryScheduleParsing(TestCase):
                 "Check all e-mail accounts": {
                     "task": "paperless_mail.tasks.process_mail_accounts",
                     "schedule": crontab(minute="*/10"),
+                    "options": {"expires": self.MAIL_EXPIRE_TIME},
                 },
                 "Train the classifier": {
                     "task": "documents.tasks.train_classifier",
                     "schedule": crontab(minute="5", hour="*/1"),
+                    "options": {"expires": self.CLASSIFIER_EXPIRE_TIME},
                 },
                 "Perform sanity check": {
                     "task": "documents.tasks.sanity_check",
                     "schedule": crontab(minute=30, hour=0, day_of_week="sun"),
+                    "options": {"expires": self.SANITY_EXPIRE_TIME},
                 },
             },
             schedule,

src/paperless/tests/test_websockets.py

@@ -14,15 +14,14 @@ TEST_CHANNEL_LAYERS = {
 }
 
 
+@override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
 class TestWebSockets(TestCase):
-    @override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
     async def test_no_auth(self):
         communicator = WebsocketCommunicator(application, "/ws/status/")
         connected, subprotocol = await communicator.connect()
         self.assertFalse(connected)
         await communicator.disconnect()
 
-    @override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
     @mock.patch("paperless.consumers.StatusConsumer._authenticated")
     async def test_auth(self, _authenticated):
         _authenticated.return_value = True
@@ -33,7 +32,6 @@ class TestWebSockets(TestCase):
 
         await communicator.disconnect()
 
-    @override_settings(CHANNEL_LAYERS=TEST_CHANNEL_LAYERS)
     @mock.patch("paperless.consumers.StatusConsumer._authenticated")
     async def test_receive(self, _authenticated):
         _authenticated.return_value = True