Bumps version to 1.13.0

Feature: allow disable warn on close saved view with changes

.build-config.json

@@ -1,6 +1,6 @@
 {
   "qpdf": {
-      "version": "11.1.1"
+      "version": "11.2.0"
     },
   "jbig2enc": {
       "version": "0.29",

.codecov.yml

@@ -0,0 +1,19 @@
+# https://docs.codecov.com/docs/pull-request-comments
+# codecov will only comment if coverage changes
+comment:
+  require_changes: true
+coverage:
+  status:
+    project:
+      default:
+        # https://docs.codecov.com/docs/commit-status#threshold
+        threshold: 1%
+        # https://docs.codecov.com/docs/commit-status#only_pulls
+        only_pulls: true
+    patch:
+      default:
+        # For the changed lines only, target 75% covered, but
+        # allow as low as 50%
+        target: 75%
+        threshold: 25%
+        only_pulls: true

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -6,7 +6,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Have a question? 👉 [Start a new discussion](https://github.com/paperless-ngx/paperless-ngx/discussions/new) or [ask in chat](https://matrix.to/#/#paperless:adnidor.de).
+        Have a question? 👉 [Start a new discussion](https://github.com/paperless-ngx/paperless-ngx/discussions/new) or [ask in chat](https://matrix.to/#/#paperlessngx:matrix.org).
 
         Before opening an issue, please double check:
 

.github/ISSUE_TEMPLATE/config.yml

@@ -4,7 +4,7 @@ contact_links:
     url: https://github.com/paperless-ngx/paperless-ngx/discussions
     about: This issue tracker is not for support questions. Please refer to our Discussions.
   - name: 💬 Chat
-    url: https://matrix.to/#/#paperless:adnidor.de
+    url: https://matrix.to/#/#paperlessngx:matrix.org
     about: Want to discuss Paperless-ngx with others? Check out our chat.
   - name: 🚀 Feature Request
     url: https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=feature-requests

.github/release-drafter.yml

@@ -4,6 +4,7 @@ autolabeler:
       - '/^fix/'
     title:
       - "/^fix/i"
+      - "/^Bugfix/i"
   - label: "enhancement"
     branch:
       - '/^feature/'
@@ -13,6 +14,9 @@ categories:
   - title: 'Breaking Changes'
     labels:
       - 'breaking-change'
+  - title: 'Notable Changes'
+    labels:
+      - 'notable'
   - title: 'Features'
     labels:
       - 'enhancement'
@@ -20,7 +24,8 @@ categories:
     labels:
       - 'bug'
   - title: 'Documentation'
-    label: 'documentation'
+    labels:
+      - 'documentation'
   - title: 'Maintenance'
     labels:
       - 'chore'
@@ -29,7 +34,8 @@ categories:
       - 'ci-cd'
   - title: 'Dependencies'
     collapse-after: 3
-    label: 'dependencies'
+    labels:
+      - 'dependencies'
   - title: 'All App Changes'
     labels:
       - 'frontend'
@@ -46,6 +52,8 @@ include-labels:
   - 'frontend'
   - 'backend'
   - 'ci-cd'
+  - 'breaking-change'
+  - 'notable'
 category-template: '### $TITLE'
 change-template: '- $TITLE @$AUTHOR ([#$NUMBER]($URL))'
 change-title-escapes: '\<*_&#@'

.github/scripts/cleanup-tags.py

@@ -7,6 +7,7 @@ import subprocess
 from argparse import ArgumentParser
 from typing import Dict
 from typing import Final
+from typing import Iterator
 from typing import List
 from typing import Optional
 
@@ -18,11 +19,14 @@ from github import GithubContainerRegistryApi
 logger = logging.getLogger("cleanup-tags")
 
 
-class DockerManifest2:
+class ImageProperties:
     """
-    Data class wrapping the Docker Image Manifest Version 2.
+    Data class wrapping the properties of an entry in the image index
+    manifests list.  It is NOT an actual image with layers, etc
 
-    See https://docs.docker.com/registry/spec/manifest-v2-2/
+    https://docs.docker.com/registry/spec/manifest-v2-2/
+    https://github.com/opencontainers/image-spec/blob/main/manifest.md
+    https://github.com/opencontainers/image-spec/blob/main/descriptor.md
     """
 
     def __init__(self, data: Dict) -> None:
@@ -39,6 +43,45 @@ class DockerManifest2:
         self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
 
 
+class ImageIndex:
+    """
+    Data class wrapping up logic for an OCI Image Index
+    JSON data.  Primary use is to access the manifests listing
+
+    See https://github.com/opencontainers/image-spec/blob/main/image-index.md
+    """
+
+    def __init__(self, package_url: str, tag: str) -> None:
+        self.qualified_name = f"{package_url}:{tag}"
+        logger.info(f"Getting image index for {self.qualified_name}")
+        try:
+            proc = subprocess.run(
+                [
+                    shutil.which("docker"),
+                    "buildx",
+                    "imagetools",
+                    "inspect",
+                    "--raw",
+                    self.qualified_name,
+                ],
+                capture_output=True,
+                check=True,
+            )
+
+            self._data = json.loads(proc.stdout)
+
+        except subprocess.CalledProcessError as e:
+            logger.error(
+                f"Failed to get image index for {self.qualified_name}: {e.stderr}",
+            )
+            raise e
+
+    @property
+    def image_pointers(self) -> Iterator[ImageProperties]:
+        for manifest_data in self._data["manifests"]:
+            yield ImageProperties(manifest_data)
+
+
 class RegistryTagsCleaner:
     """
     This is the base class for the image registry cleaning.  Given a package
@@ -85,7 +128,10 @@ class RegistryTagsCleaner:
 
     def clean(self):
         """
-        This method will delete image versions, based on the selected tags to delete
+        This method will delete image versions, based on the selected tags to delete.
+        It behaves more like an unlinking than actual deletion.  Removing the tag
+        simply removes a pointer to an image, but the actual image data remains accessible
+        if one has the sha256 digest of it.
         """
         for tag_to_delete in self.tags_to_delete:
             package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
@@ -149,27 +195,17 @@ class RegistryTagsCleaner:
 
             # Parse manifests to locate digests pointed to
             for tag in sorted(self.tags_to_keep):
-                full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
-                logger.info(f"Checking manifest for {full_name}")
                 try:
-                    proc = subprocess.run(
-                        [
-                            shutil.which("docker"),
-                            "manifest",
-                            "inspect",
-                            full_name,
-                        ],
-                        capture_output=True,
+                    image_index = ImageIndex(
+                        f"ghcr.io/{self.repo_owner}/{self.package_name}",
+                        tag,
                     )
-
-                    manifest_list = json.loads(proc.stdout)
-                    for manifest_data in manifest_list["manifests"]:
-                        manifest = DockerManifest2(manifest_data)
+                    for manifest in image_index.image_pointers:
 
                         if manifest.digest in untagged_versions:
                             logger.info(
                                 f"Skipping deletion of {manifest.digest},"
-                                f" referred to by {full_name}"
+                                f" referred to by {image_index.qualified_name}"
                                 f" for {manifest.platform}",
                             )
                             del untagged_versions[manifest.digest]
@@ -241,6 +277,55 @@ class RegistryTagsCleaner:
         # By default, keep anything which is tagged
         self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
 
+    def check_remaining_tags_valid(self):
+        """
+        Checks the non-deleted tags are still valid.  The assumption is if the
+        manifest is can be inspected and each image manifest if points to can be
+        inspected, the image will still pull.
+
+        https://github.com/opencontainers/image-spec/blob/main/image-index.md
+        """
+        logger.info("Beginning confirmation step")
+        a_tag_failed = False
+        for tag in sorted(self.tags_to_keep):
+
+            try:
+                image_index = ImageIndex(
+                    f"ghcr.io/{self.repo_owner}/{self.package_name}",
+                    tag,
+                )
+                for manifest in image_index.image_pointers:
+                    logger.info(f"Checking {manifest.digest} for {manifest.platform}")
+
+                    # This follows the pointer from the index to an actual image, layers and all
+                    # Note the format is @
+                    digest_name = f"ghcr.io/{self.repo_owner}/{self.package_name}@{manifest.digest}"
+
+                    try:
+
+                        subprocess.run(
+                            [
+                                shutil.which("docker"),
+                                "buildx",
+                                "imagetools",
+                                "inspect",
+                                "--raw",
+                                digest_name,
+                            ],
+                            capture_output=True,
+                            check=True,
+                        )
+                    except subprocess.CalledProcessError as e:
+                        logger.error(f"Failed to inspect digest: {e.stderr}")
+                        a_tag_failed = True
+            except subprocess.CalledProcessError as e:
+                a_tag_failed = True
+                logger.error(f"Failed to inspect: {e.stderr}")
+                continue
+
+        if a_tag_failed:
+            raise Exception("At least one image tag failed to inspect")
+
 
 class MainImageTagsCleaner(RegistryTagsCleaner):
     def decide_what_tags_to_keep(self):
@@ -301,7 +386,7 @@ class MainImageTagsCleaner(RegistryTagsCleaner):
 
 class LibraryTagsCleaner(RegistryTagsCleaner):
     """
-    Exists for the off change that someday, the installer library images
+    Exists for the off chance that someday, the installer library images
     will need their own logic
     """
 
@@ -397,6 +482,10 @@ def _main():
             # Clean images which are untagged
             cleaner.clean_untagged(args.is_manifest)
 
+            # Verify remaining tags still pull
+            if args.is_manifest:
+                cleaner.check_remaining_tags_valid()
+
 
 if __name__ == "__main__":
     _main()

.github/workflows/ci.yml

@@ -13,57 +13,64 @@ on:
     branches-ignore:
       - 'translations**'
 
+env:
+  # This is the version of pipenv all the steps will use
+  # If changing this, change Dockerfile
+  DEFAULT_PIP_ENV_VERSION: "2022.11.30"
+  # This is the default version of Python to use in most steps
+  # If changing this, change Dockerfile
+  DEFAULT_PYTHON_VERSION: "3.9"
+
 jobs:
   pre-commit:
     name: Linting Checks
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout repository
         uses: actions/checkout@v3
-
       -
-        name: Install tools
+        name: Install python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
-
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
       -
         name: Check files
         uses: pre-commit/action@v3.0.0
 
   documentation:
     name: "Build Documentation"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs:
       - pre-commit
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
-      -
-        name: Install pipenv
-        run: |
-          pipx install pipenv==2022.10.12
       -
         name: Set up Python
+        id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
           cache: "pipenv"
           cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install pipenv
+        run: |
+          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
       -
         name: Install dependencies
         run: |
-          pipenv sync --dev
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
       -
         name: List installed Python dependencies
         run: |
-          pipenv run pip list
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list
       -
         name: Make documentation
         run: |
-          pipenv run mkdocs build --config-file ./mkdocs.yml
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs build --config-file ./mkdocs.yml
       -
         name: Upload artifact
         uses: actions/upload-artifact@v3
@@ -73,7 +80,7 @@ jobs:
 
   documentation-deploy:
     name: "Deploy Documentation"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     needs:
       - documentation
@@ -92,7 +99,7 @@ jobs:
 
   tests-backend:
     name: "Tests (${{ matrix.python-version }})"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs:
       - pre-commit
     strategy:
@@ -106,68 +113,61 @@ jobs:
       PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
       PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
       PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
+      # Enable Gotenberg end to end testing
+      GOTENBERG_LIVE: 1
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
       -
         name: Start containers
         run: |
           docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet
           docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach
-      -
-        name: Install pipenv
-        run: |
-          pipx install pipenv==2022.10.12
       -
         name: Set up Python
+        id: setup-python
         uses: actions/setup-python@v4
         with:
           python-version: "${{ matrix.python-version }}"
           cache: "pipenv"
           cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install pipenv
+        run: |
+          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
       -
         name: Install system dependencies
         run: |
           sudo apt-get update -qq
           sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
+      -
+        name: Configure ImageMagick
+        run: |
+          sudo cp docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
       -
         name: Install Python dependencies
         run: |
-          pipenv sync --dev
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run python --version
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
       -
         name: List installed Python dependencies
         run: |
-          pipenv run pip list
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list
       -
         name: Tests
         run: |
           cd src/
-          pipenv run pytest -rfEp
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra
       -
-        name: Get changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v34
+        name: Upload coverage to Codecov
+        if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }}
+        uses: codecov/codecov-action@v3
         with:
-          files: |
-            src/**
-      -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github
+          # not required for public repos, but intermittently fails otherwise
+          token: ${{ secrets.CODECOV_TOKEN }}
+          # future expansion
+          flags: backend
       -
         name: Stop containers
         if: always()
@@ -177,7 +177,7 @@ jobs:
 
   tests-frontend:
     name: "Tests Frontend"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs:
       - pre-commit
     strategy:
@@ -191,19 +191,14 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run lint
       - run: cd src-ui && npm run test
       - run: cd src-ui && npm run e2e:ci
 
   prepare-docker-build:
     name: Prepare Docker Pipeline Data
     if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
-    runs-on: ubuntu-20.04
-    # If the push triggered the installer library workflow, wait for it to
-    # complete here.  This ensures the required versions for the final
-    # image have been built, while not waiting at all if the versions haven't changed
-    concurrency:
-      group: build-installer-library
-      cancel-in-progress: false
+    runs-on: ubuntu-22.04
     needs:
       - documentation
       - tests-backend
@@ -222,7 +217,7 @@ jobs:
         name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
       -
         name: Setup qpdf image
         id: qpdf-setup
@@ -274,7 +269,7 @@ jobs:
 
   # build and push image to docker hub.
   build-docker-image:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     concurrency:
       group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
       cancel-in-progress: true
@@ -339,7 +334,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: ./Dockerfile
@@ -379,27 +374,27 @@ jobs:
   build-release:
     needs:
       - build-docker-image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
-      -
-        name: Install pipenv
-        run: |
-          pip3 install --upgrade pip setuptools wheel pipx
-          pipx install pipenv
       -
         name: Set up Python
+        id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
           cache: "pipenv"
           cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install pipenv + tools
+        run: |
+          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
       -
         name: Install Python dependencies
         run: |
-          pipenv sync --dev
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
       -
         name: Install system dependencies
         run: |
@@ -420,35 +415,62 @@ jobs:
       -
         name: Generate requirements file
         run: |
-          pipenv requirements > requirements.txt
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} requirements > requirements.txt
       -
         name: Compile messages
         run: |
           cd src/
-          pipenv run python3 manage.py compilemessages
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py compilemessages
       -
         name: Collect static files
         run: |
           cd src/
-          pipenv run python3 manage.py collectstatic --no-input
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py collectstatic --no-input
       -
         name: Move files
         run: |
-          mkdir dist
-          mkdir dist/paperless-ngx
-          mkdir dist/paperless-ngx/scripts
-          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
-          cp paperless.conf.example dist/paperless-ngx/paperless.conf
-          cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
-          cp -r docker/ dist/paperless-ngx/docker
-          cp scripts/*.service scripts/*.sh dist/paperless-ngx/scripts/
-          cp -r src/ dist/paperless-ngx/src
-          cp -r docs/_build/html/ dist/paperless-ngx/docs
-          mv static dist/paperless-ngx
+          echo "Making dist folders"
+          for directory in dist \
+                          dist/paperless-ngx \
+                          dist/paperless-ngx/scripts;
+          do
+            mkdir --verbose --parents ${directory}
+          done
+
+          echo "Copying basic files"
+          for file_name in .dockerignore \
+                          .env \
+                          Dockerfile \
+                          Pipfile \
+                          Pipfile.lock \
+                          requirements.txt \
+                          LICENSE \
+                          README.md \
+                          paperless.conf.example \
+                          gunicorn.conf.py
+          do
+            cp --verbose ${file_name} dist/paperless-ngx/
+          done
+          mv --verbose dist/paperless-ngx/paperless.conf.example paperless.conf
+
+          echo "Copying Docker related files"
+          cp --recursive docker/ dist/paperless-ngx/docker
+
+          echo "Copying startup scripts"
+          cp --verbose scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/
+
+          echo "Copying source files"
+          cp --recursive src/ dist/paperless-ngx/src
+          echo "Copying documentation"
+          cp --recursive docs/_build/html/ dist/paperless-ngx/docs
+
+          mv --verbose static dist/paperless-ngx
       -
         name: Make release package
         run: |
+          echo "Creating release archive"
           cd dist
+          sudo chown -R 1000:1000 paperless-ngx/
           tar -cJf paperless-ngx.tar.xz paperless-ngx/
       -
         name: Upload release artifact
@@ -458,7 +480,7 @@ jobs:
           path: dist/paperless-ngx.tar.xz
 
   publish-release:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
       prerelease: ${{ steps.get_version.outputs.prerelease }}
       changelog: ${{ steps.create-release.outputs.body }}
@@ -486,7 +508,7 @@ jobs:
       -
         name: Create Release and Changelog
         id: create-release
-        uses: paperless-ngx/release-drafter@master
+        uses: release-drafter/release-drafter@v5
         with:
           name: Paperless-ngx ${{ steps.get_version.outputs.version }}
           tag: ${{ steps.get_version.outputs.version }}
@@ -507,7 +529,7 @@ jobs:
           asset_content_type: application/x-xz
 
   append-changelog:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs:
       - publish-release
     if: needs.publish-release.outputs.prerelease == 'false'
@@ -517,18 +539,17 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: main
-      -
-        name: Install pipenv
-        run: |
-          pip3 install --upgrade pip setuptools wheel pipx
-          pipx install pipenv
       -
         name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
           cache: "pipenv"
           cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install pipenv + tools
+        run: |
+          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
       -
         name: Append Changelog to docs
         id: append-Changelog

.github/workflows/cleanup-tags.yml

@@ -1,17 +1,14 @@
 # This workflow runs on certain conditions to check for and potentially
 # delete container images from the GHCR which no longer have an associated
 # code branch.
-# Requires a PAT with the correct scope set in the secrets
+# Requires a PAT with the correct scope set in the secrets.
+#
+# This workflow will not trigger runs on forked repos.
 
 name: Cleanup Image Tags
 
 on:
-  schedule:
-    - cron: '0 0 * * SAT'
   delete:
-  pull_request:
-    types:
-      - closed
   push:
     paths:
       - ".github/workflows/cleanup-tags.yml"
@@ -26,7 +23,8 @@ concurrency:
 jobs:
   cleanup-images:
     name: Cleanup Image Tags for ${{ matrix.primary-name }}
-    runs-on: ubuntu-latest
+    if: github.repository_owner == 'paperless-ngx'
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         include:
@@ -64,9 +62,9 @@ jobs:
         with:
           python-version: "3.10"
       -
-        name: Install httpx
+        name: Install Python libraries
         run: |
-          python -m pip install httpx
+          python -m pip install httpx docker
       #
       # Clean up primary package
       #
@@ -83,13 +81,3 @@ jobs:
         if: "${{ env.TOKEN != '' }}"
         run: |
           python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
-      #
-      # Verify tags which are left still pull
-      #
-      -
-        name: Check all tags still pull
-        run: |
-          ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
-          echo "Pulling all tags of ${ghcr_name}"
-          docker pull --quiet --all-tags ${ghcr_name}
-          docker image list

.github/workflows/codeql-analysis.yml

@@ -23,7 +23,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       actions: read
       contents: read

.github/workflows/installer-library.yml

@@ -34,7 +34,7 @@ concurrency:
 jobs:
   prepare-docker-build:
     name: Prepare Docker Image Version Data
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       -
         name: Set ghcr repository name
@@ -95,8 +95,8 @@ jobs:
         name: Setup other versions
         id: cache-bust-setup
         run: |
-          pillow_version=$(jq ".default.pillow.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
-          lxml_version=$(jq ".default.lxml.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+          pillow_version=$(jq -r '.default.pillow.version | gsub("=";"")' Pipfile.lock)
+          lxml_version=$(jq -r '.default.lxml.version | gsub("=";"")' Pipfile.lock)
 
           echo "Pillow is ${pillow_version}"
           echo "lxml is ${lxml_version}"
@@ -127,6 +127,7 @@ jobs:
     uses: ./.github/workflows/reusable-workflow-builder.yml
     with:
       dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-platforms: linux/amd64
       build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
       build-args: |
         QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
@@ -168,3 +169,142 @@ jobs:
         PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
         PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
         LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}
+
+  commit-binary-files:
+    name: Store installers
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+      - build-jbig2enc
+      - build-psycopg2-wheel
+      - build-pikepdf-wheel
+    runs-on: ubuntu-22.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: binary-library
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends tree
+      -
+        name: Extract qpdf files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+
+          docker pull --quiet ${tag}
+          docker create --name qpdf-extract ${tag}
+
+          mkdir --parents qpdf/${version}/amd64
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/amd64 qpdf/${version}
+
+          mkdir --parents qpdf/${version}/arm64
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/arm64 qpdf/${version}
+
+          mkdir --parents qpdf/${version}/armv7
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/armv7 qpdf/${version}
+      -
+        name: Extract psycopg2 files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/amd64
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/amd64
+          mv psycopg2/${version}/amd64/wheels/* psycopg2/${version}/amd64
+          rm -r psycopg2/${version}/amd64/wheels/
+          docker rm psycopg2-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/arm64
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/arm64
+          mv psycopg2/${version}/arm64/wheels/* psycopg2/${version}/arm64
+          rm -r psycopg2/${version}/arm64/wheels/
+          docker rm psycopg2-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/armv7
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/armv7
+          mv psycopg2/${version}/armv7/wheels/* psycopg2/${version}/armv7
+          rm -r psycopg2/${version}/armv7/wheels/
+          docker rm psycopg2-extract
+      -
+        name: Extract pikepdf files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/amd64
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/amd64
+          mv pikepdf/${version}/amd64/wheels/* pikepdf/${version}/amd64
+          rm -r pikepdf/${version}/amd64/wheels/
+          docker rm pikepdf-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/arm64
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/arm64
+          mv pikepdf/${version}/arm64/wheels/* pikepdf/${version}/arm64
+          rm -r pikepdf/${version}/arm64/wheels/
+          docker rm pikepdf-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/armv7
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/armv7
+          mv pikepdf/${version}/armv7/wheels/* pikepdf/${version}/armv7
+          rm -r pikepdf/${version}/armv7/wheels/
+          docker rm pikepdf-extract
+      -
+        name: Extract jbig2enc files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/amd64
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/amd64/
+          mv jbig2enc/${version}/amd64/build/* jbig2enc/${version}/amd64/
+          docker rm jbig2enc-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/arm64
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/arm64
+          mv jbig2enc/${version}/arm64/build/* jbig2enc/${version}/arm64/
+          docker rm jbig2enc-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/armv7
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/armv7
+          mv jbig2enc/${version}/armv7/build/* jbig2enc/${version}/armv7/
+          docker rm jbig2enc-extract
+      -
+        name: Show file structure
+        run: |
+          tree .
+      -
+        name: Commit files
+        run: |
+          git config --global user.name "github-actions"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add pikepdf/ qpdf/ psycopg2/ jbig2enc/
+          git commit -m "Updating installer packages" || true
+          git push origin || true

.github/workflows/project-actions.yml

@@ -24,7 +24,7 @@ env:
 jobs:
   issue_opened_or_reopened:
     name: issue_opened_or_reopened
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: github.event_name == 'issues' && (github.event.action == 'opened' || github.event.action == 'reopened')
     steps:
       - name: Add issue to project and set status to ${{ env.todo }}
@@ -37,7 +37,7 @@ jobs:
           status_value: ${{ env.todo }} # Target status
   pr_opened_or_reopened:
     name: pr_opened_or_reopened
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       # write permission is required for autolabeler
       pull-requests: write

.github/workflows/release-chart.yml

@@ -1,31 +0,0 @@
----
-name: Release Charts
-
-on:
-  push:
-    tags:
-      - v*
-
-jobs:
-  release_chart:
-    name: "Release Chart"
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Configure Git
-        run: |
-          git config user.name "$GITHUB_ACTOR"
-          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-      - name: Install Helm
-        uses: azure/setup-helm@v3
-        with:
-          version: v3.10.0
-
-      - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.4.1
-        env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/reusable-workflow-builder.yml

@@ -13,6 +13,10 @@ on:
         required: false
         default: ""
         type: string
+      build-platforms:
+        required: false
+        default: linux/amd64,linux/arm64,linux/arm/v7
+        type: string
 
 concurrency:
   group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
@@ -21,7 +25,7 @@ concurrency:
 jobs:
   build-image:
     name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout
@@ -41,12 +45,12 @@ jobs:
         uses: docker/setup-qemu-action@v2
       -
         name: Build ${{ fromJSON(inputs.build-json).name }}
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: ${{ inputs.dockerfile }}
           tags: ${{ fromJSON(inputs.build-json).image_tag }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: ${{ inputs.build-platforms }}
           build-args: ${{ inputs.build-args }}
           push: true
           cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}

.pre-commit-config.yaml

@@ -5,20 +5,19 @@
 repos:
   # General hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: check-docstring-first
       - id: check-json
         exclude: "tsconfig.*json"
       - id: check-yaml
-        exclude: "charts/paperless-ngx/templates/common.yaml"
       - id: check-toml
       - id: check-executables-have-shebangs
       - id: end-of-file-fixer
         exclude_types:
           - svg
           - pofile
-        exclude: "^(LICENSE|charts/paperless-ngx/README.md)$"
+        exclude: "(^LICENSE$)"
       - id: mixed-line-ending
         args:
           - "--fix=lf"
@@ -35,7 +34,7 @@ repos:
           - javascript
           - ts
           - markdown
-        exclude: "(^Pipfile\\.lock$)|(^charts/paperless-ngx/README.md$)"
+        exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/asottile/reorder_python_imports
     rev: v3.9.0
@@ -48,23 +47,23 @@ repos:
       - id: yesqa
         exclude: "(migrations)"
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: "v2.3.0"
+    rev: "v2.4.0"
     hooks:
       - id: add-trailing-comma
         exclude: "(migrations)"
   - repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
       - id: flake8
         files: ^src/
         args:
           - "--config=./src/setup.cfg"
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 22.12.0
     hooks:
       - id: black
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.2.2
+    rev: v3.3.1
     hooks:
       - id: pyupgrade
         exclude: "(migrations)"
@@ -83,6 +82,6 @@ repos:
         args:
           - "--tab"
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: "v0.8.0.4"
+    rev: "v0.9.0.2"
     hooks:
       - id: shellcheck

.python-version

@@ -0,0 +1 @@
+3.8.16

Dockerfile

@@ -1,26 +1,12 @@
 # syntax=docker/dockerfile:1.4
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md
 
-# Pull the installer images from the library
-# These are all built previously
-# They provide either a .deb or .whl
-
-ARG JBIG2ENC_VERSION
-ARG QPDF_VERSION
-ARG PIKEPDF_VERSION
-ARG PSYCOPG2_VERSION
-
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
-
+# Stage: compile-frontend
+# Purpose: Compiles the frontend
+# Notes:
+#  - Does NPM stuff with Typescript and such
 FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
 
-# This stage compiles the frontend
-# This stage runs once for the native platform, as the outputs are not
-# dependent on target arch
-# Inputs: None
-
 COPY ./src-ui /src/src-ui
 
 WORKDIR /src/src-ui
@@ -30,25 +16,27 @@ RUN set -eux \
 RUN set -eux \
   && ./node_modules/.bin/ng build --configuration production
 
+# Stage: pipenv-base
+# Purpose: Generates a requirements.txt file for building
+# Comments:
+#  - pipenv dependencies are not left in the final image
+#  - pipenv can't touch the final image somehow
 FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base
 
-# This stage generates the requirements.txt file using pipenv
-# This stage runs once for the native platform, as the outputs are not
-# dependent on target arch
-# This way, pipenv dependencies are not left in the final image
-# nor can pipenv mess up the final image somehow
-# Inputs: None
-
 WORKDIR /usr/src/pipenv
 
 COPY Pipfile* ./
 
 RUN set -eux \
   && echo "Installing pipenv" \
-    && python3 -m pip install --no-cache-dir --upgrade pipenv \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv==2022.11.30 \
   && echo "Generating requirement.txt" \
     && pipenv requirements > requirements.txt
 
+# Stage: main-app
+# Purpose: The final image
+# Comments:
+#  - Don't leave anything extra in here
 FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
@@ -58,24 +46,23 @@ LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-n
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
 ARG DEBIAN_FRONTEND=noninteractive
+# Buildx provided, must be defined to use though
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+# Workflow provided
+ARG JBIG2ENC_VERSION
+ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION
 
 #
 # Begin installation and configuration
 # Order the steps below from least often changed to most
 #
 
-# copy jbig2enc
-# Basically will never change again
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
-
 # Packages need for running
 ARG RUNTIME_PACKAGES="\
-  # Python
-  python3 \
-  python3-pip \
-  python3-setuptools \
   # General utils
   curl \
   # Docker specific
@@ -139,7 +126,7 @@ RUN set -eux \
     && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
     && rm -rf /var/lib/apt/lists/* \
   && echo "Installing supervisor" \
-    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.4
+    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.5
 
 # Copy gunicorn config
 # Changes very infrequently
@@ -148,7 +135,6 @@ WORKDIR /usr/src/paperless/
 COPY gunicorn.conf.py .
 
 # setup docker-specific things
-# Use mounts to avoid copying installer files into the image
 # These change sometimes, but rarely
 WORKDIR /usr/src/paperless/src/docker/
 
@@ -159,6 +145,7 @@ COPY [ \
   "docker/docker-prepare.sh", \
   "docker/paperless_cmd.sh", \
   "docker/wait-for-redis.py", \
+  "docker/env-from-file.sh", \
   "docker/management_script.sh", \
   "docker/flower-conditional.sh", \
   "docker/install_management_commands.sh", \
@@ -178,6 +165,8 @@ RUN set -eux \
     && chmod 755 /sbin/docker-prepare.sh \
     && mv wait-for-redis.py /sbin/wait-for-redis.py \
     && chmod 755 /sbin/wait-for-redis.py \
+    && mv env-from-file.sh /sbin/env-from-file.sh \
+    && chmod 755 /sbin/env-from-file.sh \
     && mv paperless_cmd.sh /usr/local/bin/paperless_cmd.sh \
     && chmod 755 /usr/local/bin/paperless_cmd.sh \
     && mv flower-conditional.sh /usr/local/bin/flower-conditional.sh \
@@ -187,25 +176,31 @@ RUN set -eux \
     && ./install_management_commands.sh
 
 # Install the built packages from the installer library images
-# Use mounts to avoid copying installer files into the image
 # These change sometimes
-RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
-    --mount=type=bind,from=psycopg2-builder,target=/psycopg2 \
-    --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
-  set -eux \
+RUN set -eux \
+  && echo "Getting binaries" \
+    && mkdir paperless-ngx \
+    && curl --fail --silent --show-error --output paperless-ngx.tar.gz --location https://github.com/paperless-ngx/paperless-ngx/archive/41d6e7e407af09a0882736d50c89b6e015997bff.tar.gz \
+    && tar -xf paperless-ngx.tar.gz --directory paperless-ngx --strip-components=1 \
+    && cd paperless-ngx \
+    # Setting a specific revision ensures we know what this installed
+    # and ensures cache breaking on changes
+  && echo "Installing jbig2enc" \
+    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/jbig2 /usr/local/bin/ \
+    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/libjbig2enc* /usr/local/lib/ \
   && echo "Installing qpdf" \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf29_*.deb \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
+    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
+    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
   && echo "Installing pikepdf and dependencies" \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \
+    && python3 -m pip install --no-cache-dir ./pikepdf/${PIKEPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.whl \
     && python3 -m pip list \
   && echo "Installing psycopg2" \
-    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
-    && python3 -m pip list
+    && python3 -m pip install --no-cache-dir ./psycopg2/${PSYCOPG2_VERSION}/${TARGETARCH}${TARGETVARIANT}/psycopg2*.whl \
+    && python3 -m pip list \
+  && echo "Cleaning up image layer" \
+    && cd ../ \
+    && rm -rf paperless-ngx \
+    && rm paperless-ngx.tar.gz
 
 WORKDIR /usr/src/paperless/src/
 
@@ -228,6 +223,10 @@ RUN set -eux \
     && python3 -m pip install --no-cache-dir --upgrade wheel \
   && echo "Installing Python requirements" \
     && python3 -m pip install --default-timeout=1000 --no-cache-dir --requirement requirements.txt \
+  && echo "Installing NLTK data" \
+    && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" snowball_data \
+    && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" stopwords \
+    && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" punkt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
@@ -245,11 +244,12 @@ COPY ./src ./
 COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
 
 # add users, setup scripts
+# Mount the compiled frontend to expected location
 RUN set -eux \
   && addgroup --gid 1000 paperless \
   && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
-  && chown -R paperless:paperless ../ \
-  && gosu paperless python3 manage.py collectstatic --clear --no-input \
+  && chown -R paperless:paperless /usr/src/paperless \
+  && gosu paperless python3 manage.py collectstatic --clear --no-input --link \
   && gosu paperless python3 manage.py compilemessages
 
 VOLUME ["/usr/src/paperless/data", \

Pipfile

@@ -12,6 +12,7 @@ name = "piwheels"
 dateparser = "~=1.1"
 django = "~=4.1"
 django-cors-headers = "*"
+django-compression-middleware = "*"
 django-extensions = "*"
 django-filter = "~=22.1"
 djangorestframework = "~=3.14"
@@ -30,8 +31,6 @@ psycopg2 = "*"
 rapidfuzz = "*"
 redis = {extras = ["hiredis"], version = "*"}
 scikit-learn = "~=1.1"
-# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
-scipy = "==1.8.1"
 numpy = "*"
 whitenoise = "~=6.2"
 watchdog = "~=2.1"
@@ -43,15 +42,9 @@ tika = "*"
 # TODO: This will sadly also install daphne+dependencies,
 #  which an ASGI server we don't need. Adds about 15MB image size.
 channels = "~=3.0"
-# Locked version until https://github.com/django/channels_redis/issues/332
-# is resolved
-channels-redis = "==3.4.1"
 uvicorn = {extras = ["standard"], version = "*"}
 concurrent-log-handler = "*"
 "pdfminer.six" = "*"
-"backports.zoneinfo" = {version = "*", markers = "python_version < '3.9'"}
-"importlib-resources" = {version = "*", markers = "python_version < '3.9'"}
-zipp = {version = "*", markers = "python_version < '3.9'"}
 pyzbar = "*"
 mysqlclient = "*"
 celery = {extras = ["redis"], version = "*"}
@@ -60,20 +53,29 @@ setproctitle = "*"
 nltk = "*"
 pdf2image = "*"
 flower = "*"
+bleach = "*"
+#
+# Packages locked due to issues (try to check if these are fixed in a release every so often)
+#
+# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
+scipy = "==1.8.1"
+# Newer versions aren't builting yet (see https://www.piwheels.org/project/cryptography/)
+cryptography = "==38.0.1"
+# Locked version until https://github.com/django/channels_redis/issues/332
+# is resolved
+channels-redis = "==3.4.1"
+
 
 [dev-packages]
 coveralls = "*"
 factory-boy = "*"
-pycodestyle = "*"
 pytest = "*"
 pytest-cov = "*"
 pytest-django = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-tox = "*"
 black = "*"
 pre-commit = "*"
-sphinx-autobuild = "*"
-myst-parser = "*"
+imagehash = "*"
 mkdocs-material = "*"

README.md

@@ -1,7 +1,7 @@
 [![ci](https://github.com/paperless-ngx/paperless-ngx/workflows/ci/badge.svg)](https://github.com/paperless-ngx/paperless-ngx/actions)
 [![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx)
 [![Documentation Status](https://img.shields.io/github/deployments/paperless-ngx/paperless-ngx/github-pages?label=docs)](https://docs.paperless-ngx.com)
-[![Coverage Status](https://coveralls.io/repos/github/paperless-ngx/paperless-ngx/badge.svg?branch=master)](https://coveralls.io/github/paperless-ngx/paperless-ngx?branch=master)
+[![codecov](https://codecov.io/gh/paperless-ngx/paperless-ngx/branch/main/graph/badge.svg?token=VK6OUPJ3TY)](https://codecov.io/gh/paperless-ngx/paperless-ngx)
 [![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org)
 [![demo](https://cronitor.io/badges/ve7ItY/production/W5E_B9jkelG9ZbDiNHUPQEVH3MY.svg)](https://demo.paperless-ngx.com)
 
@@ -108,15 +108,6 @@ Paperless has been around a while now, and people are starting to build stuff on
 - [Scan to Paperless](https://github.com/sbrunner/scan-to-paperless): Scan and prepare (crop, deskew, OCR, ...) your documents for Paperless.
 - [Paperless Mobile](https://github.com/astubenbord/paperless-mobile): A modern, feature rich mobile application for Paperless.
 
-These projects also exist, but their status and compatibility with paperless-ngx is unknown.
-
-- [paperless-cli](https://github.com/stgarf/paperless-cli): A golang command line binary to interact with a Paperless instance.
-
-This project also exists, but needs updates to be compatible with paperless-ngx.
-
-- [Paperless Desktop](https://github.com/thomasbrueggemann/paperless-desktop): A desktop UI for your Paperless installation. Runs on Mac, Linux, and Windows.
-  Known issues on Mac: (Could not load reminders and documents)
-
 # Important Note
 
 Document scanners are typically used to scan sensitive documents. Things like your social insurance number, tax records, invoices, etc. Everything is stored in the clear without encryption. This means that Paperless should never be run on an untrusted host. Instead, I recommend that if you do want to use it, run it locally on a server in your own home.

build-docker-image.sh

@@ -10,9 +10,9 @@
 # Example Usage:
 #	./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature
 
-set -eux
+set -eu
 
-if ! command -v jq;  then
+if ! command -v jq &> /dev/null ;  then
 	echo "jq required"
 	exit 1
 elif [ ! -f "$1" ]; then
@@ -20,28 +20,62 @@ elif [ ! -f "$1" ]; then
 	exit 1
 fi
 
-# Parse what we can from Pipfile.lock
-pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
-psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
-pillow_version=$(jq ".default.pillow.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
-lxml_version=$(jq ".default.lxml.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
-# Read this from the other config file
-qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
-jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
 # Get the branch name (used for caching)
 branch_name=$(git rev-parse --abbrev-ref HEAD)
 
-# https://docs.docker.com/develop/develop-images/build_enhancements/
-# Required to use cache-from
-export DOCKER_BUILDKIT=1
+# Parse eithe Pipfile.lock or the .build-config.json
+jbig2enc_version=$(jq -r '.jbig2enc.version' .build-config.json)
+qpdf_version=$(jq -r '.qpdf.version' .build-config.json)
+psycopg2_version=$(jq -r '.default.psycopg2.version | gsub("=";"")' Pipfile.lock)
+pikepdf_version=$(jq -r '.default.pikepdf.version | gsub("=";"")' Pipfile.lock)
+pillow_version=$(jq -r '.default.pillow.version | gsub("=";"")' Pipfile.lock)
+lxml_version=$(jq -r '.default.lxml.version | gsub("=";"")' Pipfile.lock)
 
-docker build --file "$1" \
+base_filename="$(basename -- "${1}")"
+build_args_str=""
+cache_from_str=""
+
+case "${base_filename}" in
+
+	*.jbig2enc)
+		build_args_str="--build-arg JBIG2ENC_VERSION=${jbig2enc_version}"
+		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/jbig2enc:${jbig2enc_version}"
+		;;
+
+	*.psycopg2)
+		build_args_str="--build-arg PSYCOPG2_VERSION=${psycopg2_version}"
+		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/psycopg2:${psycopg2_version}"
+		;;
+
+	*.qpdf)
+		build_args_str="--build-arg QPDF_VERSION=${qpdf_version}"
+		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/qpdf:${qpdf_version}"
+		;;
+
+	*.pikepdf)
+		build_args_str="--build-arg QPDF_VERSION=${qpdf_version} --build-arg PIKEPDF_VERSION=${pikepdf_version} --build-arg PILLOW_VERSION=${pillow_version} --build-arg LXML_VERSION=${lxml_version}"
+		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/pikepdf:${pikepdf_version}"
+		;;
+
+	Dockerfile)
+		build_args_str="--build-arg QPDF_VERSION=${qpdf_version} --build-arg PIKEPDF_VERSION=${pikepdf_version} --build-arg PSYCOPG2_VERSION=${psycopg2_version} --build-arg JBIG2ENC_VERSION=${jbig2enc_version}"
+		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:${branch_name} --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev"
+		;;
+
+	*)
+		echo "Unable to match ${base_filename}"
+		exit 1
+		;;
+esac
+
+read -r -a build_args_arr <<< "${build_args_str}"
+read -r -a cache_from_arr <<< "${cache_from_str}"
+
+set -eux
+
+docker buildx build --file "${1}" \
 	--progress=plain \
-	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
-	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
-	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
-	--build-arg QPDF_VERSION="${qpdf_version}" \
-	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
-	--build-arg PILLOW_VERSION="${pillow_version}" \
-	--build-arg LXML_VERSION="${lxml_version}" \
-	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" .
+	--output=type=docker \
+	"${cache_from_arr[@]}" \
+	"${build_args_arr[@]}" \
+	"${@:2}" .

charts/paperless-ngx/.helmignore

@@ -1,26 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
-# OWNERS file for Kubernetes
-OWNERS
-# helm-docs templates
-*.gotmpl

charts/paperless-ngx/Chart.yaml

@@ -1,35 +0,0 @@
----
-apiVersion: v2
-appVersion: "1.9.2"
-description: Paperless-ngx - Index and archive all of your scanned paper documents
-name: paperless
-version: 10.0.0
-kubeVersion: ">=1.16.0-0"
-keywords:
-  - paperless
-  - paperless-ngx
-  - dms
-  - document
-home: https://github.com/paperless-ngx/paperless-ngx/tree/main/charts/paperless-ngx
-icon: https://github.com/paperless-ngx/paperless-ngx/raw/main/resources/logo/web/svg/square.svg
-sources:
-  - https://github.com/paperless-ngx/paperless-ngx
-maintainers:
-  - name: Paperless-ngx maintainers
-dependencies:
-  - name: common
-    repository: https://library-charts.k8s-at-home.com
-    version: 4.5.2
-  - name: postgresql
-    version: 11.6.12
-    repository: https://charts.bitnami.com/bitnami
-    condition: postgresql.enabled
-  - name: redis
-    version: 16.13.1
-    repository: https://charts.bitnami.com/bitnami
-    condition: redis.enabled
-deprecated: false
-annotations:
-  artifacthub.io/changes: |
-    - kind: changed
-      description: Moved to Paperless-ngx ownership

charts/paperless-ngx/LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2020 k8s@Home
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

charts/paperless-ngx/README.md

@@ -1,50 +0,0 @@
-# paperless
-
-![Version: 10.0.0](https://img.shields.io/badge/Version-10.0.0-informational?style=flat-square) ![AppVersion: 1.9.2](https://img.shields.io/badge/AppVersion-1.9.2-informational?style=flat-square)
-
-Paperless-ngx - Index and archive all of your scanned paper documents
-
-**Homepage:** <https://github.com/paperless-ngx/paperless-ngx/tree/main/charts/paperless-ngx>
-
-## Maintainers
-
-| Name | Email | Url |
-| ---- | ------ | --- |
-| Paperless-ngx maintainers |  |  |
-
-## Source Code
-
-* <https://github.com/paperless-ngx/paperless-ngx>
-
-## Requirements
-
-Kubernetes: `>=1.16.0-0`
-
-| Repository | Name | Version |
-|------------|------|---------|
-| https://charts.bitnami.com/bitnami | postgresql | 11.6.12 |
-| https://charts.bitnami.com/bitnami | redis | 16.13.1 |
-| https://library-charts.k8s-at-home.com | common | 4.5.2 |
-
-## Values
-
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| env | object | See below | See the following files for additional environment variables: https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose/ https://github.com/paperless-ngx/paperless-ngx/blob/main/paperless.conf.example |
-| env.COMPOSE_PROJECT_NAME | string | `"paperless"` | Project name |
-| env.PAPERLESS_DBHOST | string | `nil` | Database host to use |
-| env.PAPERLESS_OCR_LANGUAGE | string | `"eng"` | OCR languages to install |
-| env.PAPERLESS_PORT | int | `8000` | Port to use |
-| env.PAPERLESS_REDIS | string | `nil` | Redis to use |
-| image.pullPolicy | string | `"IfNotPresent"` | image pull policy |
-| image.repository | string | `"ghcr.io/paperless-ngx/paperless-ngx"` | image repository |
-| image.tag | string | chart.appVersion | image tag |
-| ingress.main | object | See values.yaml | Enable and configure ingress settings for the chart under this key. |
-| persistence.consume | object | See values.yaml | Configure volume to monitor for new documents. |
-| persistence.data | object | See values.yaml | Configure persistence for data. |
-| persistence.export | object | See values.yaml | Configure export volume. |
-| persistence.media | object | See values.yaml | Configure persistence for media. |
-| postgresql | object | See values.yaml | Enable and configure postgresql database subchart under this key.    For more options see [postgresql chart documentation](https://github.com/bitnami/charts/tree/master/bitnami/postgresql) |
-| redis | object | See values.yaml | Enable and configure redis subchart under this key.    For more options see [redis chart documentation](https://github.com/bitnami/charts/tree/master/bitnami/redis) |
-| service | object | See values.yaml | Configures service settings for the chart. |
-

charts/paperless-ngx/README_CONFIG.md.gotmpl

@@ -1,8 +0,0 @@
-{{- define "custom.custom.configuration.header" -}}
-## Custom configuration
-{{- end -}}
-
-{{- define "custom.custom.configuration" -}}
-{{ template "custom.custom.configuration.header" . }}
-N/A
-{{- end -}}

charts/paperless-ngx/ci/ct-values.yaml

@@ -1,26 +0,0 @@
-env:
-  PAPERLESS_REDIS: redis://paperless-redis-headless:6379
-
-persistence:
-  data:
-    enabled: true
-    type: emptyDir
-  media:
-    enabled: true
-    type: emptyDir
-  consume:
-    enabled: true
-    type: emptyDir
-  export:
-    enabled: true
-    type: emptyDir
-
-redis:
-  enabled: true
-  architecture: standalone
-  auth:
-    enabled: false
-  master:
-    persistence:
-      enabled: false
-  fullnameOverride: paperless-redis

charts/paperless-ngx/templates/NOTES.txt

@@ -1,4 +0,0 @@
-{{- include "common.notes.defaultNotes" . }}
-2. Create a super user by running the command:
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "common.names.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  kubectl exec -it --namespace {{ .Release.Namespace }} $POD_NAME -- bash -c "python manage.py createsuperuser"

charts/paperless-ngx/templates/common.yaml

@@ -1,11 +0,0 @@
-{{/* Make sure all variables are set properly */}}
-{{- include "common.values.setup" . }}
-
-{{/* Append the hardcoded settings */}}
-{{- define "paperless.harcodedValues" -}}
-env:
-    PAPERLESS_URL: http{{if ne ( len .Values.ingress.main.tls ) 0 }}s{{end}}://{{ (first .Values.ingress.main.hosts).host }}
-{{- end -}}
-{{- $_ := merge .Values (include "paperless.harcodedValues" . | fromYaml) -}}
-
-{{ include "common.all" . }}

charts/paperless-ngx/values.yaml

@@ -1,107 +0,0 @@
-#
-# IMPORTANT NOTE
-#
-# This chart inherits from our common library chart. You can check the default values/options here:
-# https://github.com/k8s-at-home/library-charts/tree/main/charts/stable/common/values.yaml
-#
-
-image:
-  # -- image repository
-  repository: ghcr.io/paperless-ngx/paperless-ngx
-  # -- image pull policy
-  pullPolicy: IfNotPresent
-  # -- image tag
-  # @default -- chart.appVersion
-  tag:
-
-# -- See the following files for additional environment variables:
-# https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose/
-# https://github.com/paperless-ngx/paperless-ngx/blob/main/paperless.conf.example
-# @default -- See below
-env:
-  # -- Project name
-  COMPOSE_PROJECT_NAME: paperless
-  # -- Redis to use
-  PAPERLESS_REDIS:
-  # -- OCR languages to install
-  PAPERLESS_OCR_LANGUAGE: eng
-  # USERMAP_UID: 1000
-  # USERMAP_GID: 1000
-  # PAPERLESS_TIME_ZONE: Europe/London
-  # -- Database host to use
-  PAPERLESS_DBHOST:
-  # -- Port to use
-  PAPERLESS_PORT: 8000
-  # -- Username for the root user
-  # PAPERLESS_ADMIN_USER: admin
-  # -- Password for the root user
-  # PAPERLESS_ADMIN_PASSWORD: admin
-  # PAPERLESS_URL: <set to main ingress by default>
-
-# -- Configures service settings for the chart.
-# @default -- See values.yaml
-service:
-  main:
-    ports:
-      http:
-        port: 8000
-
-ingress:
-  # -- Enable and configure ingress settings for the chart under this key.
-  # @default -- See values.yaml
-  main:
-    enabled: false
-
-persistence:
-  # -- Configure persistence for data.
-  # @default -- See values.yaml
-  data:
-    enabled: false
-    mountPath: /usr/src/paperless/data
-    accessMode: ReadWriteOnce
-    emptyDir:
-      enabled: false
-  # -- Configure persistence for media.
-  # @default -- See values.yaml
-  media:
-    enabled: false
-    mountPath: /usr/src/paperless/media
-    accessMode: ReadWriteOnce
-    emptyDir:
-      enabled: false
-  # -- Configure volume to monitor for new documents.
-  # @default -- See values.yaml
-  consume:
-    enabled: false
-    mountPath: /usr/src/paperless/consume
-    accessMode: ReadWriteOnce
-    emptyDir:
-      enabled: false
-  # -- Configure export volume.
-  # @default -- See values.yaml
-  export:
-    enabled: false
-    mountPath: /usr/src/paperless/export
-    accessMode: ReadWriteOnce
-    emptyDir:
-      enabled: false
-
-# -- Enable and configure postgresql database subchart under this key.
-#    For more options see [postgresql chart documentation](https://github.com/bitnami/charts/tree/master/bitnami/postgresql)
-# @default -- See values.yaml
-postgresql:
-  enabled: false
-  postgresqlUsername: paperless
-  postgresqlPassword: paperless
-  postgresqlDatabase: paperless
-  persistence:
-    enabled: false
-    # storageClass: ""
-
-# -- Enable and configure redis subchart under this key.
-#    For more options see [redis chart documentation](https://github.com/bitnami/charts/tree/master/bitnami/redis)
-# @default -- See values.yaml
-redis:
-  enabled: false
-  auth:
-    enabled: false

docker-builders/Dockerfile.jbig2enc

@@ -29,7 +29,20 @@ RUN set -eux \
     && ./autogen.sh \
     && ./configure \
     && make \
+  && echo "Gathering package data" \
+    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./pkg-list.txt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+  && echo "Moving files around" \
+    && mkdir build \
+    # Unlink a symlink that causes problems
+    && unlink ./src/.libs/libjbig2enc.la \
+    # Move what the link pointed to
+    && mv ./src/libjbig2enc.la ./build/ \
+    # Move the shared library .so files
+    && mv ./src/.libs/libjbig2enc* ./build/ \
+    # And move the cli binary
+    && mv ./src/jbig2 ./build/ \
+    && mv ./pkg-list.txt ./build/

docker-builders/Dockerfile.pikepdf

@@ -7,16 +7,27 @@
 # Default to pulling from the main repo registry when manually building
 ARG REPO="paperless-ngx/paperless-ngx"
 
-ARG QPDF_VERSION
-FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
-
 # This does nothing, except provide a name for a copy below
+ARG QPDF_VERSION
+FROM --platform=$BUILDPLATFORM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
 
-FROM python:3.9-slim-bullseye as main
+#
+# Stage: builder
+# Purpose:
+#  - Build the pikepdf wheel
+#  - Build any dependent wheels which can't be found
+#
+FROM python:3.9-slim-bullseye as builder
 
 LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
 
+# Buildx provided
+ARG TARGETARCH
+ARG TARGETVARIANT
+
 ARG DEBIAN_FRONTEND=noninteractive
+# Workflow provided
+ARG QPDF_VERSION
 ARG PIKEPDF_VERSION
 # These are not used, but will still bust the cache if one changes
 # Otherwise, the main image will try to build thing (and fail)
@@ -54,7 +65,7 @@ ARG BUILD_PACKAGES="\
 
 WORKDIR /usr/src
 
-COPY --from=qpdf-builder /usr/src/qpdf/*.deb ./
+COPY --from=qpdf-builder /usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.deb ./
 
 # As this is an base image for a multi-stage final image
 # the added size of the install is basically irrelevant
@@ -77,6 +88,8 @@ RUN set -eux \
     && python3 -m pip wheel \
       # Build the package at the required version
       pikepdf==${PIKEPDF_VERSION} \
+      # Look to piwheels for additional pre-built wheels
+      --extra-index-url https://www.piwheels.org/simple \
       # Output the *.whl into this directory
       --wheel-dir wheels \
       # Do not use a binary packge for the package being built
@@ -86,7 +99,20 @@ RUN set -eux \
       # Don't cache build files
       --no-cache-dir \
     && ls -ahl wheels \
+  && echo "Gathering package data" \
+    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./wheels/pkg-list.txt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
     && rm -rf /var/lib/apt/lists/*
+
+#
+# Stage: package
+# Purpose: Holds the compiled .whl files in a tiny image to pull
+#
+FROM alpine:3.17 as package
+
+WORKDIR /usr/src/wheels/
+
+COPY --from=builder /usr/src/wheels/*.whl ./
+COPY --from=builder /usr/src/wheels/pkg-list.txt ./

docker-builders/Dockerfile.psycopg2

@@ -2,7 +2,12 @@
 # Inputs:
 #    - PSYCOPG2_VERSION - Version to build
 
-FROM python:3.9-slim-bullseye as main
+#
+# Stage: builder
+# Purpose:
+#  - Build the psycopg2 wheel
+#
+FROM python:3.9-slim-bullseye as builder
 
 LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
 
@@ -42,7 +47,20 @@ RUN set -eux \
       # Don't cache build files
       --no-cache-dir \
     && ls -ahl wheels/ \
+  && echo "Gathering package data" \
+    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./wheels/pkg-list.txt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
     && rm -rf /var/lib/apt/lists/*
+
+#
+# Stage: package
+# Purpose: Holds the compiled .whl files in a tiny image to pull
+#
+FROM alpine:3.17 as package
+
+WORKDIR /usr/src/wheels/
+
+COPY --from=builder /usr/src/wheels/*.whl ./
+COPY --from=builder /usr/src/wheels/pkg-list.txt ./

docker-builders/Dockerfile.qpdf

@@ -1,48 +1,156 @@
-# This Dockerfile compiles the jbig2enc library
-# Inputs:
-#    - QPDF_VERSION - the version of qpdf to build a .deb.
-#                     Must be present as a deb-src in bookworm
+#
+# Stage: pre-build
+# Purpose:
+#  - Installs common packages
+#  - Sets common environment variables related to dpkg
+#  - Aquires the qpdf source from bookwork
+# Useful Links:
+#  - https://qpdf.readthedocs.io/en/stable/installation.html#system-requirements
+#  - https://wiki.debian.org/Multiarch/HOWTO
+#  - https://wiki.debian.org/CrossCompiling
+#
 
-FROM debian:bullseye-slim as main
+FROM debian:bullseye-slim as pre-build
 
-LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
-
-ARG DEBIAN_FRONTEND=noninteractive
-# This must match to pikepdf's minimum at least
 ARG QPDF_VERSION
 
-ARG BUILD_PACKAGES="\
-  build-essential \
-  debhelper \
+ARG COMMON_BUILD_PACKAGES="\
+  cmake \
+  debhelper\
   debian-keyring \
   devscripts \
-  equivs  \
-  libtool \
-  # https://qpdf.readthedocs.io/en/stable/installation.html#system-requirements
-  libjpeg62-turbo-dev \
-  libgnutls28-dev \
+  dpkg-dev \
+  equivs \
   packaging-dev \
-  cmake \
-  zlib1g-dev"
+  libtool"
+
+ENV DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2"
 
 WORKDIR /usr/src
 
 RUN set -eux \
-  && echo "Installing build tools" \
+  && echo "Installing common packages" \
     && apt-get update --quiet \
-    && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
-  && echo "Getting qpdf src" \
+    && apt-get install --yes --quiet --no-install-recommends ${COMMON_BUILD_PACKAGES} \
+  && echo "Getting qpdf source" \
     && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
-    && apt-get update \
-    && mkdir qpdf \
-    && cd qpdf \
-    && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
-  && echo "Building qpdf" \
-    && cd qpdf-$QPDF_VERSION \
-    && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
-    && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean \
-    && ls -ahl ../*.deb \
-  && echo "Cleaning up image" \
-    && apt-get -y purge ${BUILD_PACKAGES} \
-    && apt-get -y autoremove --purge \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get update --quiet \
+    && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm
+
+#
+# Stage: amd64-builder
+# Purpose: Builds qpdf for x86_64 (native build)
+#
+FROM pre-build as amd64-builder
+
+ARG AMD64_BUILD_PACKAGES="\
+  build-essential \
+  libjpeg62-turbo-dev:amd64 \
+  libgnutls28-dev:amd64 \
+  zlib1g-dev:amd64"
+
+WORKDIR /usr/src/qpdf-${QPDF_VERSION}
+
+RUN set -eux \
+  && echo "Beginning amd64" \
+    && echo "Install amd64 packages" \
+      && apt-get update --quiet \
+      && apt-get install --yes --quiet --no-install-recommends ${AMD64_BUILD_PACKAGES} \
+    && echo "Building amd64" \
+      && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean \
+    && echo "Removing debug files" \
+      && rm -f ../libqpdf29-dbgsym* \
+      && rm -f ../qpdf-dbgsym* \
+    && echo "Gathering package data" \
+      && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt
+#
+# Stage: armhf-builder
+# Purpose:
+#  - Sets armhf specific environment
+#  - Builds qpdf for armhf (cross compile)
+#
+FROM pre-build as armhf-builder
+
+ARG ARMHF_PACKAGES="\
+  crossbuild-essential-armhf \
+  libjpeg62-turbo-dev:armhf \
+  libgnutls28-dev:armhf \
+  zlib1g-dev:armhf"
+
+WORKDIR /usr/src/qpdf-${QPDF_VERSION}
+
+ENV CXX="/usr/bin/arm-linux-gnueabihf-g++" \
+    CC="/usr/bin/arm-linux-gnueabihf-gcc"
+
+RUN set -eux \
+  && echo "Beginning armhf" \
+    && echo "Install armhf packages" \
+      && dpkg --add-architecture armhf \
+      && apt-get update --quiet \
+      && apt-get install --yes --quiet --no-install-recommends ${ARMHF_PACKAGES} \
+    && echo "Building armhf" \
+      && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean --host-arch armhf \
+    && echo "Removing debug files" \
+      && rm -f ../libqpdf29-dbgsym* \
+      && rm -f ../qpdf-dbgsym* \
+    && echo "Gathering package data" \
+      && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt
+
+#
+# Stage: aarch64-builder
+# Purpose:
+#  - Sets aarch64 specific environment
+#  - Builds qpdf for aarch64 (cross compile)
+#
+FROM pre-build as aarch64-builder
+
+ARG ARM64_PACKAGES="\
+  crossbuild-essential-arm64 \
+  libjpeg62-turbo-dev:arm64 \
+  libgnutls28-dev:arm64 \
+  zlib1g-dev:arm64"
+
+ENV CXX="/usr/bin/aarch64-linux-gnu-g++" \
+    CC="/usr/bin/aarch64-linux-gnu-gcc"
+
+WORKDIR /usr/src/qpdf-${QPDF_VERSION}
+
+RUN set -eux \
+  && echo "Beginning arm64" \
+    && echo "Install arm64 packages" \
+      && dpkg --add-architecture arm64 \
+      && apt-get update --quiet \
+      && apt-get install --yes --quiet --no-install-recommends ${ARM64_PACKAGES} \
+    && echo "Building arm64" \
+      && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean --host-arch arm64 \
+    && echo "Removing debug files" \
+      && rm -f ../libqpdf29-dbgsym* \
+      && rm -f ../qpdf-dbgsym* \
+    && echo "Gathering package data" \
+      && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt
+
+#
+# Stage: package
+# Purpose: Holds the compiled .deb files in arch/variant specific folders
+#
+FROM alpine:3.17 as package
+
+LABEL org.opencontainers.image.description="A image with qpdf installers stored in architecture & version specific folders"
+
+ARG QPDF_VERSION
+
+WORKDIR /usr/src/qpdf/${QPDF_VERSION}/amd64
+
+COPY --from=amd64-builder /usr/src/*.deb ./
+COPY --from=amd64-builder /usr/src/pkg-list.txt ./
+
+# Note this is ${TARGETARCH}${TARGETVARIANT} for armv7
+WORKDIR /usr/src/qpdf/${QPDF_VERSION}/armv7
+
+COPY --from=armhf-builder /usr/src/*.deb ./
+COPY --from=armhf-builder /usr/src/pkg-list.txt ./
+
+WORKDIR /usr/src/qpdf/${QPDF_VERSION}/arm64
+
+COPY --from=aarch64-builder /usr/src/*.deb ./
+COPY --from=aarch64-builder /usr/src/pkg-list.txt ./

docker-builders/README.md

@@ -0,0 +1,57 @@
+# Installer Library
+
+This folder contains the Dockerfiles for building certain installers or libraries, which are then pulled into the main image.
+
+## [jbig2enc](https://github.com/agl/jbig2enc)
+
+### Why
+
+JBIG is an image coding which can achieve better compression of images for PDFs.
+
+### What
+
+The Docker image builds a shared library file and utility, which is copied into the correct location in the final image.
+
+### Updating
+
+1. Ensure the given qpdf version is present in [Debian bookworm](https://packages.debian.org/bookworm/qpdf)
+2. Update `.build-config.json` to the given version
+3. If the Debian specific version has incremented, update `Dockerfile.qpdf`
+
+See Also:
+
+- [OCRMyPDF Documentation](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html)
+
+## [psycopg2](https://www.psycopg.org/)
+
+### Why
+
+The pre-built wheels of psycopg2 are built on Debian 9, which provides a quite old version of libpq-dev. This causes issue with authentication methods.
+
+### What
+
+The image builds psycopg2 wheels on Debian 10 and places the produced wheels into `/usr/src/wheels/`.
+
+See Also:
+
+- [Issue 266](https://github.com/paperless-ngx/paperless-ngx/issues/266)
+
+## [qpdf](https://qpdf.readthedocs.io/en/stable/index.html)
+
+### Why
+
+qpdf and it's library provide tools to read, manipulate and fix up PDFs. Version 11 is also required by `pikepdf` 6+ and Debian 9 does not provide above version 10.
+
+### What
+
+The Docker image cross compiles .deb installers for each supported architecture of the main image. The installers are placed in `/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/`
+
+## [pikepdf](https://pikepdf.readthedocs.io/en/latest/)
+
+### Why
+
+Required by OCRMyPdf, this is a general purpose library for PDF manipulation in Python via the qpdf libraries.
+
+### What
+
+The built wheels are placed into `/usr/src/wheels/`

docker/compose/docker-compose.ci-test.yml

@@ -6,14 +6,17 @@
 version: "3.7"
 services:
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     hostname: gotenberg
     container_name: gotenberg
     network_mode: host
     restart: unless-stopped
+    # The gotenberg chromium route is used to convert .eml files. We do not
+    # want to allow external content like tracking pixels or even javascript.
     command:
       - "gotenberg"
-      - "--chromium-disable-routes=true"
+      - "--chromium-disable-javascript=true"
+      - "--chromium-allow-list=file:///tmp/.*"
   tika:
     image: ghcr.io/paperless-ngx/tika:latest
     hostname: tika

docker/compose/docker-compose.mariadb-tika.yml

@@ -49,8 +49,6 @@ services:
       MARIADB_USER: paperless
       MARIADB_PASSWORD: paperless
       MARIADB_ROOT_PASSWORD: paperless
-    ports:
-      - "3306:3306"
 
   webserver:
     image: ghcr.io/paperless-ngx/paperless-ngx:latest
@@ -61,7 +59,7 @@ services:
       - gotenberg
       - tika
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000"]
       interval: 30s
@@ -85,11 +83,14 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     restart: unless-stopped
+    # The gotenberg chromium route is used to convert .eml files. We do not
+    # want to allow external content like tracking pixels or even javascript.
     command:
       - "gotenberg"
-      - "--chromium-disable-routes=true"
+      - "--chromium-disable-javascript=true"
+      - "--chromium-allow-list=file:///tmp/.*"
 
   tika:
     image: ghcr.io/paperless-ngx/tika:latest

docker/compose/docker-compose.mariadb.yml

@@ -45,8 +45,6 @@ services:
       MARIADB_USER: paperless
       MARIADB_PASSWORD: paperless
       MARIADB_ROOT_PASSWORD: paperless
-    ports:
-      - "3306:3306"
 
   webserver:
     image: ghcr.io/paperless-ngx/paperless-ngx:latest
@@ -55,7 +53,7 @@ services:
       - db
       - broker
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000"]
       interval: 30s

docker/compose/docker-compose.portainer.yml

@@ -53,7 +53,7 @@ services:
       - db
       - broker
     ports:
-      - 8010:8000
+      - "8010:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s

docker/compose/docker-compose.postgres-tika.yml

@@ -57,7 +57,7 @@ services:
       - gotenberg
       - tika
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
@@ -77,11 +77,15 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     restart: unless-stopped
+
+    # The gotenberg chromium route is used to convert .eml files. We do not
+    # want to allow external content like tracking pixels or even javascript.
     command:
       - "gotenberg"
-      - "--chromium-disable-routes=true"
+      - "--chromium-disable-javascript=true"
+      - "--chromium-allow-list=file:///tmp/.*"
 
   tika:
     image: ghcr.io/paperless-ngx/tika:latest

docker/compose/docker-compose.postgres.yml

@@ -51,7 +51,7 @@ services:
       - db
       - broker
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s

docker/compose/docker-compose.sqlite-tika.yml

@@ -46,7 +46,7 @@ services:
       - gotenberg
       - tika
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
@@ -65,11 +65,15 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.6
+    image: docker.io/gotenberg/gotenberg:7.8
     restart: unless-stopped
+
+    # The gotenberg chromium route is used to convert .eml files. We do not
+    # want to allow external content like tracking pixels or even javascript.
     command:
       - "gotenberg"
-      - "--chromium-disable-routes=true"
+      - "--chromium-disable-javascript=true"
+      - "--chromium-allow-list=file:///tmp/.*"
 
   tika:
     image: ghcr.io/paperless-ngx/tika:latest

docker/compose/docker-compose.sqlite.yml

@@ -37,7 +37,7 @@ services:
     depends_on:
       - broker
     ports:
-      - 8000:8000
+      - "8000:8000"
     healthcheck:
       test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s

docker/docker-entrypoint.sh

@@ -2,37 +2,6 @@
 
 set -e
 
-# Adapted from:
-# https://github.com/docker-library/postgres/blob/master/docker-entrypoint.sh
-# usage: file_env VAR
-#    ie: file_env 'XYZ_DB_PASSWORD' will allow for "$XYZ_DB_PASSWORD_FILE" to
-# fill in the value of "$XYZ_DB_PASSWORD" from a file, especially for Docker's
-# secrets feature
-file_env() {
-	local -r var="$1"
-	local -r fileVar="${var}_FILE"
-
-	# Basic validation
-	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
-		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
-		exit 1
-	fi
-
-	# Only export var if the _FILE exists
-	if [ "${!fileVar:-}" ]; then
-		# And the file exists
-		if [[ -f ${!fileVar} ]]; then
-			echo "Setting ${var} from file"
-			val="$(< "${!fileVar}")"
-			export "$var"="$val"
-		else
-			echo "File ${!fileVar} doesn't exist"
-			exit 1
-		fi
-	fi
-
-}
-
 # Source: https://github.com/sameersbn/docker-gitlab/
 map_uidgid() {
 	local -r usermap_original_uid=$(id -u paperless)
@@ -53,30 +22,6 @@ map_folders() {
 	export CONSUME_DIR="${PAPERLESS_CONSUMPTION_DIR:-/usr/src/paperless/consume}"
 }
 
-nltk_data () {
-	# Store the NLTK data outside the Docker container
-	local -r nltk_data_dir="${DATA_DIR}/nltk"
-	local -r truthy_things=("yes y 1 t true")
-
-	# If not set, or it looks truthy
-	if [[ -z "${PAPERLESS_ENABLE_NLTK}" ]] || [[ "${truthy_things[*]}" =~ ${PAPERLESS_ENABLE_NLTK,} ]]; then
-
-		# Download or update the snowball stemmer data
-		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" snowball_data
-
-		# Download or update the stopwords corpus
-		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" stopwords
-
-		# Download or update the punkt tokenizer data
-		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" punkt
-
-	else
-		echo "Skipping NLTK data download"
-
-	fi
-
-}
-
 custom_container_init() {
 	# Mostly borrowed from the LinuxServer.io base image
 	# https://github.com/linuxserver/docker-baseimage-ubuntu/tree/bionic/root/etc/cont-init.d
@@ -120,19 +65,11 @@ custom_container_init() {
 initialize() {
 
 	# Setup environment from secrets before anything else
-	for env_var in \
-		PAPERLESS_DBUSER \
-		PAPERLESS_DBPASS \
-		PAPERLESS_SECRET_KEY \
-		PAPERLESS_AUTO_LOGIN_USERNAME \
-		PAPERLESS_ADMIN_USER \
-		PAPERLESS_ADMIN_MAIL \
-		PAPERLESS_ADMIN_PASSWORD \
-		PAPERLESS_REDIS; do
-		# Check for a version of this var with _FILE appended
-		# and convert the contents to the env var value
-		file_env ${env_var}
-	done
+	# Check for a version of this var with _FILE appended
+	# and convert the contents to the env var value
+	# Source it so export is persistent
+	# shellcheck disable=SC1091
+	source /sbin/env-from-file.sh
 
 	# Change the user and group IDs if needed
 	map_uidgid
@@ -157,8 +94,6 @@ initialize() {
 	echo "Creating directory ${tmp_dir}"
 	mkdir -p "${tmp_dir}"
 
-	nltk_data
-
 	set +e
 	echo "Adjusting permissions of paperless files. This may take a while."
 	chown -R paperless:paperless ${tmp_dir}
@@ -191,10 +126,6 @@ install_languages() {
 
 	for lang in "${langs[@]}"; do
 		pkg="tesseract-ocr-$lang"
-		# English is installed by default
-		#if [[ "$lang" ==  "eng" ]]; then
-		#    continue
-		#fi
 
 		if dpkg -s "$pkg" &>/dev/null; then
 			echo "Package $pkg already installed!"

docker/docker-prepare.sh

@@ -20,7 +20,6 @@ wait_for_postgres() {
 			exit 1
 		else
 			echo "Attempt $attempt_num failed! Trying again in 5 seconds..."
-
 		fi
 
 		attempt_num=$(("$attempt_num" + 1))
@@ -37,6 +36,8 @@ wait_for_mariadb() {
 	local attempt_num=1
 	local -r max_attempts=5
 
+	# Disable warning, host and port can't have spaces
+	# shellcheck disable=SC2086
 	while ! true > /dev/tcp/$host/$port; do
 
 		if [ $attempt_num -eq $max_attempts ]; then
@@ -67,13 +68,19 @@ migrations() {
 		# of the current container starts.
 		flock 200
 		echo "Apply database migrations..."
-		python3 manage.py migrate
+		python3 manage.py migrate --skip-checks --no-input
 	) 200>"${DATA_DIR}/migration_lock"
 }
 
+django_checks() {
+	# Explicitly run the Django system checks
+	echo "Running Django checks"
+	python3 manage.py check
+}
+
 search_index() {
 
-	local -r index_version=1
+	local -r index_version=3
 	local -r index_version_file=${DATA_DIR}/.index_version
 
 	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
@@ -100,6 +107,8 @@ do_work() {
 
 	migrations
 
+	django_checks
+
 	search_index
 
 	superuser

docker/env-from-file.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Scans the environment variables for those with the suffix _FILE
+# When located, checks the file exists, and exports the contents
+# of the file as the same name, minus the suffix
+# This allows the use of Docker secrets or mounted files
+# to fill in any of the settings configurable via environment
+# variables
+
+set -eu
+
+for line in $(printenv)
+do
+	# Extract the name of the environment variable
+	env_name=${line%%=*}
+	# Check if it starts with "PAPERLESS_" and ends in "_FILE"
+	if [[ ${env_name} == PAPERLESS_*_FILE ]]; then
+		# Extract the value of the environment
+		env_value=${line#*=}
+
+		# Check the file exists
+		if [[ -f ${env_value} ]]; then
+
+			# Trim off the _FILE suffix
+			non_file_env_name=${env_name%"_FILE"}
+			echo "Setting ${non_file_env_name} from file"
+
+			# Reads the value from th file
+			val="$(< "${!env_name}")"
+
+			# Sets the normal name to the read file contents
+			export "${non_file_env_name}"="${val}"
+
+		else
+			echo "File ${env_value} referenced by ${env_name} doesn't exist"
+		fi
+	fi
+done

docker/flower-conditional.sh

@@ -3,5 +3,10 @@
 echo "Checking if we should start flower..."
 
 if [[ -n  "${PAPERLESS_ENABLE_FLOWER}" ]]; then
-	celery --app paperless flower
+	# Small delay to allow celery to be up first
+	echo "Starting flower in 5s"
+	sleep 5
+	celery --app paperless flower --conf=/usr/src/paperless/src/paperless/flowerconfig.py
+else
+	echo "Not starting flower"
 fi

docker/management_script.sh

@@ -3,6 +3,9 @@
 set -e
 
 cd /usr/src/paperless/src/
+# This ensures environment is setup
+# shellcheck disable=SC1091
+source /sbin/env-from-file.sh
 
 if [[ $(id -u) == 0 ]] ;
 then

docs/administration.md

@@ -9,7 +9,7 @@ Before making backups, make sure that paperless is not running.
 
 Options available to any installation of paperless:
 
-- Use the [document exporter](administration#exporter). The document exporter exports all your documents,
+- Use the [document exporter](#exporter). The document exporter exports all your documents,
   thumbnails and metadata to a specific folder. You may import your
   documents into a fresh instance of paperless again or store your
   documents in another DMS with this export.
@@ -52,7 +52,7 @@ Options available to bare-metal and non-docker installations:
 
 ## Updating Paperless {#updating}
 
-### Docker Route
+### Docker Route {#docker-updating}
 
 If a new release of paperless-ngx is available, upgrading depends on how
 you installed paperless-ngx in the first place. The releases are
@@ -68,23 +68,23 @@ $ docker-compose down
 
 After that, [make a backup](#backup).
 
-A. If you pull the image from the docker hub, all you need to do is:
+1. If you pull the image from the docker hub, all you need to do is:
 
-    ``` shell-session
-    $ docker-compose pull
-    $ docker-compose up
-    ```
+   ```shell-session
+   $ docker-compose pull
+   $ docker-compose up
+   ```
 
-    The docker-compose files refer to the `latest` version, which is
-    always the latest stable release.
+   The docker-compose files refer to the `latest` version, which is
+   always the latest stable release.
 
-B. If you built the image yourself, do the following:
+2. If you built the image yourself, do the following:
 
-    ``` shell-session
-    $ git pull
-    $ docker-compose build
-    $ docker-compose up
-    ```
+   ```shell-session
+   $ git pull
+   $ docker-compose build
+   $ docker-compose up
+   ```
 
 Running `docker-compose up` will also apply any new database migrations.
 If you see everything working, press CTRL+C once to gracefully stop
@@ -131,7 +131,7 @@ the background.
     image: ghcr.io/paperless-ngx/paperless-ngx:1.7
     ```
 
-### Bare Metal Route
+### Bare Metal Route {#bare-metal-updating}
 
 After grabbing the new release and unpacking the contents, do the
 following:
@@ -152,13 +152,15 @@ following:
 
     ```shell-session
     $ cd src
-    $ python3 manage.py migrate
+    $ python3 manage.py migrate # (1)
     ```
 
+    1.  Including `sudo -Hu <paperless_user>` may be required
+
     This might not actually do anything. Not every new paperless version
     comes with new database migrations.
 
-## Downgrading Paperless
+## Downgrading Paperless {#downgrade-paperless}
 
 Downgrades are possible. However, some updates also contain database
 migrations (these change the layout of the database and may move data).
@@ -210,9 +212,11 @@ Bare metal:
 
 ```shell-session
 $ cd /path/to/paperless/src
-$ python3 manage.py <command> <arguments>
+$ python3 manage.py <command> <arguments> # (1)
 ```
 
+1.  Including `sudo -Hu <paperless_user>` may be required
+
 All commands have built-in help, which can be accessed by executing them
 with the argument `--help`.
 
@@ -227,12 +231,17 @@ is not a TTY" errors. For example:
 `docker-compose exec -T webserver document_exporter ../export`
 
 ```
-document_exporter target [-c] [-f] [-d]
+document_exporter target [-c] [-d] [-f] [-na] [-nt] [-p] [-sm] [-z]
 
 optional arguments:
 -c, --compare-checksums
--f, --use-filename-format
 -d, --delete
+-f, --use-filename-format
+-na, --no-archive
+-nt, --no-thumbnail
+-p, --use-folder-prefix
+-sm, --split-manifest
+-z  --zip
 ```
 
 `target` is a folder to which the data gets written. This includes
@@ -248,25 +257,58 @@ will assume that the contents of the export directory are a previous
 export and will attempt to update the previous export. Paperless will
 only export changed and added files. Paperless determines whether a file
 has changed by inspecting the file attributes "date/time modified" and
-"size". If that does not work out for you, specify
+"size". If that does not work out for you, specify `-c` or
 `--compare-checksums` and paperless will attempt to compare file
 checksums instead. This is slower.
 
 Paperless will not remove any existing files in the export directory. If
 you want paperless to also remove files that do not belong to the
-current export such as files from deleted documents, specify `--delete`.
+current export such as files from deleted documents, specify `-d` or `--delete`.
 Be careful when pointing paperless to a directory that already contains
 other files.
 
 The filenames generated by this command follow the format
 `[date created] [correspondent] [title].[extension]`. If you want
 paperless to use `PAPERLESS_FILENAME_FORMAT` for exported filenames
-instead, specify `--use-filename-format`.
+instead, specify `-f` or `--use-filename-format`.
+
+If `-na` or `--no-archive` is provided, no archive files will be exported,
+only the original files.
+
+If `-nt` or `--no-thumbnail` is provided, thumbnail files will not be exported.
+
+!!! note
+
+    When using the `-na`/`--no-archive` or `-nt`/`--no-thumbnail` options
+    the exporter will not output these files for backup.  After importing,
+    the [sanity checker](#sanity-checker) will warn about missing thumbnails and archive files
+    until they are regenerated with `document_thumbnails` or [`document_archiver`](#archiver).
+    It can make sense to omit these files from backup as their content and checksum
+    can change (new archiver algorithm) and may then cause additional used space in
+    a deduplicated backup.
+
+If `-p` or `--use-folder-prefix` is provided, files will be exported
+in dedicated folders according to their nature: `archive`, `originals`,
+`thumbnails` or `json`
+
+If `-sm` or `--split-manifest` is provided, information about document
+will be placed in individual json files, instead of a single JSON file. The main
+manifest.json will still contain application wide information (e.g. tags, correspondent,
+documenttype, etc)
+
+If `-z` or `--zip` is provided, the export will be a zipfile
+in the target directory, named according to the current date.
+
+!!! warning
+
+    If exporting with the file name format, there may be errors due to
+    your operating system's maximum path lengths.  Try adjusting the export
+    target or consider not using the filename format.
 
 ### Document importer {#importer}
 
 The document importer takes the export produced by the [Document
-exporter](#document-exporter) and imports it into paperless.
+exporter](#exporter) and imports it into paperless.
 
 The importer works just like the exporter. You point it at a directory,
 and the script does the rest of the work:
@@ -343,6 +385,14 @@ document_create_classifier
 
 This command takes no arguments.
 
+### Document thumbnails {#thumbnails}
+
+Use this command to re-create document thumbnails. Optionally include the ` --document {id}` option to generate thumbnails for a specific document only.
+
+```
+document_thumbnails
+```
+
 ### Managing the document search index {#index}
 
 The document search index is responsible for delivering search results
@@ -366,7 +416,7 @@ task scheduler.
 ### Managing filenames {#renamer}
 
 If you use paperless' feature to
-[assign custom filenames to your documents](/advanced_usage#file_name_handling), you can use this command to move all your files after
+[assign custom filenames to your documents](/advanced_usage#file-name-handling), you can use this command to move all your files after
 changing the naming scheme.
 
 !!! warning
@@ -430,9 +480,7 @@ rules.
     As of October 2022 Microsoft no longer supports IMAP authentication
     for Exchange servers, thus Exchange is no longer supported until a
     solution is implemented in the Python IMAP library used by Paperless.
-    See
-
-[learn.microsoft.com](https://learn.microsoft.com/en-us/exchange/clients-and-mobile-in-exchange-online/deprecation-of-basic-authentication-exchange-online)
+    See [learn.microsoft.com](https://learn.microsoft.com/en-us/exchange/clients-and-mobile-in-exchange-online/deprecation-of-basic-authentication-exchange-online)
 
 ### Creating archived documents {#archiver}
 
@@ -467,13 +515,13 @@ document.
     documents, such as encrypted PDF documents. The archiver will skip over
     these documents each time it sees them.
 
-### Managing encryption {#encyption}
+### Managing encryption {#encryption}
 
 Documents can be stored in Paperless using GnuPG encryption.
 
 !!! warning
 
-    Encryption is deprecated since paperless-ngx 0.9 and doesn't really
+    Encryption is deprecated since [paperless-ng 0.9](/changelog#paperless-ng-090) and doesn't really
     provide any additional security, since you have to store the passphrase
     in a configuration file on the same system as the encrypted documents
     for paperless to work. Furthermore, the entire text content of the

docs/advanced_usage.md

@@ -50,7 +50,7 @@ and run another document through the consumer. Once complete, you should
 see the newly-created document, automatically tagged with the
 appropriate data.
 
-### Automatic matching {#automatic_matching}
+### Automatic matching {#automatic-matching}
 
 Paperless-ngx comes with a new matching algorithm called _Auto_. This
 matching algorithm tries to assign tags, correspondents, document types,
@@ -59,8 +59,8 @@ assigned these on existing documents. It uses a neural network under the
 hood.
 
 If, for example, all your bank statements of your account 123 at the
-Bank of America are tagged with the tag "bofa*123" and the matching
-algorithm of this tag is set to \_Auto*, this neural network will examine
+Bank of America are tagged with the tag "bofa123" and the matching
+algorithm of this tag is set to _Auto_, this neural network will examine
 your documents and automatically learn when to assign this tag.
 
 Paperless tries to hide much of the involved complexity with this
@@ -95,7 +95,7 @@ when using this feature:
   of these correspondents to ANY new document, if both are set to
   automatic matching.
 
-## Hooking into the consumption process
+## Hooking into the consumption process {#consume-hooks}
 
 Sometimes you may want to do something arbitrary whenever a document is
 consumed. Rather than try to predict what you may want to do, Paperless
@@ -115,13 +115,23 @@ and then put the path to that script in `paperless.conf` or
     asynchronously, you'll have to fork the process in your script and
     exit.
 
-### Pre-consumption script
+### Pre-consumption script {#pre-consume-script}
 
 Executed after the consumer sees a new document in the consumption
 folder, but before any processing of the document is performed. This
 script can access the following relevant environment variables set:
 
-- `DOCUMENT_SOURCE_PATH`
+| Environment Variable    | Description                                                  |
+| ----------------------- | ------------------------------------------------------------ |
+| `DOCUMENT_SOURCE_PATH`  | Original path of the consumed document                       |
+| `DOCUMENT_WORKING_PATH` | Path to a copy of the original that consumption will work on |
+
+!!! note
+
+    Pre-consume scripts which modify the document should only change
+    the `DOCUMENT_WORKING_PATH` file or a second consume task may
+    be triggered, leading to failures as two tasks work on the
+    same document path
 
 A simple but common example for this would be creating a simple script
 like this:
@@ -130,7 +140,7 @@ like this:
 
 ```bash
 #!/usr/bin/env bash
-pdf2pdfocr.py -i ${DOCUMENT_SOURCE_PATH}
+pdf2pdfocr.py -i ${DOCUMENT_WORKING_PATH}
 ```
 
 `/etc/paperless.conf`
@@ -151,63 +161,74 @@ with the newly modified file.
 The script's stdout and stderr will be logged line by line to the
 webserver log, along with the exit code of the script.
 
-### Post-consumption script {#post_consume_script}
+### Post-consumption script {#post-consume-script}
 
 Executed after the consumer has successfully processed a document and
 has moved it into paperless. It receives the following environment
 variables:
 
-- `DOCUMENT_ID`
-- `DOCUMENT_FILE_NAME`
-- `DOCUMENT_CREATED`
-- `DOCUMENT_MODIFIED`
-- `DOCUMENT_ADDED`
-- `DOCUMENT_SOURCE_PATH`
-- `DOCUMENT_ARCHIVE_PATH`
-- `DOCUMENT_THUMBNAIL_PATH`
-- `DOCUMENT_DOWNLOAD_URL`
-- `DOCUMENT_THUMBNAIL_URL`
-- `DOCUMENT_CORRESPONDENT`
-- `DOCUMENT_TAGS`
-- `DOCUMENT_ORIGINAL_FILENAME`
+| Environment Variable         | Description                                   |
+| ---------------------------- | --------------------------------------------- |
+| `DOCUMENT_ID`                | Database primary key of the document          |
+| `DOCUMENT_FILE_NAME`         | Formatted filename, not including paths       |
+| `DOCUMENT_CREATED`           | Date & time when document created             |
+| `DOCUMENT_MODIFIED`          | Date & time when document was last modified   |
+| `DOCUMENT_ADDED`             | Date & time when document was added           |
+| `DOCUMENT_SOURCE_PATH`       | Path to the original document file            |
+| `DOCUMENT_ARCHIVE_PATH`      | Path to the generate archive file (if any)    |
+| `DOCUMENT_THUMBNAIL_PATH`    | Path to the generated thumbnail               |
+| `DOCUMENT_DOWNLOAD_URL`      | URL for document download                     |
+| `DOCUMENT_THUMBNAIL_URL`     | URL for the document thumbnail                |
+| `DOCUMENT_CORRESPONDENT`     | Assigned correspondent (if any)               |
+| `DOCUMENT_TAGS`              | Comma separated list of tags applied (if any) |
+| `DOCUMENT_ORIGINAL_FILENAME` | Filename of original document                 |
 
-The script can be in any language, but for a simple shell script
-example, you can take a look at
-[post-consumption-example.sh](https://github.com/paperless-ngx/paperless-ngx/blob/main/scripts/post-consumption-example.sh)
-in this project.
+The script can be in any language, A simple shell script example:
 
-The post consumption script cannot cancel the consumption process.
+```bash title="post-consumption-example"
+--8<-- "./scripts/post-consumption-example.sh"
+```
+
+!!! note
+
+    The post consumption script cannot cancel the consumption process.
+
+!!! warning
+
+    The post consumption script should not modify the document files
+    directly
 
 The script's stdout and stderr will be logged line by line to the
 webserver log, along with the exit code of the script.
 
-#### Docker
+### Docker {#docker-consume-hooks}
 
-Assumed you have
-`/home/foo/paperless-ngx/scripts/post-consumption-example.sh`.
+To hook into the consumption process when using Docker, you
+will need to pass the scripts into the container via a host mount
+in your `docker-compose.yml`.
 
-You can pass that script into the consumer container via a host mount in
-your `docker-compose.yml`.
+Assuming you have
+`/home/paperless-ngx/scripts/post-consumption-example.sh` as a
+script which you'd like to run.
 
-```bash
+You can pass that script into the consumer container via a host mount:
+
+```yaml
 ...
-consumer:
+webserver:
   ...
   volumes:
     ...
-    - /home/paperless-ngx/scripts:/path/in/container/scripts/
+    - /home/paperless-ngx/scripts:/path/in/container/scripts/ # (1)!
+  environment: # (3)!
+    ...
+    PAPERLESS_POST_CONSUME_SCRIPT: /path/in/container/scripts/post-consumption-example.sh # (2)!
 ...
 ```
 
-Example (docker-compose.yml):
-`- /home/foo/paperless-ngx/scripts:/usr/src/paperless/scripts`
-
-which in turn requires the variable `PAPERLESS_POST_CONSUME_SCRIPT` in
-`docker-compose.env` to point to
-`/path/in/container/scripts/post-consumption-example.sh`.
-
-Example (docker-compose.env):
-`PAPERLESS_POST_CONSUME_SCRIPT=/usr/src/paperless/scripts/post-consumption-example.sh`
+1. The external scripts directory is mounted to a location inside the container.
+2. The internal location of the script is used to set the script to run
+3. This can also be set in `docker-compose.env`
 
 Troubleshooting:
 
@@ -218,7 +239,7 @@ Troubleshooting:
 - Pipe your scripts's output to a log file e.g.
   `echo "${DOCUMENT_ID}" | tee --append /usr/src/paperless/scripts/post-consumption-example.log`
 
-## File name handling {#file_name_handling}
+## File name handling {#file-name-handling}
 
 By default, paperless stores your documents in the media directory and
 renames them using the identifier which it has assigned to each
@@ -301,7 +322,7 @@ value.
 !!! tip
 
     You can affect how empty placeholders are treated by changing the
-    following setting to [true]{.title-ref}.
+    following setting to `true`.
 
     ```
     PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=True
@@ -316,7 +337,7 @@ value.
     Paperless checks the filename of a document whenever it is saved.
     Therefore, you need to update the filenames of your documents and move
     them after altering this setting by invoking the
-    [`document renamer <utilities-renamer>`]().
+    [`document renamer`](/administration#renamer).
 
 !!! warning
 
@@ -335,6 +356,13 @@ value.
     However, keep in mind that inside docker, if files get stored outside of
     the predefined volumes, they will be lost after a restart of paperless.
 
+!!! warning
+
+    When file naming handling, in particular when using `{tag_list}`,
+    you may run into the limits of your operating system's maximum
+    path lengths.  Files will retain the previous path instead and
+    the issue logged.
+
 ## Storage paths
 
 One of the best things in Paperless is that you can not only access the
@@ -344,7 +372,7 @@ When as single storage layout is not sufficient for your use case,
 storage paths come to the rescue. Storage paths allow you to configure
 more precisely where each document is stored in the file system.
 
-- Each storage path is a [PAPERLESS_FILENAME_FORMAT]{.title-ref} and
+- Each storage path is a `PAPERLESS_FILENAME_FORMAT` and
   follows the rules described above
 - Each document is assigned a storage path using the matching
   algorithms described above, but can be overwritten at any time
@@ -352,7 +380,7 @@ more precisely where each document is stored in the file system.
 For example, you could define the following two storage paths:
 
 1.  Normal communications are put into a folder structure sorted by
-    [year/correspondent]{.title-ref}
+    `year/correspondent`
 2.  Communications with insurance companies are stored in a flat
     structure with longer file names, but containing the full date of
     the correspondence.
@@ -363,7 +391,7 @@ Insurances = Insurances/{correspondent}/{created_year}-{created_month}-{created_
 ```
 
 If you then map these storage paths to the documents, you might get the
-following result. For simplicity, [By Year]{.title-ref} defines the same
+following result. For simplicity, `By Year` defines the same
 structure as in the previous example above.
 
 ```text
@@ -384,14 +412,14 @@ structure as in the previous example above.
 !!! tip
 
     Defining a storage path is optional. If no storage path is defined for a
-    document, the global [PAPERLESS_FILENAME_FORMAT]{.title-ref} is applied.
+    document, the global `PAPERLESS_FILENAME_FORMAT` is applied.
 
 !!! warning
 
     If you adjust the format of an existing storage path, old documents
     don't get relocated automatically. You need to run the
     [document renamer](/administration#renamer) to
-    adjust their pathes.
+    adjust their paths.
 
 ## Celery Monitoring {#celery-monitoring}
 
@@ -403,33 +431,38 @@ queued and completed tasks, timing and more. Flower can also be used
 with Prometheus, as it exports metrics. For details on its capabilities,
 refer to the Flower documentation.
 
-To configure Flower further, create a [flowerconfig.py]{.title-ref} and
-place it into the [src/paperless]{.title-ref} directory. For a Docker
+To configure Flower further, create a `flowerconfig.py` and
+place it into the `src/paperless` directory. For a Docker
 installation, you can use volumes to accomplish this:
 
 ```yaml
 services:
   # ...
   webserver:
+    ports:
+      - 5555:5555 # (2)!
     # ...
     volumes:
-      - /path/to/my/flowerconfig.py:/usr/src/paperless/src/paperless/flowerconfig.py:ro
+      - /path/to/my/flowerconfig.py:/usr/src/paperless/src/paperless/flowerconfig.py:ro # (1)!
 ```
 
+1. Note the `:ro` tag means the file will be mounted as read only.
+2. `flower` runs by default on port 5555, but this can be configured
+
 ## Custom Container Initialization
 
 The Docker image includes the ability to run custom user scripts during
 startup. This could be utilized for installing additional tools or
-Python packages, for example.
+Python packages, for example. Scripts are expected to be shell scripts.
 
 To utilize this, mount a folder containing your scripts to the custom
-initialization directory, [/custom-cont-init.d]{.title-ref} and place
+initialization directory, `/custom-cont-init.d` and place
 scripts you wish to run inside. For security, the folder must be owned
 by `root` and should have permissions of `a=rx`. Additionally, scripts
 must only be writable by `root`.
 
 Your scripts will be run directly before the webserver completes
-startup. Scripts will be run by the [root]{.title-ref} user.
+startup. Scripts will be run by the `root` user.
 If you would like to switch users, the utility `gosu` is available and
 preferred over `sudo`.
 
@@ -445,9 +478,11 @@ services:
   webserver:
     # ...
     volumes:
-      - /path/to/my/scripts:/custom-cont-init.d:ro
+      - /path/to/my/scripts:/custom-cont-init.d:ro # (1)!
 ```
 
+1. Note the `:ro` tag means the folder will be mounted as read only. This is for extra security against changes
+
 ## MySQL Caveats {#mysql-caveats}
 
 ### Case Sensitivity
@@ -466,3 +501,9 @@ You can also set the default for new tables (this does NOT affect
 existing tables) with:
 
 `ALTER DATABASE <db_name> CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`
+
+!!! warning
+
+    Using mariadb version 10.4+ is recommended. Using the `utf8mb3` character set on
+    an older system may fix issues that can arise while setting up Paperless-ngx but
+    `utf8mb3` can cause issues with consumption (where `utf8mb4` does not).

docs/api.md

@@ -1,12 +1,12 @@
 # The REST API
 
 Paperless makes use of the [Django REST
-Framework](http://django-rest-framework.org/) standard API interface. It
+Framework](https://django-rest-framework.org/) standard API interface. It
 provides a browsable API for most of its endpoints, which you can
 inspect at `http://<paperless-host>:<port>/api/`. This also documents
 most of the available filters and ordering fields.
 
-The API provides 5 main endpoints:
+The API provides 7 main endpoints:
 
 - `/api/documents/`: Full CRUD support, except POSTing new documents.
   See below.
@@ -162,7 +162,7 @@ specific query parameters cause the API to return full text search
 results:
 
 - `/api/documents/?query=your%20search%20query`: Search for a document
-  using a full text query. For details on the syntax, see [Basic Usage - Searching](usage#basic-usage_searching).
+  using a full text query. For details on the syntax, see [Basic Usage - Searching](/usage#basic-usage_searching).
 - `/api/documents/?more_like=1234`: Search for documents similar to
   the document with id 1234.
 
@@ -225,7 +225,7 @@ Query parameters:
 
 Results returned by the endpoint are ordered by importance of the term
 in the document index. The first result is the term that has the highest
-Tf/Idf score in the index.
+[Tf/Idf](https://en.wikipedia.org/wiki/Tf%E2%80%93idf) score in the index.
 
 ```json
 ["term1", "term3", "term6", "term4"]

docs/changelog.md

@@ -1,5 +1,285 @@
 # Changelog
 
+## paperless-ngx 1.12.2
+
+_Note: Version 1.12.x introduced searching of comments which will work for comments added after the upgrade but a reindex of the search index is required in order to be able to search
+older comments. The Docker image will automatically perform this reindex, bare metal installations will have to perform this manually, see [the docs](https://docs.paperless-ngx.com/administration/#index)._
+
+### Bug Fixes
+
+- Bugfix: Allow pre-consume scripts to modify incoming file [@stumpylog](https://github.com/stumpylog) ([#2547](https://github.com/paperless-ngx/paperless-ngx/pull/2547))
+- Bugfix: Return to page based barcode scanning [@stumpylog](https://github.com/stumpylog) ([#2544](https://github.com/paperless-ngx/paperless-ngx/pull/2544))
+- Fix: Try to prevent title debounce overwriting [@shamoon](https://github.com/shamoon) ([#2543](https://github.com/paperless-ngx/paperless-ngx/pull/2543))
+- Fix comment search highlight + multi-word search [@shamoon](https://github.com/shamoon) ([#2542](https://github.com/paperless-ngx/paperless-ngx/pull/2542))
+- Bugfix: Request PDF/A format from Gotenberg [@stumpylog](https://github.com/stumpylog) ([#2530](https://github.com/paperless-ngx/paperless-ngx/pull/2530))
+- Fix: Trigger reindex for pre-existing comments [@shamoon](https://github.com/shamoon) ([#2519](https://github.com/paperless-ngx/paperless-ngx/pull/2519))
+
+### Documentation
+
+- Bugfix: Allow pre-consume scripts to modify incoming file [@stumpylog](https://github.com/stumpylog) ([#2547](https://github.com/paperless-ngx/paperless-ngx/pull/2547))
+- Fix: Trigger reindex for pre-existing comments [@shamoon](https://github.com/shamoon) ([#2519](https://github.com/paperless-ngx/paperless-ngx/pull/2519))
+- Minor updates to development documentation [@clemensrieder](https://github.com/clemensrieder) ([#2474](https://github.com/paperless-ngx/paperless-ngx/pull/2474))
+- [Documentation] Add v1.12.1 changelog [@github-actions](https://github.com/github-actions) ([#2515](https://github.com/paperless-ngx/paperless-ngx/pull/2515))
+
+### Maintenance
+
+- Chore: Fix tag cleaner to work with attestations [@stumpylog](https://github.com/stumpylog) ([#2532](https://github.com/paperless-ngx/paperless-ngx/pull/2532))
+- Chore: Make installers statically versioned [@stumpylog](https://github.com/stumpylog) ([#2517](https://github.com/paperless-ngx/paperless-ngx/pull/2517))
+
+### All App Changes
+
+- Bugfix: Allow pre-consume scripts to modify incoming file [@stumpylog](https://github.com/stumpylog) ([#2547](https://github.com/paperless-ngx/paperless-ngx/pull/2547))
+- Bugfix: Return to page based barcode scanning [@stumpylog](https://github.com/stumpylog) ([#2544](https://github.com/paperless-ngx/paperless-ngx/pull/2544))
+- Fix: Try to prevent title debounce overwriting [@shamoon](https://github.com/shamoon) ([#2543](https://github.com/paperless-ngx/paperless-ngx/pull/2543))
+- Fix comment search highlight + multi-word search [@shamoon](https://github.com/shamoon) ([#2542](https://github.com/paperless-ngx/paperless-ngx/pull/2542))
+- Bugfix: Request PDF/A format from Gotenberg [@stumpylog](https://github.com/stumpylog) ([#2530](https://github.com/paperless-ngx/paperless-ngx/pull/2530))
+
+## paperless-ngx 1.12.1
+
+### Bug Fixes
+
+- Fix: comments not showing in search until after manual reindex in v1.12 [@shamoon](https://github.com/shamoon) ([#2513](https://github.com/paperless-ngx/paperless-ngx/pull/2513))
+- Fix: date range search broken in 1.12 [@shamoon](https://github.com/shamoon) ([#2509](https://github.com/paperless-ngx/paperless-ngx/pull/2509))
+
+### Documentation
+
+- [Documentation] Add v1.12.0 changelog [@github-actions](https://github.com/github-actions) ([#2507](https://github.com/paperless-ngx/paperless-ngx/pull/2507))
+
+### Maintenance
+
+- Moves back to the main release-drafter now that it does what we wanted [@stumpylog](https://github.com/stumpylog) ([#2503](https://github.com/paperless-ngx/paperless-ngx/pull/2503))
+
+### All App Changes
+
+- Fix: comments not showing in search until after manual reindex in v1.12 [@shamoon](https://github.com/shamoon) ([#2513](https://github.com/paperless-ngx/paperless-ngx/pull/2513))
+- Fix: date range search broken in 1.12 [@shamoon](https://github.com/shamoon) ([#2509](https://github.com/paperless-ngx/paperless-ngx/pull/2509))
+
+## paperless-ngx 1.12.0
+
+### Features
+
+- New document_exporter options [@mhelleboid](https://github.com/mhelleboid) ([#2448](https://github.com/paperless-ngx/paperless-ngx/pull/2448))
+- Read ASN from barcode on page [@peterkappelt](https://github.com/peterkappelt) ([#2437](https://github.com/paperless-ngx/paperless-ngx/pull/2437))
+- Add AppleMail color tag support [@clemensrieder](https://github.com/clemensrieder) ([#2407](https://github.com/paperless-ngx/paperless-ngx/pull/2407))
+- Feature: Retain original filename on upload [@stumpylog](https://github.com/stumpylog) ([#2404](https://github.com/paperless-ngx/paperless-ngx/pull/2404))
+- Feature: Control scheduled tasks via cron expressions [@stumpylog](https://github.com/stumpylog) ([#2403](https://github.com/paperless-ngx/paperless-ngx/pull/2403))
+- Simplify json parsing in build scripts [@tribut](https://github.com/tribut) ([#2370](https://github.com/paperless-ngx/paperless-ngx/pull/2370))
+- Feature: include comments in advanced search [@shamoon](https://github.com/shamoon) ([#2351](https://github.com/paperless-ngx/paperless-ngx/pull/2351))
+
+### Bug Fixes
+
+- Fix: limit asn integer size [@shamoon](https://github.com/shamoon) ([#2498](https://github.com/paperless-ngx/paperless-ngx/pull/2498))
+- Bugfix: Rescales images for better barcode locating [@stumpylog](https://github.com/stumpylog) ([#2468](https://github.com/paperless-ngx/paperless-ngx/pull/2468))
+- Fix: fix downgrade migration [@shamoon](https://github.com/shamoon) ([#2494](https://github.com/paperless-ngx/paperless-ngx/pull/2494))
+- Fix: Allow setting mailrule order from frontend [@shamoon](https://github.com/shamoon) ([#2459](https://github.com/paperless-ngx/paperless-ngx/pull/2459))
+- Fix: tag color ordering [@shamoon](https://github.com/shamoon) ([#2456](https://github.com/paperless-ngx/paperless-ngx/pull/2456))
+- Fix: Better Handle arbitrary ISO 8601 strings after celery serializing [@shamoon](https://github.com/shamoon) ([#2441](https://github.com/paperless-ngx/paperless-ngx/pull/2441))
+- Use correct canonical path for nltk_data [@amo13](https://github.com/amo13) ([#2429](https://github.com/paperless-ngx/paperless-ngx/pull/2429))
+- Fix: Include optional socket file in release [@stumpylog](https://github.com/stumpylog) ([#2409](https://github.com/paperless-ngx/paperless-ngx/pull/2409))
+- Fix: display rtl content in correct direction [@shamoon](https://github.com/shamoon) ([#2302](https://github.com/paperless-ngx/paperless-ngx/pull/2302))
+- Fixed endpoint count in Docs The REST API [@PascalSenn](https://github.com/PascalSenn) ([#2386](https://github.com/paperless-ngx/paperless-ngx/pull/2386))
+- Fix subpath for websockets [@tribut](https://github.com/tribut) ([#2371](https://github.com/paperless-ngx/paperless-ngx/pull/2371))
+- Fix: Make missing environment from file files informational only [@stumpylog](https://github.com/stumpylog) ([#2368](https://github.com/paperless-ngx/paperless-ngx/pull/2368))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Fix: preview content remains hidden on mobile [@shamoon](https://github.com/shamoon) ([#2346](https://github.com/paperless-ngx/paperless-ngx/pull/2346))
+- Bugfix: Removal of alpha channel truncates multipage TIFFs [@stumpylog](https://github.com/stumpylog) ([#2335](https://github.com/paperless-ngx/paperless-ngx/pull/2335))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+
+### Documentation
+
+- Docs: Fix typo - docker-compose.yml file name in setup doc [@muli](https://github.com/muli) ([#2477](https://github.com/paperless-ngx/paperless-ngx/pull/2477))
+- document existence of document_thumbnails [@frrad](https://github.com/frrad) ([#2470](https://github.com/paperless-ngx/paperless-ngx/pull/2470))
+- Add optional sudo command to bare metal docs [@shamoon](https://github.com/shamoon) ([#2464](https://github.com/paperless-ngx/paperless-ngx/pull/2464))
+- Fix link [@edenhaus](https://github.com/edenhaus) ([#2458](https://github.com/paperless-ngx/paperless-ngx/pull/2458))
+- Documentation: Fix comment re bare metal runserver command [@shamoon](https://github.com/shamoon) ([#2420](https://github.com/paperless-ngx/paperless-ngx/pull/2420))
+- Fix formatting of config variable in docs [@peterkappelt](https://github.com/peterkappelt) ([#2445](https://github.com/paperless-ngx/paperless-ngx/pull/2445))
+- Update docs nginx reverse proxy example [@Sprinterfreak](https://github.com/Sprinterfreak) ([#2443](https://github.com/paperless-ngx/paperless-ngx/pull/2443))
+- [Documentation] Add note re for dev server [@shamoon](https://github.com/shamoon) ([#2387](https://github.com/paperless-ngx/paperless-ngx/pull/2387))
+- Fixed endpoint count in Docs The REST API [@PascalSenn](https://github.com/PascalSenn) ([#2386](https://github.com/paperless-ngx/paperless-ngx/pull/2386))
+- [ Docs] Update bare metal setup instructions [@natrius](https://github.com/natrius) ([#2281](https://github.com/paperless-ngx/paperless-ngx/pull/2281))
+- [Docs] Add Paperless Mobile app to docs [@astubenbord](https://github.com/astubenbord) ([#2378](https://github.com/paperless-ngx/paperless-ngx/pull/2378))
+- Tiny spelling change [@veverkap](https://github.com/veverkap) ([#2369](https://github.com/paperless-ngx/paperless-ngx/pull/2369))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+- [Documentation] Add note that PAPERLESS_URL cant contain a path [@shamoon](https://github.com/shamoon) ([#2319](https://github.com/paperless-ngx/paperless-ngx/pull/2319))
+- [Documentation] Add v1.11.3 changelog [@github-actions](https://github.com/github-actions) ([#2311](https://github.com/paperless-ngx/paperless-ngx/pull/2311))
+
+### Maintenance
+
+- Fix: Include optional socket file in release [@stumpylog](https://github.com/stumpylog) ([#2409](https://github.com/paperless-ngx/paperless-ngx/pull/2409))
+- Chore: remove helm chart code [@shamoon](https://github.com/shamoon) ([#2388](https://github.com/paperless-ngx/paperless-ngx/pull/2388))
+- Simplify json parsing in build scripts [@tribut](https://github.com/tribut) ([#2370](https://github.com/paperless-ngx/paperless-ngx/pull/2370))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Bump tj-actions/changed-files from 34 to 35 [@dependabot](https://github.com/dependabot) ([#2303](https://github.com/paperless-ngx/paperless-ngx/pull/2303))
+
+### Dependencies
+
+<details>
+<summary>4 changes</summary>
+
+- Chore: Backend library updates [@stumpylog](https://github.com/stumpylog) ([#2401](https://github.com/paperless-ngx/paperless-ngx/pull/2401))
+- Bump tj-actions/changed-files from 34 to 35 [@dependabot](https://github.com/dependabot) ([#2303](https://github.com/paperless-ngx/paperless-ngx/pull/2303))
+- Bump [@<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2306](https://github.com/paperless-ngx/paperless-ngx/pull/2306))
+- Bump [@<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2308](https://github.com/paperless-ngx/paperless-ngx/pull/2308))
+</details>
+
+### All App Changes
+
+- New document_exporter options [@mhelleboid](https://github.com/mhelleboid) ([#2448](https://github.com/paperless-ngx/paperless-ngx/pull/2448))
+- Fix: limit asn integer size [@shamoon](https://github.com/shamoon) ([#2498](https://github.com/paperless-ngx/paperless-ngx/pull/2498))
+- Fix: fix downgrade migration [@shamoon](https://github.com/shamoon) ([#2494](https://github.com/paperless-ngx/paperless-ngx/pull/2494))
+- Read ASN from barcode on page [@peterkappelt](https://github.com/peterkappelt) ([#2437](https://github.com/paperless-ngx/paperless-ngx/pull/2437))
+- Fix: Allow setting mailrule order from frontend [@shamoon](https://github.com/shamoon) ([#2459](https://github.com/paperless-ngx/paperless-ngx/pull/2459))
+- Chore: Update to Angular 15 \& associated frontend deps [@shamoon](https://github.com/shamoon) ([#2411](https://github.com/paperless-ngx/paperless-ngx/pull/2411))
+- Fix: tag color ordering [@shamoon](https://github.com/shamoon) ([#2456](https://github.com/paperless-ngx/paperless-ngx/pull/2456))
+- Fix: Better Handle arbitrary ISO 8601 strings after celery serializing [@shamoon](https://github.com/shamoon) ([#2441](https://github.com/paperless-ngx/paperless-ngx/pull/2441))
+- Use correct canonical path for nltk_data [@amo13](https://github.com/amo13) ([#2429](https://github.com/paperless-ngx/paperless-ngx/pull/2429))
+- Add AppleMail color tag support [@clemensrieder](https://github.com/clemensrieder) ([#2407](https://github.com/paperless-ngx/paperless-ngx/pull/2407))
+- Chore: Convert document exporter to use pathlib [@stumpylog](https://github.com/stumpylog) ([#2416](https://github.com/paperless-ngx/paperless-ngx/pull/2416))
+- Feature: Retain original filename on upload [@stumpylog](https://github.com/stumpylog) ([#2404](https://github.com/paperless-ngx/paperless-ngx/pull/2404))
+- Feature: Control scheduled tasks via cron expressions [@stumpylog](https://github.com/stumpylog) ([#2403](https://github.com/paperless-ngx/paperless-ngx/pull/2403))
+- Fix: display rtl content in correct direction [@shamoon](https://github.com/shamoon) ([#2302](https://github.com/paperless-ngx/paperless-ngx/pull/2302))
+- Fix subpath for websockets [@tribut](https://github.com/tribut) ([#2371](https://github.com/paperless-ngx/paperless-ngx/pull/2371))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Feature: include comments in advanced search [@shamoon](https://github.com/shamoon) ([#2351](https://github.com/paperless-ngx/paperless-ngx/pull/2351))
+- Chore: More frontend tests [@shamoon](https://github.com/shamoon) ([#2352](https://github.com/paperless-ngx/paperless-ngx/pull/2352))
+- Chore: Fixing up some minor annoyances [@stumpylog](https://github.com/stumpylog) ([#2348](https://github.com/paperless-ngx/paperless-ngx/pull/2348))
+- Bugfix: Removal of alpha channel truncates multipage TIFFs [@stumpylog](https://github.com/stumpylog) ([#2335](https://github.com/paperless-ngx/paperless-ngx/pull/2335))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+- Add Arabic language to frontend [@KhaledEmad7](https://github.com/KhaledEmad7) ([#2313](https://github.com/paperless-ngx/paperless-ngx/pull/2313))
+- Bump [@<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2306](https://github.com/paperless-ngx/paperless-ngx/pull/2306))
+- Bump [@<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2308](https://github.com/paperless-ngx/paperless-ngx/pull/2308))
+
+## paperless-ngx 1.11.3
+
+### Breaking Changes
+
+_Note: PR #2279 could represent a breaking change to the API which may affect third party applications that were only checking the `post_document` endpoint for e.g. result = 'OK' as opposed to e.g. HTTP status = 200_
+
+- Bugfix: Return created task ID when posting document to API [@stumpylog](https://github.com/stumpylog) ([#2279](https://github.com/paperless-ngx/paperless-ngx/pull/2279))
+
+### Bug Fixes
+
+- Bugfix: Fix no content when processing some RTL files [@stumpylog](https://github.com/stumpylog) ([#2295](https://github.com/paperless-ngx/paperless-ngx/pull/2295))
+- Bugfix: Handle email dates maybe being naive [@stumpylog](https://github.com/stumpylog) ([#2293](https://github.com/paperless-ngx/paperless-ngx/pull/2293))
+- Fix: live filterable dropdowns broken in 1.11.x [@shamoon](https://github.com/shamoon) ([#2292](https://github.com/paperless-ngx/paperless-ngx/pull/2292))
+- Bugfix: Reading environment from files didn't work for management commands [@stumpylog](https://github.com/stumpylog) ([#2261](https://github.com/paperless-ngx/paperless-ngx/pull/2261))
+- Bugfix: Return created task ID when posting document to API [@stumpylog](https://github.com/stumpylog) ([#2279](https://github.com/paperless-ngx/paperless-ngx/pull/2279))
+
+### All App Changes
+
+- Bugfix: Fix no content when processing some RTL files [@stumpylog](https://github.com/stumpylog) ([#2295](https://github.com/paperless-ngx/paperless-ngx/pull/2295))
+- Bugfix: Handle email dates maybe being naive [@stumpylog](https://github.com/stumpylog) ([#2293](https://github.com/paperless-ngx/paperless-ngx/pull/2293))
+- Fix: live filterable dropdowns broken in 1.11.x [@shamoon](https://github.com/shamoon) ([#2292](https://github.com/paperless-ngx/paperless-ngx/pull/2292))
+- Bugfix: Return created task ID when posting document to API [@stumpylog](https://github.com/stumpylog) ([#2279](https://github.com/paperless-ngx/paperless-ngx/pull/2279))
+
+## paperless-ngx 1.11.2
+
+Versions 1.11.1 and 1.11.2 contain bug fixes from v1.11.0 that prevented use of the new email consumption feature
+
+### Bug Fixes
+
+- Fix frontend mailrule missing consumption scope parameter [@shamoon](https://github.com/shamoon) ([#2280](https://github.com/paperless-ngx/paperless-ngx/pull/2280))
+- Fix: missing frontend email attachment options [@shamoon](https://github.com/shamoon) ([#2272](https://github.com/paperless-ngx/paperless-ngx/pull/2272))
+- Fix: edit dialog creation in v1.11.0 [@shamoon](https://github.com/shamoon) ([#2273](https://github.com/paperless-ngx/paperless-ngx/pull/2273))
+
+### All App Changes
+
+- Fix frontend mailrule missing consumption scope parameter [@shamoon](https://github.com/shamoon) ([#2280](https://github.com/paperless-ngx/paperless-ngx/pull/2280))
+- Fix: missing frontend email attachment options [@shamoon](https://github.com/shamoon) ([#2272](https://github.com/paperless-ngx/paperless-ngx/pull/2272))
+- Fix: edit dialog creation in v1.11.0 [@shamoon](https://github.com/shamoon) ([#2273](https://github.com/paperless-ngx/paperless-ngx/pull/2273))
+
+## paperless-ngx 1.11.0
+
+### Notable Changes
+
+- Feature: frontend paperless mail [@shamoon](https://github.com/shamoon) ([#2000](https://github.com/paperless-ngx/paperless-ngx/pull/2000))
+- Feature: Ability to consume mails and eml files [@p-h-a-i-l](https://github.com/p-h-a-i-l) ([#848](https://github.com/paperless-ngx/paperless-ngx/pull/848))
+
+### Features
+
+- Chore: Downgrade hiredis to 2.0.0 [@stumpylog](https://github.com/stumpylog) ([#2262](https://github.com/paperless-ngx/paperless-ngx/pull/2262))
+- Add ability to provide the configuration file path using an env variable [@hashworks](https://github.com/hashworks) ([#2241](https://github.com/paperless-ngx/paperless-ngx/pull/2241))
+- Feature: Adds option to allow a user to export directly to a zipfile [@stumpylog](https://github.com/stumpylog) ([#2004](https://github.com/paperless-ngx/paperless-ngx/pull/2004))
+- Feature: Adds PaperlessTask admin page interface [@stumpylog](https://github.com/stumpylog) ([#2184](https://github.com/paperless-ngx/paperless-ngx/pull/2184))
+- Feature: speed up frontend by truncating content [@shamoon](https://github.com/shamoon) ([#2028](https://github.com/paperless-ngx/paperless-ngx/pull/2028))
+- Feature: Allow bulk download API to follow file name formatting [@stumpylog](https://github.com/stumpylog) ([#2003](https://github.com/paperless-ngx/paperless-ngx/pull/2003))
+- Feature: Bake NLTK into Docker image [@stumpylog](https://github.com/stumpylog) ([#2129](https://github.com/paperless-ngx/paperless-ngx/pull/2129))
+- Feature: frontend paperless mail [@shamoon](https://github.com/shamoon) ([#2000](https://github.com/paperless-ngx/paperless-ngx/pull/2000))
+- Feature: Ability to consume mails and eml files [@p-h-a-i-l](https://github.com/p-h-a-i-l) ([#848](https://github.com/paperless-ngx/paperless-ngx/pull/848))
+
+### Bug Fixes
+
+- Bugfix: Handle RTL languages better [@stumpylog](https://github.com/stumpylog) ([#1665](https://github.com/paperless-ngx/paperless-ngx/pull/1665))
+- Fixed typo in docs [@mendelk](https://github.com/mendelk) ([#2256](https://github.com/paperless-ngx/paperless-ngx/pull/2256))
+- Fix: support in advanced search, fix tags filter badge count for excluded [@shamoon](https://github.com/shamoon) ([#2205](https://github.com/paperless-ngx/paperless-ngx/pull/2205))
+- Bugfix: Don't run system checks on migrate [@stumpylog](https://github.com/stumpylog) ([#2183](https://github.com/paperless-ngx/paperless-ngx/pull/2183))
+- Bugfix: Decoding task signals could fail on datetime type [@stumpylog](https://github.com/stumpylog) ([#2058](https://github.com/paperless-ngx/paperless-ngx/pull/2058))
+
+### Documentation
+
+- Fixed typo in docs [@mendelk](https://github.com/mendelk) ([#2256](https://github.com/paperless-ngx/paperless-ngx/pull/2256))
+- Docs: More fixes and improvements [@tooomm](https://github.com/tooomm) ([#2203](https://github.com/paperless-ngx/paperless-ngx/pull/2203))
+- Docs: Fix leftover issues from conversion [@tooomm](https://github.com/tooomm) ([#2172](https://github.com/paperless-ngx/paperless-ngx/pull/2172))
+- Docs: Fix broken internal links [@tooomm](https://github.com/tooomm) ([#2165](https://github.com/paperless-ngx/paperless-ngx/pull/2165))
+- Update setup.md [@Weltraumschaf](https://github.com/Weltraumschaf) ([#2157](https://github.com/paperless-ngx/paperless-ngx/pull/2157))
+- Chore: Cleanup of new documentation [@stumpylog](https://github.com/stumpylog) ([#2137](https://github.com/paperless-ngx/paperless-ngx/pull/2137))
+- [Documentation] Add v1.10.2 changelog [@github-actions](https://github.com/github-actions) ([#2114](https://github.com/paperless-ngx/paperless-ngx/pull/2114))
+
+### Maintenance
+
+- Chore: Adds notable label for release drafter [@stumpylog](https://github.com/stumpylog) ([#2200](https://github.com/paperless-ngx/paperless-ngx/pull/2200))
+- Chore: Prevent forks from having failing CI runs by default [@tooomm](https://github.com/tooomm) ([#2166](https://github.com/paperless-ngx/paperless-ngx/pull/2166))
+- Chore: migrate to eslint [@shamoon](https://github.com/shamoon) ([#2199](https://github.com/paperless-ngx/paperless-ngx/pull/2199))
+- Feature: Adds PaperlessTask admin page interface [@stumpylog](https://github.com/stumpylog) ([#2184](https://github.com/paperless-ngx/paperless-ngx/pull/2184))
+- Chore: Changes qpdf to be cross compiled for large speed up [@stumpylog](https://github.com/stumpylog) ([#2181](https://github.com/paperless-ngx/paperless-ngx/pull/2181))
+- Chore: Decrease time to build pikepdf [@stumpylog](https://github.com/stumpylog) ([#2178](https://github.com/paperless-ngx/paperless-ngx/pull/2178))
+- Chore: Minor CI cleanups [@stumpylog](https://github.com/stumpylog) ([#2175](https://github.com/paperless-ngx/paperless-ngx/pull/2175))
+
+### All App Changes
+
+- Add ability to provide the configuration file path using an env variable [@hashworks](https://github.com/hashworks) ([#2241](https://github.com/paperless-ngx/paperless-ngx/pull/2241))
+- Fix: support in advanced search, fix tags filter badge count for excluded [@shamoon](https://github.com/shamoon) ([#2205](https://github.com/paperless-ngx/paperless-ngx/pull/2205))
+- Chore: migrate to eslint [@shamoon](https://github.com/shamoon) ([#2199](https://github.com/paperless-ngx/paperless-ngx/pull/2199))
+- Feature: Adds option to allow a user to export directly to a zipfile [@stumpylog](https://github.com/stumpylog) ([#2004](https://github.com/paperless-ngx/paperless-ngx/pull/2004))
+- Feature: Adds PaperlessTask admin page interface [@stumpylog](https://github.com/stumpylog) ([#2184](https://github.com/paperless-ngx/paperless-ngx/pull/2184))
+- Bugfix: Decoding task signals could fail on datetime type [@stumpylog](https://github.com/stumpylog) ([#2058](https://github.com/paperless-ngx/paperless-ngx/pull/2058))
+- Feature: speed up frontend by truncating content [@shamoon](https://github.com/shamoon) ([#2028](https://github.com/paperless-ngx/paperless-ngx/pull/2028))
+- Feature: Allow bulk download API to follow file name formatting [@stumpylog](https://github.com/stumpylog) ([#2003](https://github.com/paperless-ngx/paperless-ngx/pull/2003))
+- Feature: Bake NLTK into Docker image [@stumpylog](https://github.com/stumpylog) ([#2129](https://github.com/paperless-ngx/paperless-ngx/pull/2129))
+- Chore: Apply live testing backoff logic to new mail tests [@stumpylog](https://github.com/stumpylog) ([#2134](https://github.com/paperless-ngx/paperless-ngx/pull/2134))
+- Feature: frontend paperless mail [@shamoon](https://github.com/shamoon) ([#2000](https://github.com/paperless-ngx/paperless-ngx/pull/2000))
+- Feature: Ability to consume mails and eml files [@p-h-a-i-l](https://github.com/p-h-a-i-l) ([#848](https://github.com/paperless-ngx/paperless-ngx/pull/848))
+
+## paperless-ngx 1.10.2
+
+### Features
+
+- Take ownership of k8s-at-home Helm chart [@alexander-bauer](https://github.com/alexander-bauer) ([#1947](https://github.com/paperless-ngx/paperless-ngx/pull/1947))
+
+### Bug Fixes
+
+- Bugfix: Language code checks around two part languages [@stumpylog](https://github.com/stumpylog) ([#2112](https://github.com/paperless-ngx/paperless-ngx/pull/2112))
+- Bugfix: Redis socket compatibility didn't handle URLs with ports [@stumpylog](https://github.com/stumpylog) ([#2109](https://github.com/paperless-ngx/paperless-ngx/pull/2109))
+- Bugfix: Incompatible URL schemes for socket based Redis [@stumpylog](https://github.com/stumpylog) ([#2092](https://github.com/paperless-ngx/paperless-ngx/pull/2092))
+- Fix doc links in contributing [@tooomm](https://github.com/tooomm) ([#2102](https://github.com/paperless-ngx/paperless-ngx/pull/2102))
+
+### Documentation
+
+- Docs: Some more small MkDocs updates [@tooomm](https://github.com/tooomm) ([#2106](https://github.com/paperless-ngx/paperless-ngx/pull/2106))
+- Chore: Cleans up documentation links [@stumpylog](https://github.com/stumpylog) ([#2104](https://github.com/paperless-ngx/paperless-ngx/pull/2104))
+- Feature: Move docs to material-mkdocs [@shamoon](https://github.com/shamoon) ([#2067](https://github.com/paperless-ngx/paperless-ngx/pull/2067))
+- Chore: Add v1.10.1 changelong [@shamoon](https://github.com/shamoon) ([#2082](https://github.com/paperless-ngx/paperless-ngx/pull/2082))
+
+### Maintenance
+
+- Take ownership of k8s-at-home Helm chart [@alexander-bauer](https://github.com/alexander-bauer) ([#1947](https://github.com/paperless-ngx/paperless-ngx/pull/1947))
+
+### All App Changes
+
+- Bugfix: Language code checks around two part languages [@stumpylog](https://github.com/stumpylog) ([#2112](https://github.com/paperless-ngx/paperless-ngx/pull/2112))
+- Bugfix: Redis socket compatibility didn't handle URLs with ports [@stumpylog](https://github.com/stumpylog) ([#2109](https://github.com/paperless-ngx/paperless-ngx/pull/2109))
+- Bugfix: Incompatible URL schemes for socket based Redis [@stumpylog](https://github.com/stumpylog) ([#2092](https://github.com/paperless-ngx/paperless-ngx/pull/2092))
+
 ## paperless-ngx 1.10.1
 
 ### Features
@@ -961,11 +1241,10 @@ This is a maintenance release.
 
 !!! note
 
-The changed to the full text searching require you to reindex your
-documents. _The docker image does this automatically, you don't need to
-do anything._ To do this, execute the `document_index reindex`
-management command (see `administration-index`{.interpreted-text
-role="ref"}).
+    The changed to the full text searching require you to reindex your
+    documents. _The docker image does this automatically, you don't need to
+    do anything._ To do this, execute the `document_index reindex`
+    management command (see [Managing the document search index](/administration#index)).
 
 ### paperless-ng 1.3.2
 
@@ -1004,8 +1283,7 @@ This release contains new database migrations.
 - Changes
   - The REST API is versioned from this point onwards. This will
     allow me to make changes without breaking existing clients. See
-    the documentation about `api-versioning`{.interpreted-text
-    role="ref"} for details.
+    the documentation about [API versioning](/api#api-versioning) for details.
   - Added a color picker for tag colors.
   - Added the ability to use the filter for searching the document
     content as well.
@@ -1039,7 +1317,7 @@ This release contains new database migrations.
 - Changes to the OCRmyPDF integration
   - Added support for deskewing and automatic rotation of
     incorrectly rotated pages. This is enabled by default, see
-    `configuration-ocr`{.interpreted-text role="ref"}.
+    [OCR settings](/configuration#ocr).
   - Better support for encrypted files.
   - Better support for various other PDF files: Paperless will now
     attempt to force OCR with safe options when OCR fails with the
@@ -1066,7 +1344,7 @@ This release contains new database migrations.
 
 - Added a docker-specific configuration option to adjust the number of
   worker processes of the web server. See
-  `configuration-docker`{.interpreted-text role="ref"}.
+  [Docker options](/configuration#docker).
 - Some more memory usage optimizations.
 - Don't show inbox statistics if no inbox tag is defined.
 
@@ -1075,8 +1353,7 @@ This release contains new database migrations.
 - Always show top left corner of thumbnails, even for extra wide
   documents.
 - Added a management command for executing the sanity checker
-  directly. See `utilities-sanity-checker`{.interpreted-text
-  role="ref"}.
+  directly. See [management utilities](/administration#sanity-checker).
 - The weekly sanity check now reports messages in the log files.
 - Fixed an issue with the metadata tab not reporting anything in case
   of missing files.
@@ -1110,7 +1387,7 @@ This release contains new database migrations.
   management commands, since these also ensure that they're always
   executed as the paperless user and you're less likely to run into
   permission issues. See
-  `utilities-management-commands`{.interpreted-text role="ref"}.
+  [management commands](/administration#management-commands).
 
 ### paperless-ng 1.1.0
 
@@ -1135,7 +1412,7 @@ This release contains new database migrations.
   For status notifications and live updates to work, paperless now
   requires an [ASGI](https://asgi.readthedocs.io/en/latest/)-enabled
   web server. The docker images uses `gunicorn` and an ASGI-enabled
-  worker called [uvicorn](http://www.uvicorn.org/), and there is no
+  worker called [uvicorn](https://www.uvicorn.org/), and there is no
   need to configure anything.
 
   For bare metal installations, changes are required for the
@@ -1152,7 +1429,7 @@ This release contains new database migrations.
   status notifications.
 
   Apache `mod_wsgi` users, see
-  `this note <faq-mod_wsgi>`{.interpreted-text role="ref"}.
+  [this note](/faq#how-do-i-get-websocket-support-with-apache-mod_wsgi).
 
 - Paperless now offers suggestions for tags, correspondents and types
   on the document detail page.
@@ -1197,9 +1474,7 @@ bug reports coming in, I think that this is reasonably stable.
   - The document exporter locks the media directory and the database
     during execution to ensure that the resulting export is
     consistent.
-  - See the
-    `updated documentation <utilities-exporter>`{.interpreted-text
-    role="ref"} for more details.
+  - See the [updated documentation](/administration#exporter) for more details.
 - Other changes and additions
   - Added a language selector to the settings.
   - Added date format options to the settings.
@@ -1288,11 +1563,11 @@ paperless.
 - Thanks to [Jo Vandeginste](https://github.com/jovandeginste),
   Paperless has optional support for Office documents such as .docx,
   .doc, .odt and more.
-  - See the `configuration<configuration-tika>`{.interpreted-text
-    role="ref"} on how to enable this feature. This feature requires
-    two additional services (one for parsing Office documents and
-    metadata extraction and another for converting Office documents
-    to PDF), and is therefore not enabled on default installations.
+  - See the [Tika settings](/configuration#tika) on how to enable this
+    feature. This feature requires two additional services (one for
+    parsing Office documents and metadata extraction and another for
+    converting Office documents to PDF), and is therefore not enabled
+    on default installations.
   - As with all other documents, paperless converts Office documents
     to PDF and stores both the original as well as the archived PDF.
 - Dark mode
@@ -1368,15 +1643,14 @@ paperless.
 
 !!! note
 
-The bulk delete operations did not update the search index. Therefore,
-documents that you deleted remained in the index and caused the search
-to return messages about missing documents when searching. Further bulk
-operations will properly update the index.
+    The bulk delete operations did not update the search index. Therefore,
+    documents that you deleted remained in the index and caused the search
+    to return messages about missing documents when searching. Further bulk
+    operations will properly update the index.
 
-However, this change is not retroactive: If you used the delete method
-of the bulk editor, you need to reindex your search index by
-`running the management command document_index with the argument reindex <administration-index>`{.interpreted-text
-role="ref"}.
+    However, this change is not retroactive: If you used the delete method
+    of the bulk editor, you need to reindex your search index by
+    [running the management command `document_index` with the argument `reindex`](/administration#index).
 
 ### paperless-ng 0.9.9
 
@@ -1533,19 +1807,16 @@ primarily.
     edit page. If available, a dropdown menu will appear next to the
     download button.
   - Many of the configuration options regarding OCR have changed.
-    See `configuration-ocr`{.interpreted-text role="ref"} for
-    details.
+    See [OCR settings](/configuration#ocr) for details.
   - Paperless no longer guesses the language of your documents. It
     always uses the language that you specified with
     `PAPERLESS_OCR_LANGUAGE`. Be sure to set this to the language
     the majority of your documents are in. Multiple languages can be
     specified, but that requires more CPU time.
-  - The management command
-    `document_archiver <utilities-archiver>`{.interpreted-text
-    role="ref"} can be used to create archived versions for already
-    existing documents.
+  - The management command [`document_archiver`](/administration#archiver)
+    can be used to create archived versions for already existing documents.
 - Tags from consumption folder.
-  - Thanks to [jayme-github](http://github.com/jayme-github),
+  - Thanks to [jayme-github](https://github.com/jayme-github),
     paperless now consumes files from sub folders in the consumption
     folder and is able to assign tags based on the sub folders a
     document was found in. This can be configured with
@@ -1556,7 +1827,7 @@ primarily.
   - The endpoint for uploading documents now supports specifying
     custom titles, correspondents, tags and types. This can be used
     by clients to override the default behavior of paperless. See
-    `api-file_uploads`{.interpreted-text role="ref"}.
+    [POSTing documents](/api#file-uploads).
   - The document endpoint of API now serves documents in this form:
     - correspondents, document types and tags are referenced by
       their ID in the fields `correspondent`, `document_type` and
@@ -1590,16 +1861,14 @@ primarily.
   - Paperless now supports searching by tags, types and dates and
     correspondents. In order to have this applied to your existing
     documents, you need to perform a `document_index reindex`
-    management command (see `administration-index`{.interpreted-text
-    role="ref"}) that adds the data to the search index. You only
-    need to do this once, since the schema of the search index
-    changed. Paperless keeps the index updated after that whenever
-    something changes.
+    management command (see [document search index](/administration#index))
+    that adds the data to the search index. You only need to do this
+    once, since the schema of the search index changed. Paperless
+    keeps the index updated after that whenever something changes.
   - Paperless now has spelling corrections ("Did you mean") for
     miss-typed queries.
   - The documentation contains
-    `information about the query syntax <basic-searching>`{.interpreted-text
-    role="ref"}.
+    [information about the query syntax](/usage#basic-usage_searching).
 - Front end:
   - Clickable tags, correspondents and types allow quick filtering
     for related documents.
@@ -1660,10 +1929,8 @@ primarily.
 
 ### paperless-ng 0.9.0
 
-- **Deprecated:** GnuPG.
-  `See this note on the state of GnuPG in paperless-ng. <utilities-encyption>`{.interpreted-text
-  role="ref"} This features will most likely be removed in future
-  versions.
+- **Deprecated:** GnuPG. [See this note on the state of GnuPG in paperless-ng.](/administration#encryption)
+  This features will most likely be removed in future versions.
 - **Added:** New frontend. Features:
   - Single page application: It's much more responsive than the
     django admin pages.
@@ -1720,7 +1987,7 @@ primarily.
     uses PostgreSQL instead of SQLite. Username, database and
     password all default to `paperless` if not specified.
 - **Modified \[breaking\]:** document_retagger management command
-  rework. See `utilities-retagger`{.interpreted-text role="ref"} for
+  rework. See [Document retagger](/administration#retagger) for
   details. Replaces `document_correspondents` management command.
 - **Removed \[breaking\]:** Reminders.
 - **Removed:** All customizations made to the django admin pages.
@@ -1830,7 +2097,7 @@ primarily.
 ### 2.5.0
 
 - **New dependency**: Paperless now optimises thumbnail generation
-  with [optipng](http://optipng.sourceforge.net/), so you'll need to
+  with [optipng](https://optipng.sourceforge.net/), so you'll need to
   install that somewhere in your PATH or declare its location in
   `PAPERLESS_OPTIPNG_BINARY`. The Docker image has already been
   updated on the Docker Hub, so you just need to pull the latest one

docs/configuration.md

@@ -10,12 +10,10 @@ run paperless, these settings have to be defined in different places.
 - If you are running paperless on anything else, paperless will search
   for the configuration file in these locations and use the first one
   it finds:
-
-  ```
-  /path/to/paperless/paperless.conf
-  /etc/paperless.conf
-  /usr/local/etc/paperless.conf
-  ```
+  - The environment variable `PAPERLESS_CONFIGURATION_PATH`
+  - `/path/to/paperless/paperless.conf`
+  - `/etc/paperless.conf`
+  - `/usr/local/etc/paperless.conf`
 
 ## Required services
 
@@ -33,19 +31,19 @@ matcher.
     [More information on securing your Redis
     Instance](https://redis.io/docs/getting-started/#securing-redis).
 
-    Defaults to <redis://localhost:6379>.
+    Defaults to `redis://localhost:6379`.
 
 `PAPERLESS_DBENGINE=<engine_name>`
 
 : Optional, gives the ability to choose Postgres or MariaDB for
-database engine. Available options are [postgresql]{.title-ref} and
-[mariadb]{.title-ref}.
+database engine. Available options are `postgresql` and
+`mariadb`.
 
-    Default is [postgresql]{.title-ref}.
+    Default is `postgresql`.
 
     !!! warning
 
-        Using MariaDB comes with some caveats. See [MySQL Caveats](advanced_usage#mysql-caveats).
+        Using MariaDB comes with some caveats. See [MySQL Caveats](/advanced_usage#mysql-caveats).
 
 `PAPERLESS_DBHOST=<hostname>`
 
@@ -143,32 +141,46 @@ directory.
 files created using "collectstatic" manager command are stored.
 
     Unless you're doing something fancy, there is no need to override
-    this.
+    this.  If this is changed, you may need to run
+    `collectstatic` again.
 
     Defaults to "../static/", relative to the "src" directory.
 
 `PAPERLESS_FILENAME_FORMAT=<format>`
 
 : Changes the filenames paperless uses to store documents in the media
-directory. See [File name handling](advanced_usage#file_name_handling) for details.
+directory. See [File name handling](/advanced_usage#file-name-handling) for details.
 
     Default is none, which disables this feature.
 
 `PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=<bool>`
 
 : Tells paperless to replace placeholders in
-[PAPERLESS_FILENAME_FORMAT]{.title-ref} that would resolve to
+`PAPERLESS_FILENAME_FORMAT` that would resolve to
 'none' to be omitted from the resulting filename. This also holds
-true for directory names. See [File name handling](advanced_usage#file_name_handling) for
+true for directory names. See [File name handling](/advanced_usage#file-name-handling) for
 details.
 
-    Defaults to [false]{.title-ref} which disables this feature.
+    Defaults to `false` which disables this feature.
 
 `PAPERLESS_LOGGING_DIR=<path>`
 
 : This is where paperless will store log files.
 
-    Defaults to "`PAPERLESS_DATA_DIR`/log/".
+    Defaults to `PAPERLESS_DATA_DIR/log/`.
+
+`PAPERLESS_NLTK_DIR=<path>`
+
+: This is where paperless will search for the data required for NLTK
+processing, if you are using it. If you are using the Docker image,
+this should not be changed, as the data is included in the image
+already.
+
+Previously, the location defaulted to `PAPERLESS_DATA_DIR/nltk`.
+Unless you are using this in a bare metal install or other setup,
+this folder is no longer needed and can be removed manually.
+
+Defaults to `/usr/share/nltk_data`
 
 ## Logging
 
@@ -206,6 +218,11 @@ not include a trailing slash. E.g. <https://paperless.domain.com>
 
     Defaults to empty string, leaving the other settings unaffected.
 
+    !!! note
+
+        This value cannot contain a path (e.g. domain.com/path), even if
+        you are installing paperless-ngx at a subpath.
+
 `PAPERLESS_CSRF_TRUSTED_ORIGINS=<comma-separated-list>`
 
 : A list of trusted origins for unsafe requests (e.g. POST). As of
@@ -283,10 +300,10 @@ login with the selected user.
 : If this environment variable is specified, Paperless automatically
 creates a superuser with the provided username at start. This is
 useful in cases where you can not run the
-[createsuperuser]{.title-ref} command separately, such as Kubernetes
+`createsuperuser` command separately, such as Kubernetes
 or AWS ECS.
 
-    Requires [PAPERLESS_ADMIN_PASSWORD]{.title-ref} to be set.
+    Requires PAPERLESS_ADMIN_PASSWORD be set.
 
     !!! note
 
@@ -297,13 +314,13 @@ or AWS ECS.
 `PAPERLESS_ADMIN_MAIL=<email>`
 
 : (Optional) Specify superuser email address. Only used when
-[PAPERLESS_ADMIN_USER]{.title-ref} is set.
+PAPERLESS_ADMIN_USER is set.
 
     Defaults to `root@localhost`.
 
 `PAPERLESS_ADMIN_PASSWORD=<password>`
 
-: Only used when [PAPERLESS_ADMIN_USER]{.title-ref} is set. This will
+: Only used when PAPERLESS_ADMIN_USER is set. This will
 be the password of the automatically created superuser.
 
 `PAPERLESS_COOKIE_PREFIX=<str>`
@@ -331,26 +348,25 @@ applications.
         If you're exposing paperless to the internet directly, do not use
         this.
 
-        Also see the warning [in the official documentation
-        <https://docs.djangoproject.com/en/3.1/howto/auth-remote-user/#configuration>]{.title-ref}.
+        Also see the warning [in the official documentation](https://docs.djangoproject.com/en/3.1/howto/auth-remote-user/#configuration).
 
-    Defaults to [false]{.title-ref} which disables this feature.
+    Defaults to "false" which disables this feature.
 
 `PAPERLESS_HTTP_REMOTE_USER_HEADER_NAME=<str>`
 
-: If [PAPERLESS_ENABLE_HTTP_REMOTE_USER]{.title-ref} is enabled, this
+: If "PAPERLESS*ENABLE_HTTP_REMOTE_USER" is enabled, this
 property allows to customize the name of the HTTP header from which
 the authenticated username is extracted. Values are in terms of
-\[HttpRequest.META\](<https://docs.djangoproject.com/en/3.1/ref/request-response/#django.http.HttpRequest.META>).
-Thus, the configured value must start with [HTTP\_]{.title-ref}
+[HttpRequest.META](https://docs.djangoproject.com/en/3.1/ref/request-response/#django.http.HttpRequest.META).
+Thus, the configured value must start with `HTTP*`
 followed by the normalized actual header name.
 
-    Defaults to [HTTP_REMOTE_USER]{.title-ref}.
+    Defaults to "HTTP_REMOTE_USER".
 
 `PAPERLESS_LOGOUT_REDIRECT_URL=<str>`
 
 : URL to redirect the user to after a logout. This can be used
-together with [PAPERLESS_ENABLE_HTTP_REMOTE_USER]{.title-ref} to
+together with PAPERLESS_ENABLE_HTTP_REMOTE_USER to
 redirect the user back to the SSO application's logout page.
 
     Defaults to None, which disables this feature.
@@ -368,7 +384,7 @@ needs.
 parsing documents.
 
     It should be a 3-letter language code consistent with ISO 639:
-    <https://www.loc.gov/standards/iso639-2/php/code_list.php>
+    https://www.loc.gov/standards/iso639-2/php/code_list.php
 
     Set this to the language most of your documents are written in.
 
@@ -565,8 +581,10 @@ they use underscores instead of dashes.
 
 Paperless can make use of [Tika](https://tika.apache.org/) and
 [Gotenberg](https://gotenberg.dev/) for parsing and converting
-"Office" documents (such as ".doc", ".xlsx" and ".odt"). If you
-wish to use this, you must provide a Tika server and a Gotenberg server,
+"Office" documents (such as ".doc", ".xlsx" and ".odt").
+Tika and Gotenberg are also needed to allow parsing of E-Mails (.eml).
+
+If you wish to use this, you must provide a Tika server and a Gotenberg server,
 configure their endpoints, and enable the feature.
 
 `PAPERLESS_TIKA_ENABLED=<bool>`
@@ -605,14 +623,17 @@ services:
       PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
-  # ...
+    # ...
 
-  gotenberg:
-    image: gotenberg/gotenberg:7.6
-    restart: unless-stopped
-    command:
-      - 'gotenberg'
-      - '--chromium-disable-routes=true'
+    gotenberg:
+      image: gotenberg/gotenberg:7.8
+      restart: unless-stopped
+      # The gotenberg chromium route is used to convert .eml files. We do not
+      # want to allow external content like tracking pixels or even javascript.
+      command:
+        - 'gotenberg'
+        - '--chromium-disable-javascript=true'
+        - '--chromium-allow-list=file:///tmp/.*'
 
   tika:
     image: ghcr.io/paperless-ngx/tika:latest
@@ -624,8 +645,7 @@ Add the configuration variables to the environment of the webserver
 and add the additional services below the webserver service. Watch out
 for indentation.
 
-Make sure to use the correct format [PAPERLESS_TIKA_ENABLED =
-1]{.title-ref} so python_dotenv can parse the statement correctly.
+Make sure to use the correct format `PAPERLESS_TIKA_ENABLED = 1` so python_dotenv can parse the statement correctly.
 
 ## Software tweaks {#software_tweaks}
 
@@ -648,7 +668,7 @@ paperless will process in parallel on a single document.
 
         Ensure that the product
 
-        `PAPERLESS_TASK_WORKERS \:   PAPERLESS_THREADS_PER_WORKER`
+        `PAPERLESS_TASK_WORKERS * PAPERLESS_THREADS_PER_WORKER`
 
         does not exceed your CPU core count or else paperless will be
         extremely slow. If you want paperless to process many documents in
@@ -660,7 +680,7 @@ paperless will process in parallel on a single document.
     count, with a slight favor towards threads per worker:
 
     | CPU core count | Workers | Threads |
-    |----------------|---------|---------|
+    | -------------- | ------- | ------- |
     | > 1            | > 1     | > 1     |
     | > 2            | > 2     | > 1     |
     | > 4            | > 2     | > 2     |
@@ -693,6 +713,63 @@ for details on how to set it.
 
     Defaults to UTC.
 
+`PAPERLESS_ENABLE_NLTK=<bool>`
+
+: Enables or disables the advanced natural language processing
+used during automatic classification. If disabled, paperless will
+still perform some basic text pre-processing before matching.
+
+: See also `PAPERLESS_NLTK_DIR`.
+
+    Defaults to 1.
+
+`PAPERLESS_EMAIL_TASK_CRON=<cron expression>`
+
+: Configures the scheduled email fetching frequency. The value
+should be a valid crontab(5) expression describing when to run.
+
+: If set to the string "disable", no emails will be fetched automatically.
+
+    Defaults to `*/10 * * * *` or every ten minutes.
+
+`PAPERLESS_TRAIN_TASK_CRON=<cron expression>`
+
+: Configures the scheduled automatic classifier training frequency. The value
+should be a valid crontab(5) expression describing when to run.
+
+: If set to the string "disable", the classifier will not be trained automatically.
+
+    Defaults to `5 */1 * * *` or every hour at 5 minutes past the hour.
+
+`PAPERLESS_INDEX_TASK_CRON=<cron expression>`
+
+: Configures the scheduled search index update frequency. The value
+should be a valid crontab(5) expression describing when to run.
+
+: If set to the string "disable", the search index will not be automatically updated.
+
+    Defaults to `0 0 * * *` or daily at midnight.
+
+`PAPERLESS_SANITY_TASK_CRON=<cron expression>`
+
+: Configures the scheduled sanity checker frequency.
+
+: If set to the string "disable", the sanity checker will not run automatically.
+
+    Defaults to `30 0 * * sun` or Sunday at 30 minutes past midnight.
+
+`PAPERLESS_ENABLE_COMPRESSION=<bool>`
+
+: Enables compression of the responses from the webserver.
+
+: Defaults to 1, enabling compression.
+
+    !!! note
+
+        If you are using a proxy such as nginx, it is likely more efficient
+        to enable compression in your proxy configuration rather than
+        the webserver
+
 ## Polling {#polling}
 
 `PAPERLESS_CONSUMER_POLLING=<num>`
@@ -752,7 +829,7 @@ consumption directory as well.
 `PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=<bool>`
 
 : Set the names of subdirectories as tags for consumed files. E.g.
-<CONSUMPTION_DIR>/foo/bar/file.pdf will add the tags "foo" and
+`<CONSUMPTION_DIR>/foo/bar/file.pdf` will add the tags "foo" and
 "bar" to the consumed file. Paperless will create any tags that
 don't exist yet.
 
@@ -789,7 +866,7 @@ PAPERLESS_CONSUMER_ENABLE_BARCODES has been enabled.
 
     Defaults to false.
 
-PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
+`PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT`
 
 : Defines the string to be detected as a separator barcode. If
 paperless is used with the PATCH-T separator pages, users shouldn't
@@ -797,6 +874,31 @@ change this.
 
     Defaults to "PATCHT"
 
+`PAPERLESS_CONSUMER_ENABLE_ASN_BARCODE=<bool>`
+
+: Enables the detection of barcodes in the scanned document and
+setting the ASN (archive serial number) if a properly formatted
+barcode is detected.
+
+    The barcode must consist of a (configurable) prefix and the ASN
+    to be set, for instance `ASN00123`.
+
+    This option is compatible with barcode page separation, since
+    pages will be split up before reading the ASN.
+
+    If no ASN barcodes are detected in the uploaded file, no ASN will
+    be set. If a barcode with an already existing ASN is detected, no ASN
+    will be set either and a warning will be logged.
+
+    Defaults to false.
+
+`PAPERLESS_CONSUMER_ASN_BARCODE_PREFIX=ASN`
+
+: Defines the prefix that is used to identify a barcode as an ASN
+barcode.
+
+    Defaults to "ASN"
+
 `PAPERLESS_CONVERT_MEMORY_LIMIT=<num>`
 
 : On smaller systems, or even in the case of Very Large Documents, the
@@ -827,7 +929,7 @@ documents.
 
 : After a document is consumed, Paperless can trigger an arbitrary
 script if you like. This script will be passed a number of arguments
-for you to work with. For more information, take a look at [Post-consumption script](advanced_usage#post_consume_script).
+for you to work with. For more information, take a look at [Post-consumption script](/advanced_usage#post-consume-script).
 
     The default is blank, which means nothing will be executed.
 
@@ -841,8 +943,7 @@ option as specified in
 The filename will be checked first, and if nothing is found, the
 document text will be checked as normal.
 
-    A date in a filename must have some separators ([.]{.title-ref},
-    [-]{.title-ref}, [/]{.title-ref}, etc) for it to be parsed.
+    A date in a filename must have some separators (`.`, `,`, `-`, `/`, etc) for it to be parsed.
 
     Defaults to none, which disables this feature.
 
@@ -898,13 +999,20 @@ within your documents.
 `PAPERLESS_CONSUMER_IGNORE_PATTERNS=<json>`
 
 : By default, paperless ignores certain files and folders in the
-consumption directory, such as system files created by the Mac OS.
+consumption directory, such as system files created by the Mac OS
+or hidden folders some tools use to store data.
 
     This can be adjusted by configuring a custom json array with
     patterns to exclude.
 
+    For example, `.DS_STORE/*` will ignore any files found in a folder
+    named `.DS_STORE`, including `.DS_STORE/bar.pdf` and `foo/.DS_STORE/bar.pdf`
+
+    A pattern like `._*` will ignore anything starting with `._`, including:
+    `._foo.pdf` and `._bar/foo.pdf`
+
     Defaults to
-    `[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]`.
+    `[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]`.
 
 ## Binaries
 
@@ -928,7 +1036,7 @@ the literal path for that program.
 
 These options don't have any effect in `paperless.conf`. These options
 adjust the behavior of the docker container. Configure these in
-[docker-compose.env]{.title-ref}.
+`docker-compose.env`.
 
 `PAPERLESS_WEBSERVER_WORKERS=<num>`
 
@@ -946,7 +1054,7 @@ increase RAM usage.
 There are special setups where you may need to configure this value
 to restrict the Ip address or interface the webserver listens on.
 
-    Defaults to \[::\], meaning all interfaces, including IPv6.
+    Defaults to `[::]`, meaning all interfaces, including IPv6.
 
 `PAPERLESS_PORT=<port>`
 
@@ -1017,7 +1125,7 @@ configuration option:
 [Flower](https://flower.readthedocs.io/en/latest/index.html) will be
 started by the container.
 
-    You can read more about this in the [advanced documentation](advanced#celery-monitoring).
+    You can read more about this in the [advanced documentation](/advanced_usage#celery-monitoring).
 
 ## Update Checking {#update-checking}
 

docs/development.md

@@ -1,9 +1,9 @@
 # Development
 
 This section describes the steps you need to take to start development
-on paperless-ngx.
+on Paperless-ngx.
 
-Check out the source from github. The repository is organized in the
+Check out the source from GitHub. The repository is organized in the
 following way:
 
 - `main` always represents the latest release and will only see
@@ -12,7 +12,7 @@ following way:
 - `feature-X` contain bigger changes that will be in some release, but
   not necessarily the next one.
 
-When making functional changes to paperless, _always_ make your changes
+When making functional changes to Paperless-ngx, _always_ make your changes
 on the `dev` branch.
 
 Apart from that, the folder structure is as follows:
@@ -24,9 +24,9 @@ Apart from that, the folder structure is as follows:
   development.
 - `docker/` - Files required to build the docker image.
 
-## Contributing to Paperless
+## Contributing to Paperless-ngx
 
-Maybe you've been using Paperless for a while and want to add a feature
+Maybe you've been using Paperless-ngx for a while and want to add a feature
 or two, or maybe you've come across a bug that you have some ideas how
 to solve. The beauty of open source software is that you can see what's
 wrong and help to get it fixed for everyone!
@@ -36,242 +36,227 @@ conduct](https://github.com/paperless-ngx/paperless-ngx/blob/main/CODE_OF_CONDUC
 and other important information in the [contributing
 guidelines](https://github.com/paperless-ngx/paperless-ngx/blob/main/CONTRIBUTING.md).
 
-## Code formatting with pre-commit Hooks
+## Code formatting with pre-commit hooks
 
 To ensure a consistent style and formatting across the project source,
-the project utilizes a Git [pre-commit]{.title-ref} hook to perform some
-formatting and linting before a commit is allowed. That way, everyone
-uses the same style and some common issues can be caught early on. See
-below for installation instructions.
+the project utilizes Git [`pre-commit`](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks)
+hooks to perform some formatting and linting before a commit is allowed.
+That way, everyone uses the same style and some common issues can be caught
+early on.
 
 Once installed, hooks will run when you commit. If the formatting isn't
 quite right or a linter catches something, the commit will be rejected.
 You'll need to look at the output and fix the issue. Some hooks, such
-as the Python formatting tool [black]{.title-ref}, will format failing
-files, so all you need to do is [git add]{.title-ref} those files again
+as the Python formatting tool `black`, will format failing
+files, so all you need to do is `git add` those files again
 and retry your commit.
 
-## Initial setup and first start
+## General setup
 
-After you forked and cloned the code from github you need to perform a
-first-time setup. To do the setup you need to perform the steps from the
-following chapters in a certain order:
+After you forked and cloned the code from GitHub you need to perform a
+first-time setup.
+
+!!! note
+
+    Every command is executed directly from the root folder of the project unless specified otherwise.
 
 1.  Install prerequisites + pipenv as mentioned in
-    `[Bare metal route](/setup#bare_metal)
+    [Bare metal route](/setup#bare_metal).
 
 2.  Copy `paperless.conf.example` to `paperless.conf` and enable debug
-    mode.
+    mode within the file via `PAPERLESS_DEBUG=true`.
 
-3.  Install the Angular CLI interface:
+3.  Create `consume` and `media` directories:
 
-    ```shell-session
-    $ npm install -g @angular/cli
+    ```bash
+    $ mkdir -p consume media
     ```
 
-4.  Install pre-commit
+4.  Install the Python dependencies:
 
-    ```shell-session
-    pre-commit install
+    ```bash
+    $ pipenv install --dev
     ```
 
-5.  Create `consume` and `media` folders in the cloned root folder.
+    !!! note
 
-    ```shell-session
-    mkdir -p consume media
+        Using a virtual environment is highly recommended. You can spawn one via `pipenv shell`.
+        Make sure you're using Python 3.10.x or lower. Otherwise you might
+        get issues with building dependencies. You can use
+        [pyenv](https://github.com/pyenv/pyenv) to install a specific
+        Python version.
+
+5.  Install pre-commit hooks:
+
+    ```bash
+    $ pre-commit install
     ```
 
-6.  You can now either \...
+6.  Apply migrations and create a superuser for your development instance:
+
+    ```bash
+    # src/
+
+    $ python3 manage.py migrate
+    $ python3 manage.py createsuperuser
+    ```
+
+7.  You can now either ...
 
     - install redis or
 
-    - use the included scripts/start-services.sh to use docker to fire
+    - use the included `scripts/start_services.sh` to use docker to fire
       up a redis instance (and some other services such as tika,
       gotenberg and a database server) or
 
     - spin up a bare redis container
 
-      > ```shell-session
-      > docker run -d -p 6379:6379 --restart unless-stopped redis:latest
-      > ```
+      ```
+      $ docker run -d -p 6379:6379 --restart unless-stopped redis:latest
+      ```
 
-7.  Install the python dependencies by performing in the src/ directory.
-
-    ```shell-session
-    pipenv install --dev
-    ```
-
-> - Make sure you're using python 3.9.x or lower. Otherwise you might
->   get issues with building dependencies. You can use
->   [pyenv](https://github.com/pyenv/pyenv) to install a specific
->   python version.
-
-8.  Generate the static UI so you can perform a login to get session
-    that is required for frontend development (this needs to be done one
-    time only). From src-ui directory:
-
-    ```shell-session
-    npm install .
-    ./node_modules/.bin/ng build --configuration production
-    ```
-
-9.  Apply migrations and create a superuser for your dev instance:
-
-    ```shell-session
-    python3 manage.py migrate
-    python3 manage.py createsuperuser
-    ```
-
-10. Now spin up the dev backend. Depending on which part of paperless
-    you're developing for, you need to have some or all of them
-    running.
-
-> ```shell-session
-> python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
-> ```
-
-11. Login with the superuser credentials provided in step 8 at
-    `http://localhost:8000` to create a session that enables you to use
-    the backend.
-
-Backend development environment is now ready, to start Frontend
-development go to `/src-ui` and run `ng serve`. From there you can use
-`http://localhost:4200` for a preview.
+8.  Continue with either back-end or front-end development – or both :-).
 
 ## Back end development
 
-The backend is a django application. PyCharm works well for development,
-but you can use whatever you want.
+The back end is a [Django](https://www.djangoproject.com/) application. [PyCharm](https://www.jetbrains.com/de-de/pycharm/) as well as [Visual Studio Code](https://code.visualstudio.com) work well for development, but you can use whatever you want.
 
-Configure the IDE to use the src/ folder as the base source folder.
+Configure the IDE to use the `src/`-folder as the base source folder.
 Configure the following launch configurations in your IDE:
 
-- python3 manage.py runserver
-- celery \--app paperless worker
-- python3 manage.py document_consumer
+- `python3 manage.py runserver`
+- `python3 manage.py document_consumer`
+- `celery --app paperless worker -l DEBUG` (or any other log level)
 
 To start them all:
 
-```shell-session
-python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
+```bash
+# src/
+
+$ python3 manage.py runserver & \
+  python3 manage.py document_consumer & \
+  celery --app paperless worker -l DEBUG
 ```
 
-Testing and code style:
+You might need the front end to test your back end code. This assumes that you have AngularJS installed on your system. Go to the [Front end development](#front-end-development) section for further details. To build the front end once use this commmand:
 
-- Run `pytest` in the src/ directory to execute all tests. This also
-  generates a HTML coverage report. When runnings test, paperless.conf
-  is loaded as well. However: the tests rely on the default
+```bash
+# src-ui/
+
+$ npm install
+$ ng build --configuration production
+```
+
+### Testing
+
+- Run `pytest` in the `src/` directory to execute all tests. This also
+  generates a HTML coverage report. When runnings test, `paperless.conf`
+  is loaded as well. However, the tests rely on the default
   configuration. This is not ideal. But for now, make sure no settings
   except for DEBUG are overridden when testing.
 
-- Coding style is enforced by the Git pre-commit hooks. These will
-  ensure your code is formatted and do some linting when you do a [git
-  commit]{.title-ref}.
-
-- You can also run `black` manually to format your code
-
-  !!! note
+!!! note
 
       The line length rule E501 is generally useful for getting multiple
       source files next to each other on the screen. However, in some
       cases, its just not possible to make some lines fit, especially
-      complicated IF cases. Append `# NOQA: E501` to disable this check
+      complicated IF cases. Append `# noqa: E501` to disable this check
       for certain lines.
 
 ## Front end development
 
-The front end is built using Angular. In order to get started, you need
-`npm`. Install the Angular CLI interface with
+The front end is built using AngularJS. In order to get started, you need Node.js (version 14.15+) and
+`npm`.
 
-```shell-session
-$ npm install -g @angular/cli
-```
+!!! note
 
-and make sure that it's on your path. Next, in the src-ui/ directory,
-install the required dependencies of the project.
+    The following commands are all performed in the `src-ui`-directory. You will need a running back end (including an active session) to connect to the back end API. To spin it up refer to the commands under the section [above](#back-end-development).
 
-```shell-session
-$ npm install
-```
+1. Install the Angular CLI. You might need sudo privileges
+   to perform this command:
 
-You can launch a development server by running
+   ```bash
+   $ npm install -g @angular/cli
+   ```
 
-```shell-session
-$ ng serve
-```
+2. Make sure that it's on your path.
 
-This will automatically update whenever you save. However, in-place
-compilation might fail on syntax errors, in which case you need to
-restart it.
+3. Install all neccessary modules:
 
-By default, the development server is available on
-`http://localhost:4200/` and is configured to access the API at
-`http://localhost:8000/api/`, which is the default of the backend. If
-you enabled DEBUG on the back end, several security overrides for
-allowed hosts, CORS and X-Frame-Options are in place so that the front
-end behaves exactly as in production. This also relies on you being
-logged into the back end. Without a valid session, The front end will
-simply not work.
+   ```bash
+   $ npm install
+   ```
 
-Testing and code style:
+4. You can launch a development server by running:
 
-- The frontend code (.ts, .html, .scss) use `prettier` for code
+   ```bash
+   $ ng serve
+   ```
+
+   This will automatically update whenever you save. However, in-place
+   compilation might fail on syntax errors, in which case you need to
+   restart it.
+
+   By default, the development server is available on `http://localhost:4200/` and is configured to access the API at
+   `http://localhost:8000/api/`, which is the default of the backend. If you enabled `DEBUG` on the back end, several security overrides for allowed hosts, CORS and X-Frame-Options are in place so that the front end behaves exactly as in production.
+
+### Testing and code style
+
+- The front end code (.ts, .html, .scss) use `prettier` for code
   formatting via the Git `pre-commit` hooks which run automatically on
-  commit. See
-  [above](#code-formatting-with-pre-commit-hooks) for installation. You can also run this via cli with a
+  commit. See [above](#code-formatting-with-pre-commit-hooks) for installation instructions. You can also run this via the CLI with a
   command such as
 
-  ```shell-session
+  ```bash
   $ git ls-files -- '*.ts' | xargs pre-commit run prettier --files
   ```
 
-- Frontend testing uses jest and cypress. There is currently a need
-  for significantly more frontend tests. Unit tests and e2e tests,
+- Front end testing uses jest and cypress. There is currently a need
+  for significantly more front end tests. Unit tests and e2e tests,
   respectively, can be run non-interactively with:
 
-  ```shell-session
+  ```bash
   $ ng test
   $ npm run e2e:ci
   ```
 
-  Cypress also includes a UI which can be run from within the `src-ui`
-  directory with
+  - Cypress also includes a UI which can be run with:
 
-  ```shell-session
-  $ ./node_modules/.bin/cypress open
+    ```bash
+    $ ./node_modules/.bin/cypress open
+    ```
+
+- In order to build the front end and serve it as part of Django, execute:
+
+  ```bash
+  $ ng build --configuration production
   ```
 
-In order to build the front end and serve it as part of django, execute
-
-```shell-session
-$ ng build --prod
-```
-
-This will build the front end and put it in a location from which the
-Django server will serve it as static content. This way, you can verify
-that authentication is working.
+  This will build the front end and put it in a location from which the
+  Django server will serve it as static content. This way, you can verify
+  that authentication is working.
 
 ## Localization
 
-Paperless is available in many different languages. Since paperless
-consists both of a django application and an Angular front end, both
+Paperless-ngx is available in many different languages. Since Paperless-ngx
+consists both of a Django application and an AngularJS front end, both
 these parts have to be translated separately.
 
 ### Front end localization
 
-- The Angular front end does localization according to the [Angular
+- The AngularJS front end does localization according to the [Angular
   documentation](https://angular.io/guide/i18n).
 - The source language of the project is "en_US".
-- The source strings end up in the file "src-ui/messages.xlf".
+- The source strings end up in the file `src-ui/messages.xlf`.
 - The translated strings need to be placed in the
-  "src-ui/src/locale/" folder.
+  `src-ui/src/locale/` folder.
 - In order to extract added or changed strings from the source files,
   call `ng xi18n --ivy`.
 
 Adding new languages requires adding the translated files in the
-"src-ui/src/locale/" folder and adjusting a couple files.
+`src-ui/src/locale/` folder and adjusting a couple files.
 
-1.  Adjust "src-ui/angular.json":
+1.  Adjust `src-ui/angular.json`:
 
     ```json
     "i18n": {
@@ -288,7 +273,7 @@ Adding new languages requires adding the translated files in the
     ```
 
 2.  Add the language to the available options in
-    "src-ui/src/app/services/settings.service.ts":
+    `src-ui/src/app/services/settings.service.ts`:
 
     ```typescript
     getLanguageOptions(): LanguageOption[] {
@@ -309,7 +294,7 @@ Adding new languages requires adding the translated files in the
     and "yyyy".
 
 3.  Import and register the Angular data for this locale in
-    "src-ui/src/app/app.module.ts":
+    `src-ui/src/app/app.module.ts`:
 
     ```typescript
     import localeDe from '@angular/common/locales/de'
@@ -322,10 +307,10 @@ A majority of the strings that appear in the back end appear only when
 the admin is used. However, some of these are still shown on the front
 end (such as error messages).
 
-- The django application does localization according to the [django
+- The django application does localization according to the [Django
   documentation](https://docs.djangoproject.com/en/3.1/topics/i18n/translation/).
 - The source language of the project is "en_US".
-- Localization files end up in the folder "src/locale/".
+- Localization files end up in the folder `src/locale/`.
 - In order to extract strings from the application, call
   `python3 manage.py makemessages -l en_US`. This is important after
   making changes to translatable strings.
@@ -336,8 +321,8 @@ end (such as error messages).
   command.
 
 Adding new languages requires adding the translated files in the
-"src/locale/" folder and adjusting the file
-"src/paperless/settings.py" to include the new language:
+`src/locale/`-folder and adjusting the file
+`src/paperless/settings.py` to include the new language:
 
 ```python
 LANGUAGES = [
@@ -353,20 +338,30 @@ LANGUAGES = [
 
 ## Building the documentation
 
-The documentation is built using material-mkdocs, see their [documentation](https://squidfunk.github.io/mkdocs-material/reference/). If you want to build the documentation locally, this is how you do it:
+The documentation is built using material-mkdocs, see their [documentation](https://squidfunk.github.io/mkdocs-material/reference/).
+If you want to build the documentation locally, this is how you do it:
 
-1.  Install python dependencies.
+1.  Have an active pipenv shell (`pipenv shell`) and install Python dependencies:
 
-    ```shell-session
-    $ cd /path/to/paperless
+    ```bash
     $ pipenv install --dev
     ```
 
 2.  Build the documentation
 
-    ```shell-session
-    $ cd /path/to/paperless
-    $ pipenv mkdocs build
+    ```bash
+    $ mkdocs build --config-file mkdocs.yml
+    ```
+
+    _alternatively..._
+
+3.  Serve the documentation. This will spin up a
+    copy of the documentation at http://127.0.0.1:8000
+    that will automatically refresh everytime you change
+    something.
+
+    ```bash
+    $ mkdocs serve
     ```
 
 ## Building the Docker image
@@ -379,35 +374,35 @@ helper script `build-docker-image.sh`.
 
 Building the docker image from source:
 
-> ```shell-session
-> ./build-docker-image.sh Dockerfile -t <your-tag>
-> ```
+```bash
+./build-docker-image.sh Dockerfile -t <your-tag>
+```
 
-## Extending Paperless
+## Extending Paperless-ngx
 
-Paperless does not have any fancy plugin systems and will probably never
+Paperless-ngx does not have any fancy plugin systems and will probably never
 have. However, some parts of the application have been designed to allow
 easy integration of additional features without any modification to the
 base code.
 
 ### Making custom parsers
 
-Paperless uses parsers to add documents to paperless. A parser is
+Paperless-ngx uses parsers to add documents. A parser is
 responsible for:
 
-- Retrieve the content from the original
-- Create a thumbnail
-- Optional: Retrieve a created date from the original
-- Optional: Create an archived document from the original
+- Retrieving the content from the original
+- Creating a thumbnail
+- _optional:_ Retrieving a created date from the original
+- _optional:_ Creainge an archived document from the original
 
-Custom parsers can be added to paperless to support more file types. In
+Custom parsers can be added to Paperless-ngx to support more file types. In
 order to do that, you need to write the parser itself and announce its
-existence to paperless.
+existence to Paperless-ngx.
 
 The parser itself must extend `documents.parsers.DocumentParser` and
 must implement the methods `parse` and `get_thumbnail`. You can provide
 your own implementation to `get_date` if you don't want to rely on
-paperless' default date guessing mechanisms.
+Paperless-ngx' default date guessing mechanisms.
 
 ```python
 class MyCustomParser(DocumentParser):
@@ -428,7 +423,7 @@ class MyCustomParser(DocumentParser):
     def get_thumbnail(self, document_path, mime_type):
         # This should return the path to a thumbnail you created for this
         # document.
-        return os.path.join(self.tempdir, "thumb.png")
+        return os.path.join(self.tempdir, "thumb.webp")
 ```
 
 If you encounter any issues during parsing, raise a
@@ -439,7 +434,7 @@ to be empty and removed after consumption finished. You can use that
 directory to store any intermediate files and also use it to store the
 thumbnail / archived document.
 
-After that, you need to announce your parser to paperless. You need to
+After that, you need to announce your parser to Paperless-ngx. You need to
 connect a handler to the `document_consumer_declaration` signal. Have a
 look in the file `src/paperless_tesseract/apps.py` on how that's done.
 The handler is a method that returns information about your parser:
@@ -459,11 +454,11 @@ def myparser_consumer_declaration(sender, **kwargs):
 - `parser` is a reference to a class that extends `DocumentParser`.
 - `weight` is used whenever two or more parsers are able to parse a
   file: The parser with the higher weight wins. This can be used to
-  override the parsers provided by paperless.
+  override the parsers provided by Paperless-ngx.
 - `mime_types` is a dictionary. The keys are the mime types your
   parser supports and the value is the default file extension that
-  paperless should use when storing files and serving them for
+  Paperless-ngx should use when storing files and serving them for
   download. We could guess that from the file extensions, but some
-  mime types have many extensions associated with them and the python
+  mime types have many extensions associated with them and the Python
   methods responsible for guessing the extension do not always return
   the same value.

docs/faq.md

@@ -1,6 +1,6 @@
 # Frequently Asked Questions
 
-### _What's the general plan for Paperless-ngx?_
+## _What's the general plan for Paperless-ngx?_
 
 **A:** While Paperless-ngx is already considered largely
 "feature-complete" it is a community-driven project and development
@@ -9,7 +9,7 @@ discussions and "up-voted" by the community but this is not a
 guarantee the feature will be implemented. This project will always be
 open to collaboration in the form of PRs, ideas etc.
 
-### _I'm using docker. Where are my documents?_
+## _I'm using docker. Where are my documents?_
 
 **A:** Your documents are stored inside the docker volume
 `paperless_media`. Docker manages this volume automatically for you. It
@@ -27,9 +27,7 @@ system. On Linux, chances are high that this location is
     files around manually. This folder is meant to be entirely managed by
     docker and paperless.
 
-### Let's say I want to switch tools in a year. Can I easily move
-
-to other systems?\*
+## Let's say I want to switch tools in a year. Can I easily move to other systems?
 
 **A:** Your documents are stored as plain files inside the media folder.
 You can always drag those files out of that folder to use them
@@ -41,27 +39,27 @@ elsewhere. Here are a couple notes about that.
 - By default, paperless uses the internal ID of each document as its
   filename. This might not be very convenient for export. However, you
   can adjust the way files are stored in paperless by
-  [configuring the filename format](advanced_usage#file_name_handling).
-- [The exporter](administration#exporter) is
+  [configuring the filename format](/advanced_usage#file-name-handling).
+- [The exporter](/administration#exporter) is
   another easy way to get your files out of paperless with reasonable
   file names.
 
-### _What file types does paperless-ngx support?_
+## _What file types does paperless-ngx support?_
 
 **A:** Currently, the following files are supported:
 
-- PDF documents, PNG images, JPEG images, TIFF images and GIF images
-  are processed with OCR and converted into PDF documents.
+- PDF documents, PNG images, JPEG images, TIFF images, GIF images and
+  WebP images are processed with OCR and converted into PDF documents.
 - Plain text documents are supported as well and are added verbatim to
   paperless.
-- With the optional Tika integration enabled (see [Tika configuration](configuration#tika),
+- With the optional Tika integration enabled (see [Tika configuration](/configuration#tika),
   Paperless also supports various Office documents (.docx, .doc, odt,
   .ppt, .pptx, .odp, .xls, .xlsx, .ods).
 
 Paperless-ngx determines the type of a file by inspecting its content.
 The file extensions do not matter.
 
-### _Will paperless-ngx run on Raspberry Pi?_
+## _Will paperless-ngx run on Raspberry Pi?_
 
 **A:** The short answer is yes. I've tested it on a Raspberry Pi 3 B.
 The long answer is that certain parts of Paperless will run very slow,
@@ -73,11 +71,11 @@ has to do much less work to serve the data.
 !!! note
 
     You can adjust some of the settings so that paperless uses less
-    processing power. See [setup](setup#less_powerful_devices) for details.
+    processing power. See [setup](/setup#less-powerful-devices) for details.
 
-### _How do I install paperless-ngx on Raspberry Pi?_
+## _How do I install paperless-ngx on Raspberry Pi?_
 
-**A:** Docker images are available for arm and arm64 hardware, so just
+**A:** Docker images are available for armv7 and arm64 hardware, so just
 follow the docker-compose instructions. Apart from more required disk
 space compared to a bare metal installation, docker comes with close to
 zero overhead, even on Raspberry Pi.
@@ -87,13 +85,13 @@ the python requirements do not have precompiled packages for ARM /
 ARM64. Installation of these will require additional development
 libraries and compilation will take a long time.
 
-### _How do I run this on Unraid?_
+## _How do I run this on Unraid?_
 
 **A:** Paperless-ngx is available as [community
 app](https://unraid.net/community/apps?q=paperless-ngx) in Unraid. [Uli
 Fahrer](https://github.com/Tooa) created a container template for that.
 
-### _How do I run this on my toaster?_
+## _How do I run this on my toaster?_
 
 **A:** I honestly don't know! As for all other devices that might be
 able to run paperless, you're a bit on your own. If you can't run the
@@ -103,11 +101,11 @@ This is also what I use to test new releases with. Apart from that, I
 also have a Raspberry Pi, which I occasionally build the image on and
 see if it works.
 
-### _How do I proxy this with NGINX?_
+## _How do I proxy this with NGINX?_
 
-**A:** See [here](setup#nginx).
+**A:** See [here](/setup#nginx).
 
-### _How do I get WebSocket support with Apache mod_wsgi_?
+## _How do I get WebSocket support with Apache mod_wsgi_?
 
 **A:** `mod_wsgi` by itself does not support ASGI. Paperless will
 continue to work with WSGI, but certain features such as status

docs/index.md

@@ -5,7 +5,7 @@
 **Paperless-ngx** is a _community-supported_ open-source document management system that transforms your
 physical documents into a searchable online archive so you can keep, well, _less paper_.
 
-[Get started](/setup/){ .md-button .md-button--primary .index-callout }
+[Get started](/setup){ .md-button .md-button--primary .index-callout }
 [Demo](https://demo.paperless-ngx.com){ .md-button .md-button--secondary target=\_blank }
 
 </div>
@@ -50,12 +50,12 @@ If you want to learn about what's different in paperless-ngx from
 Paperless, check out these resources in the documentation:
 
 - [Some screenshots](#screenshots) of the new UI are available.
-- Read [this section](/advanced_usage/#advanced-automatic_matching) if you want to learn about how paperless automates all
+- Read [this section](/advanced_usage#automatic-matching) if you want to learn about how paperless automates all
   tagging using machine learning.
-- Paperless now comes with a [proper email consumer](/usage/#usage-email) that's fully tested and production ready.
+- Paperless now comes with a [proper email consumer](/usage#usage-email) that's fully tested and production ready.
 - Paperless creates searchable PDF/A documents from whatever you put into the consumption directory. This means
   that you can select text in image-only documents coming from your scanner.
-- See [this note](/administration/#utilities-encyption) about GnuPG encryption in paperless-ngx.
+- See [this note](/administration#encryption) about GnuPG encryption in paperless-ngx.
 - Paperless is now integrated with a
   [task processing queue](/setup#task_processor) that tells you at a glance when and why something is not working.
 - The [changelog](/changelog) contains a detailed list of all changes in paperless-ngx.

docs/setup.md

@@ -2,10 +2,10 @@
 
 You can go multiple routes to setup and run Paperless:
 
-- [Use the easy install docker script](/setup#docker_script)
-- [Pull the image from Docker Hub](/setup#docker_hub)
-- [Build the Docker image yourself](/setup#docker_build)
-- [Install Paperless directly on your system manually (bare metal)](/setup#bare_metal)
+- [Use the easy install docker script](#docker_script)
+- [Pull the image from Docker Hub](#docker_hub)
+- [Build the Docker image yourself](#docker_build)
+- [Install Paperless directly on your system manually (bare metal)](#bare_metal)
 
 The Docker routes are quick & easy. These are the recommended routes.
 This configures all the stuff from the above automatically so that it
@@ -46,7 +46,7 @@ steps described in [Docker setup](#docker_hub) automatically.
     page](https://github.com/paperless-ngx/paperless-ngx/tree/master/docker/compose)
     and download one of the `docker-compose.*.yml` files,
     depending on which database backend you want to use. Rename this
-    file to `docker-compose.*.yml`. If you want to enable
+    file to `docker-compose.yml`. If you want to enable
     optional support for Office documents, download a file with
     `-tika` in the file name. Download the
     `docker-compose.env` file and the `.env` file as well and store them
@@ -64,8 +64,7 @@ steps described in [Docker setup](#docker_hub) automatically.
 
         If you want to use the included `docker-compose.*.yml` file, you
         need to have at least Docker version **17.09.0** and docker-compose
-        version **1.17.0**. To check do: `docker-compose -v` or
-        `docker -v`
+        version **1.17.0**. To check do: `docker-compose -v` or `docker -v`
 
         See the [Docker installation guide](https://docs.docker.com/engine/install/) on how to install the current
         version of Docker for your operating system or Linux distribution of
@@ -144,21 +143,13 @@ steps described in [Docker setup](#docker_hub) automatically.
     !!! note
 
         You can copy any setting from the file `paperless.conf.example` and
-        paste it here. Have a look at [configuration](/configuration] to see what's available.
+        paste it here. Have a look at [configuration](/configuration) to see what's available.
 
     !!! note
 
-        You can utilize Docker secrets for some configuration settings by
-        appending `_FILE` to some configuration values. This is
-        supported currently only by:
-
-        - PAPERLESS_DBUSER
-        - PAPERLESS_DBPASS
-        - PAPERLESS_SECRET_KEY
-        - PAPERLESS_AUTO_LOGIN_USERNAME
-        - PAPERLESS_ADMIN_USER
-        - PAPERLESS_ADMIN_MAIL
-        - PAPERLESS_ADMIN_PASSWORD
+        You can utilize Docker secrets for configuration settings by
+        appending `_FILE` to configuration values. For example `PAPERLESS_DBUSER`
+        can be set using `PAPERLESS_DBUSER_FILE=/var/run/secrets/password.txt`.
 
     !!! warning
 
@@ -234,9 +225,8 @@ steps described in [Docker setup](#docker_hub) automatically.
         certain, more updated software. If you want to build these images
         your self, that is possible, but beyond the scope of these steps.
 
-4.  Follow steps 3 to 8 of [Docker Setup](setup#docker_hub)
-    role="ref"}. When asked to run `docker-compose pull` to pull the
-    image, do
+4.  Follow steps 3 to 8 of [Docker Setup](#docker_hub). When asked to run
+    `docker-compose pull` to pull the image, do
 
     ```shell-session
     $ docker-compose build
@@ -315,21 +305,42 @@ supported.
         extension](https://code.djangoproject.com/wiki/JSON1Extension) is
         enabled. This is usually the case, but not always.
 
-4.  Get the release archive from
-    <https://github.com/paperless-ngx/paperless-ngx/releases>. If you
-    clone the git repo as it is, you also have to compile the front end
-    by yourself. Extract the archive to a place from where you wish to
-    execute it, such as `/opt/paperless`.
+4.  Create a system user with a new home folder under which you wish
+    to run paperless.
 
-5.  Configure paperless. See [configuration](configuration) for details.
+    ```shell-session
+    adduser paperless --system --home /opt/paperless --group
+    ```
+
+5.  Get the release archive from
+    <https://github.com/paperless-ngx/paperless-ngx/releases> for example with
+
+    ```shell-session
+    curl -O -L https://github.com/paperless-ngx/paperless-ngx/releases/download/v1.10.2/paperless-ngx-v1.10.2.tar.xz
+    ```
+
+    Extract the archive with
+
+    ```shell-session
+    tar -xf paperless-ngx-v1.10.2.tar.xz
+    ```
+
+    and copy the contents to the
+    home folder of the user you created before (`/opt/paperless`).
+
+    Optional: If you cloned the git repo, you will have to
+    compile the frontend yourself, see [here](/development#front-end-development)
+    and use the `build` step, not `serve`.
+
+6.  Configure paperless. See [configuration](/configuration) for details.
     Edit the included `paperless.conf` and adjust the settings to your
     needs. Required settings for getting
     paperless running are:
 
     - `PAPERLESS_REDIS` should point to your redis server, such as
       <redis://localhost:6379>.
-    - `PAPERLESS_DBENGINE` optional, and should be one of [postgres,
-      mariadb, or sqlite]{.title-ref}
+    - `PAPERLESS_DBENGINE` optional, and should be one of `postgres`,
+      `mariadb`, or `sqlite`
     - `PAPERLESS_DBHOST` should be the hostname on which your
       PostgreSQL server is running. Do not configure this to use
       SQLite instead. Also configure port, database name, user and
@@ -344,7 +355,7 @@ supported.
       allows third parties to forge authentication credentials.
     - `PAPERLESS_URL` if you are behind a reverse proxy. This should
       point to your domain. Please see
-      [configuration](configuration) for more
+      [configuration](/configuration) for more
       information.
 
     Many more adjustments can be made to paperless, especially the OCR
@@ -354,29 +365,31 @@ supported.
       documents are written in.
     - Set `PAPERLESS_TIME_ZONE` to your local time zone.
 
-6.  Create a system user under which you wish to run paperless.
-
-    ```shell-session
-    adduser paperless --system --home /opt/paperless --group
-    ```
-
-7.  Ensure that these directories exist and that the paperless user has
-    write permissions to the following directories:
+7.  Create the following directories if they are missing:
 
     - `/opt/paperless/media`
     - `/opt/paperless/data`
     - `/opt/paperless/consume`
 
     Adjust as necessary if you configured different folders.
-
-8.  Install python requirements from the `requirements.txt` file. It is
-    up to you if you wish to use a virtual environment or not. First you
-    should update your pip, so it gets the actual packages.
+    Ensure that the paperless user has write permissions for every one
+    of these folders with
 
     ```shell-session
-    sudo -Hu paperless pip3 install --upgrade pip
+    ls -l -d /opt/paperless/media
     ```
 
+    If needed, change the owner with
+
+    ```shell-session
+    sudo chown paperless:paperless /opt/paperless/media
+    sudo chown paperless:paperless /opt/paperless/data
+    sudo chown paperless:paperless /opt/paperless/consume
+    ```
+
+8.  Install python requirements from the `requirements.txt` file. It is
+    up to you if you wish to use a virtual environment or not. First you should update your pip, so it gets the actual packages.
+
     ```shell-session
     sudo -Hu paperless pip3 install -r requirements.txt
     ```
@@ -387,21 +400,25 @@ supported.
 9.  Go to `/opt/paperless/src`, and execute the following commands:
 
     ```bash
-    \# This creates the database schema.
+    # This creates the database schema.
     sudo -Hu paperless python3 manage.py migrate
 
-    \# This creates your first paperless user
+    # This creates your first paperless user
     sudo -Hu paperless python3 manage.py createsuperuser
     ```
 
 10. Optional: Test that paperless is working by executing
 
     ```bash
-    \# This collects static files from paperless and django.
+    # Manually starts the webserver
     sudo -Hu paperless python3 manage.py runserver
     ```
 
-    and pointing your browser to <http://localhost:8000/>.
+    and pointing your browser to http://localhost:8000 if
+    accessing from the same devices on which paperless is installed.
+    If accessing from another machine, set up systemd services. You may need
+    to set `PAPERLESS_DEBUG=true` in order for the development server to work
+    normally in your browser.
 
     !!! warning
 
@@ -452,6 +469,14 @@ supported.
         For instructions on how to use nginx for that,
         [see the instructions below](/setup#nginx).
 
+    !!! warning
+
+        If celery won't start (check with
+        `sudo systemctl status paperless-task-queue.service` for
+        paperless-task-queue.service and paperless-scheduler.service
+        ) you need to change the path in the files. Example:
+        `ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
+
 12. Optional: Install a samba server and make the consumption folder
     available as a network share.
 
@@ -480,7 +505,7 @@ supported.
     not available for most distributions.
 
 15. Optional: If using the NLTK machine learning processing (see
-    `PAPERLESS_ENABLE_NLTK` in [configuration](configuration#software_tweaks) for details),
+    `PAPERLESS_ENABLE_NLTK` in [configuration](/configuration#software_tweaks) for details),
     download the NLTK data for the Snowball
     Stemmer, Stopwords and Punkt tokenizer to your
     `PAPERLESS_DATA_DIR/nltk`. Refer to the [NLTK
@@ -562,7 +587,7 @@ Migration to paperless-ngx is then performed in a few simple steps:
     the docker-compose files from
     [here](https://github.com/paperless-ngx/paperless-ngx/tree/master/docker/compose)
     or clone the repository to build the image yourself (see
-    [above](/setup#docker_build)). You can
+    [above](#docker_build)). You can
     either replace your current paperless folder or put paperless-ngx in
     a different location.
 
@@ -586,7 +611,7 @@ Migration to paperless-ngx is then performed in a few simple steps:
     after you migrated your existing SQLite database.
 
 5.  Adjust `docker-compose.yml` and `docker-compose.env` to your needs.
-    See [Docker setup](setup#docker_hub) details on
+    See [Docker setup](#docker_hub) details on
     which edits are advised.
 
 6.  [Update paperless.](/administration#updating)
@@ -676,13 +701,19 @@ below use PostgreSQL, but are applicable to MySQL/MariaDB with the
 !!! warning
 
     MySQL is case insensitive by default, treating values like "Name" and
-    "NAME" as identical. See [MySQL caveats](advanced##mysql-caveats) for details.
+    "NAME" as identical. See [MySQL caveats](/advanced_usage#mysql-caveats) for details.
 
 !!! warning
 
     MySQL also enforces limits on maximum lengths, but does so differently than
     PostgreSQL.  It may not be possible to migrate to MySQL due to this.
 
+!!! warning
+
+    Using mariadb version 10.4+ is recommended. Using the `utf8mb3` character set on
+    an older system may fix issues that can arise while setting up Paperless-ngx but
+    `utf8mb3` can cause issues with consumption (where `utf8mb4` does not).
+
 1.  Stop paperless, if it is running.
 
 2.  Tell paperless to use PostgreSQL:
@@ -691,7 +722,7 @@ below use PostgreSQL, but are applicable to MySQL/MariaDB with the
     file to `docker-compose.yml`. Remember to adjust the consumption
     directory, if necessary.
     b) Without docker, configure the database in your `paperless.conf`
-    file. See [configuration](configuration) for
+    file. See [configuration](/configuration) for
     details.
 
 3.  Open a shell and initialize the database:
@@ -746,7 +777,9 @@ with a few simple steps.
 
 Paperless-ngx modified the database schema slightly, however, these
 changes can be reverted while keeping your current data, so that your
-current data will be compatible with original Paperless.
+current data will be compatible with original Paperless. Thumbnails
+were also changed from PNG to WEBP format and will need to be
+re-generated.
 
 Execute this:
 
@@ -762,11 +795,11 @@ $ cd /path/to/paperless/src
 $ python3 manage.py migrate documents 0023
 ```
 
-After that, you need to clear your cookies (Paperless-ngx comes with
-updated dependencies that do cookie-processing differently) and probably
-your cache as well.
+After regenerating thumbnails, you'll need to clear your cookies
+(Paperless-ngx comes with updated dependencies that do cookie-processing
+differently) and probably your cache as well.
 
-# Considerations for less powerful devices {#less_powerful_devices}
+# Considerations for less powerful devices {#less-powerful-devices}
 
 Paperless runs on Raspberry Pi. However, some things are rather slow on
 the Pi and configuring some options in paperless can help improve
@@ -797,12 +830,12 @@ performance immensely:
   more advanced language processing, which can take more memory and
   processing time.
 
-For details, refer to [configuration](configuration).
+For details, refer to [configuration](/configuration).
 
 !!! note
 
     Updating the
-    [automatic matching algorithm](/advanced_usage#automatic_matching) takes quite a bit of time. However, the update mechanism
+    [automatic matching algorithm](/advanced_usage#automatic-matching) takes quite a bit of time. However, the update mechanism
     checks if your data has changed before doing the heavy lifting. If you
     experience the algorithm taking too much cpu time, consider changing the
     schedule in the admin interface to daily. You can also manually invoke
@@ -843,13 +876,14 @@ http {
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Host $server_name;
+            add_header P3P 'CP=""'; # may not be required in all setups
         }
     }
 }
 ```
 
 The `PAPERLESS_URL` configuration variable is also required when using a
-reverse proxy. Please refer to the [hosting and security](configuration#hosting-and-security) docs.
+reverse proxy. Please refer to the [hosting and security](/configuration#hosting-and-security) docs.
 
 Also read
 [this](https://channels.readthedocs.io/en/stable/deploying.html#nginx-supervisor-ubuntu),

docs/troubleshooting.md

@@ -32,7 +32,7 @@ If you find the OCR accuracy to be too low, and/or the document consumer
 warns that
 `OCR for XX failed, but we're going to stick with what we've got since FORGIVING_OCR is enabled`,
 then you might need to install the [Tesseract language
-files](http://packages.ubuntu.com/search?keywords=tesseract-ocr)
+files](https://packages.ubuntu.com/search?keywords=tesseract-ocr)
 marching your document's languages.
 
 As an example, if you are running Paperless-ngx from any Ubuntu or
@@ -125,13 +125,13 @@ using docker-compose, this is achieved by the following configuration
 change in the `docker-compose.yml` file:
 
 ```yaml
-gotenberg:
-  image: gotenberg/gotenberg:7.6
-  restart: unless-stopped
-  command:
-    - 'gotenberg'
-    - '--chromium-disable-routes=true'
-    - '--api-timeout=60'
+# The gotenberg chromium route is used to convert .eml files. We do not
+# want to allow external content like tracking pixels or even javascript.
+command:
+  - 'gotenberg'
+  - '--chromium-disable-javascript=true'
+  - '--chromium-allow-list=file:///tmp/.*'
+  - '--api-timeout=60'
 ```
 
 ## Permission denied errors in the consumption directory
@@ -145,7 +145,7 @@ The following error occured while consuming document.pdf: [Errno 13] Permission
 This happens when paperless does not have permission to delete files
 inside the consumption directory. Ensure that `USERMAP_UID` and
 `USERMAP_GID` are set to the user id and group id you use on the host
-operating system, if these are different from `1000`. See [Docker setup](setup#docker_hub).
+operating system, if these are different from `1000`. See [Docker setup](/setup#docker_hub).
 
 Also ensure that you are able to read and write to the consumption
 directory on the host.
@@ -222,7 +222,7 @@ This might have multiple reasons.
     SENDFILE=0
     ```
 
-    to your [docker-compose.env]{.title-ref} file.
+    to your `docker-compose.env` file.
 
 ## Error while reading metadata
 
@@ -326,9 +326,9 @@ unlock. This may have minor performance implications.
 ## gunicorn fails to start with "is not a valid port number"
 
 You are likely running using Kubernetes, which automatically creates an
-environment variable named [\${serviceName}\_PORT]{.title-ref}. This is
+environment variable named `${serviceName}_PORT`. This is
 the same environment variable which is used by Paperless to optionally
 change the port gunicorn listens on.
 
-To fix this, set [PAPERLESS_PORT]{.title-ref} again to your desired
-port, or the default of 8000.
+To fix this, set `PAPERLESS_PORT` again to your desired port, or the
+default of 8000.

docs/usage.md

@@ -1,9 +1,9 @@
 # Usage Overview
 
 Paperless is an application that manages your personal documents. With
-the help of a document scanner (see [the scanners wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Scanner-&-Software-Recommendations)), paperless transforms your wieldy physical document binders
-into a searchable archive and provides many utilities for finding and
-managing your documents.
+the help of a document scanner (see [the scanners wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Scanner-&-Software-Recommendations)),
+paperless transforms your unwieldy physical document binders into a searchable archive
+and provides many utilities for finding and managing your documents.
 
 ## Terms and definitions
 
@@ -37,7 +37,7 @@ Each document has a couple of fields that you can assign to them:
   date you signed a contract, or the date a letter was sent to you.
 - The _archive serial number_ (short: ASN) of a document is the
   identifier of the document in your physical document binders. See
-  [recommended workflow](#usage-reccomended_workflow) below.
+  [recommended workflow](#usage-recommended-workflow) below.
 - The _content_ of a document is the text that was OCR'ed from the
   document. This text is fed into the search engine and is used for
   matching tags, correspondents and document types.
@@ -74,8 +74,8 @@ following operations on your documents:
 ### The consumption directory
 
 The primary method of getting documents into your database is by putting
-them in the consumption directory. The consumer runs in an infinite
-loop, looking for new additions to this directory. When it finds them,
+them in the consumption directory. The consumer waits patiently, looking
+for new additions to this directory. When it finds them,
 the consumer goes about the process of parsing them with the OCR,
 indexing what it finds, and storing it in the media directory.
 
@@ -86,7 +86,7 @@ scanner to automatically push files to this directory, you'll need to
 setup some sort of service to accept the files from the scanner.
 Typically, you're looking at an FTP server like
 [Proftpd](http://www.proftpd.org/) or a Windows folder share with
-[Samba](http://www.samba.org/).
+[Samba](https://www.samba.org/).
 
 ### Web UI Upload
 
@@ -99,7 +99,7 @@ dragging-and-dropping files into your browser window.
 
 ### Mobile upload {#usage-mobile_upload}
 
-The mobile app over at <https://github.com/qcasey/paperless_share>
+The mobile app over at [https://github.com/qcasey/paperless_share](https://github.com/qcasey/paperless_share)
 allows Android users to share any documents with paperless. This can be
 combined with any of the mobile scanning apps out there, such as Office
 Lens.
@@ -108,6 +108,8 @@ Furthermore, there is the [Paperless
 App](https://github.com/bauerj/paperless_app) as well, which not only
 has document upload, but also document browsing and download features.
 
+Another option is [Paperless Mobile](https://github.com/astubenbord/paperless-mobile), an Android app that supports document upload, scanning, management of labels and more.
+
 ### IMAP (Email) {#usage-email}
 
 You can tell paperless-ngx to consume documents from your email
@@ -151,6 +153,8 @@ different means. These are as follows:
   will not consume mails already tagged. Not all mail servers support
   this feature!
 
+  - **Apple Mail support:** Apple Mail clients allow differently colored tags. For this to work use `apple:<color>` (e.g. _apple:green_) as a custom tag. Available colors are _red_, _orange_, _yellow_, _blue_, _green_, _violet_ and _grey_.
+
 !!! warning
 
     The mail consumer will perform these actions on all mails it has
@@ -191,11 +195,11 @@ different means. These are as follows:
     them further.
 
 Paperless is set up to check your mails every 10 minutes. This can be
-configured on the 'Scheduled tasks' page in the admin.
+configured via `PAPERLESS_EMAIL_TASK_CRON` (see [software tweaks](/configuration#software_tweaks))
 
 ### REST API
 
-You can also submit a document using the REST API, see [docs][api#file-uploads]
+You can also submit a document using the REST API, see [POSTing documents](/api#file-uploads)
 for details.
 
 ## Best practices {#basic-searching}
@@ -325,7 +329,7 @@ language](https://whoosh.readthedocs.io/en/latest/querylang.html). For
 details on what date parsing utilities are available, see [Date
 parsing](https://whoosh.readthedocs.io/en/latest/dates.html#parsing-date-queries).
 
-## The recommended workflow {#usage-recommended_workflow}
+## The recommended workflow {#usage-recommended-workflow}
 
 Once you have familiarized yourself with paperless and are ready to use
 it for all your documents, the recommended workflow for managing your
@@ -359,6 +363,14 @@ documents in your inbox:
     sorted by ASN. Don't order this binder in any other way.
 5.  If the document has no ASN, throw it away. Yay!
 
+!!! tip
+
+    Instead of writing a number on the document by hand, you may also prepare
+    a spool of labels with barcodes with an ascending serial number, that are
+    formatted like `ASN00001`.
+    This also enables Paperless to automatically parse and process the ASN
+    (if enabled in the config), so that you don't need to manually assign it.
+
 Over time, you will notice that your physical binder will fill up. If it
 is full, label the binder with the range of ASNs in this binder (i.e.,
 "Documents 1 to 343"), store the binder in your cellar or elsewhere,
@@ -398,7 +410,7 @@ Once you have scanned in a document, proceed in paperless as follows.
     paperless will assign them automatically. After consuming a couple
     documents, you can even ask paperless to *learn* when to assign tags and
     correspondents by itself. For details on this feature, see
-    [advanced matching](advanced_usage#matching).
+    [advanced matching](/advanced_usage#matching).
 
 ### Task management
 
@@ -416,7 +428,7 @@ how regularly you intend to scan documents and use paperless.
   performed the task associated with the document, move it to the
   inbox.
 
-## Architectue
+## Architecture
 
 Paperless-ngx consists of the following components:
 

install-paperless-ngx.sh

@@ -321,7 +321,7 @@ fi
 wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/docker-compose.$DOCKER_COMPOSE_VERSION.yml" -O docker-compose.yml
 wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/.env" -O .env
 
-SECRET_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1)
+SECRET_KEY=$(tr --delete --complement 'a-zA-Z0-9' < /dev/urandom 2>/dev/null | head --bytes 64)
 
 DEFAULT_LANGUAGES=("deu eng fra ita spa")
 
@@ -346,7 +346,7 @@ read -r -a OCR_LANGUAGES_ARRAY <<< "${_split_langs}"
 	fi
 } > docker-compose.env
 
-sed -i "s/- 8000:8000/- $PORT:8000/g" docker-compose.yml
+sed -i "s/- \"8000:8000\"/- \"$PORT:8000\"/g" docker-compose.yml
 
 sed -i "s#- \./consume:/usr/src/paperless/consume#- $CONSUME_FOLDER:/usr/src/paperless/consume#g" docker-compose.yml
 

mkdocs.yml

@@ -23,6 +23,7 @@ theme:
     - navigation.tabs
     - navigation.top
     - toc.integrate
+    - content.code.annotate
   icon:
     repo: fontawesome/brands/github
   favicon: assets/favicon.png
@@ -39,6 +40,9 @@ markdown_extensions:
   - pymdownx.highlight:
       anchor_linenums: true
   - pymdownx.superfences
+  - pymdownx.inlinehilite
+  - pymdownx.snippets
+strict: true
 nav:
     - index.md
     - setup.md
@@ -51,7 +55,7 @@ nav:
     - 'FAQs': faq.md
     - troubleshooting.md
     - changelog.md
-copyright: Copyright © 2016 - 2022 Daniel Quinn, Jonas Winkler, and the Paperless-ngx team
+copyright: Copyright © 2016 - 2023 Daniel Quinn, Jonas Winkler, and the Paperless-ngx team
 extra:
   social:
     - icon: fontawesome/brands/github

scripts/start_services.sh

@@ -2,5 +2,5 @@
 
 docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
-docker run -p 3000:3000 -d gotenberg/gotenberg:7.6
+docker run -p 3000:3000 -d gotenberg/gotenberg:7.8 gotenberg --chromium-disable-javascript=true --chromium-allow-list="file:///tmp/.*"
 docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/.browserslistrc

@@ -1,18 +0,0 @@
-# This file is used by the build system to adjust CSS and JS output to support the specified browsers below.
-# For additional information regarding the format and rule options, please see:
-# https://github.com/browserslist/browserslist#queries
-
-# For the full list of supported browsers by the Angular framework, please see:
-# https://angular.io/guide/browser-support
-
-# You can see what browsers were selected by your queries by running:
-#   npx browserslist
-
-last 1 Chrome version
-last 1 Firefox version
-last 2 Edge major versions
-last 2 Safari major versions
-last 2 iOS major versions
-Firefox ESR
-not IE 9-10 # Angular support for IE 9-10 has been deprecated and will be removed as of Angular v11. To opt-in, remove the 'not' prefix on this line.
-not IE 11 # Angular supports IE 11 only as an opt-in. To opt-in, remove the 'not' prefix on this line.

src-ui/.eslintrc.json

@@ -0,0 +1,51 @@
+{
+  "root": true,
+  "ignorePatterns": [
+    "projects/**/*"
+  ],
+  "overrides": [
+    {
+      "files": [
+        "*.ts"
+      ],
+      "parserOptions": {
+        "project": [
+          "tsconfig.json",
+          "e2e/tsconfig.json"
+        ],
+        "createDefaultProgram": true
+      },
+      "extends": [
+        "plugin:@angular-eslint/recommended",
+        "plugin:@angular-eslint/template/process-inline-templates"
+      ],
+      "rules": {
+        "@angular-eslint/directive-selector": [
+          "error",
+          {
+            "type": "attribute",
+            "prefix": "app",
+            "style": "camelCase"
+          }
+        ],
+        "@angular-eslint/component-selector": [
+          "error",
+          {
+            "type": "element",
+            "prefix": "app",
+            "style": "kebab-case"
+          }
+        ]
+      }
+    },
+    {
+      "files": [
+        "*.html"
+      ],
+      "extends": [
+        "plugin:@angular-eslint/template/recommended"
+      ],
+      "rules": {}
+    }
+  ]
+}

src-ui/README.md

@@ -12,7 +12,7 @@ Run `ng generate component component-name` to generate a new component. You can
 
 ## Build
 
-Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `--prod` flag for a production build.
+Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `--configuration production` flag for a production build.
 
 ## Running unit tests
 

src-ui/angular.json

@@ -1,179 +1,206 @@
 {
-	"$schema": "./node_modules/@angular/cli/lib/config/schema.json",
-	"version": 1,
-	"newProjectRoot": "projects",
-	"projects": {
-		"paperless-ui": {
-			"projectType": "application",
-			"schematics": {
-				"@schematics/angular:component": {
-					"style": "scss"
-				}
-			},
-			"root": "",
-			"sourceRoot": "src",
-			"prefix": "app",
-			"i18n": {
-				"sourceLocale": "en-US",
-				"locales": {
-					"be-BY": "src/locale/messages.be_BY.xlf",
-					"cs-CZ": "src/locale/messages.cs_CZ.xlf",
-					"da-DK": "src/locale/messages.da_DK.xlf",
-					"de-DE": "src/locale/messages.de_DE.xlf",
-					"en-GB": "src/locale/messages.en_GB.xlf",
-					"es-ES": "src/locale/messages.es_ES.xlf",
-					"fr-FR": "src/locale/messages.fr_FR.xlf",
-					"it-IT": "src/locale/messages.it_IT.xlf",
-					"lb-LU": "src/locale/messages.lb_LU.xlf",
-					"nl-NL": "src/locale/messages.nl_NL.xlf",
-					"pl-PL": "src/locale/messages.pl_PL.xlf",
-					"pt-BR": "src/locale/messages.pt_BR.xlf",
-					"pt-PT": "src/locale/messages.pt_PT.xlf",
-					"ro-RO": "src/locale/messages.ro_RO.xlf",
-					"ru-RU": "src/locale/messages.ru_RU.xlf",
-					"sl-SI": "src/locale/messages.sl_SI.xlf",
-					"sr-CS": "src/locale/messages.sr_CS.xlf",
-					"sv-SE": "src/locale/messages.sv_SE.xlf",
-					"tr-TR": "src/locale/messages.tr_TR.xlf",
-					"zh-CN": "src/locale/messages.zh_CN.xlf"
-				}
-			},
-			"architect": {
-				"build": {
-					"builder": "@angular-devkit/build-angular:browser",
-					"options": {
-						"outputPath": "dist/paperless-ui",
-						"outputHashing": "none",
-						"index": "src/index.html",
-						"main": "src/main.ts",
-						"polyfills": "src/polyfills.ts",
-						"tsConfig": "tsconfig.app.json",
-						"localize": true,
-						"assets": [
-							"src/favicon.ico",
-							"src/apple-touch-icon.png",
-							"src/assets",
-							"src/manifest.webmanifest", {
-								"glob": "pdf.worker.min.js",
-								"input": "node_modules/pdfjs-dist/build/",
-								"output": "/assets/js/"
-							}
-						],
-						"styles": [
-							"src/styles.scss"
-						],
-						"scripts": [],
-						"allowedCommonJsDependencies": [
-							"ng2-pdf-viewer"
-						],
-						"vendorChunk": true,
-						"extractLicenses": false,
-						"buildOptimizer": false,
-						"sourceMap": true,
-						"optimization": false,
-						"namedChunks": true
-					},
-					"configurations": {
-						"production": {
-							"fileReplacements": [
-								{
-									"replace": "src/environments/environment.ts",
-									"with": "src/environments/environment.prod.ts"
-								}
-							],
-							"outputPath": "../src/documents/static/frontend/",
-							"optimization": true,
-							"outputHashing": "none",
-							"sourceMap": false,
-							"namedChunks": false,
-							"extractLicenses": true,
-							"vendorChunk": false,
-							"buildOptimizer": true,
-							"budgets": [
-								{
-									"type": "initial",
-									"maximumWarning": "2mb",
-									"maximumError": "5mb"
-								},
-								{
-									"type": "anyComponentStyle",
-									"maximumWarning": "6kb",
-									"maximumError": "10kb"
-								}
-							]
-						},
-						"en-US": {
-							"localize": ["en-US"]
-						}
-					},
-					"defaultConfiguration": ""
-				},
-				"serve": {
-					"builder": "@angular-devkit/build-angular:dev-server",
-					"options": {
-						"browserTarget": "paperless-ui:build:en-US"
-					},
-					"configurations": {
-						"production": {
-							"browserTarget": "paperless-ui:build:production"
-						}
-					}
-				},
-				"extract-i18n": {
-					"builder": "@angular-devkit/build-angular:extract-i18n",
-					"options": {
-						"browserTarget": "paperless-ui:build"
-					}
-				},
-				"test": {
-					"builder": "@angular-builders/jest:run",
-					"options": {
-						"tsConfig": "tsconfig.spec.json",
-						"assets": [
-							"src/favicon.ico",
-							"src/apple-touch-icon.png",
-							"src/assets",
-							"src/manifest.webmanifest"
-						],
-						"styles": [
-							"src/styles.scss"
-						],
-						"scripts": []
-					}
-				},
-				"e2e": {
-					"builder": "@cypress/schematic:cypress",
-					"options": {
-						"devServerTarget": "paperless-ui:serve",
-						"watch": true,
-						"headless": false
-					},
-					"configurations": {
-						"production": {
-							"devServerTarget": "paperless-ui:serve:production"
-						}
-					}
-				},
-				"cypress-run": {
-					"builder": "@cypress/schematic:cypress",
-					"options": {
-						"devServerTarget": "paperless-ui:serve"
-					},
-					"configurations": {
-						"production": {
-							"devServerTarget": "paperless-ui:serve:production"
-						}
-					}
-				},
-				"cypress-open": {
-					"builder": "@cypress/schematic:cypress",
-					"options": {
-						"watch": true,
-						"headless": false
-					}
-				}
-			}
-		}
-	},
-	"defaultProject": "paperless-ui"
+  "$schema": "./node_modules/@angular/cli/lib/config/schema.json",
+  "version": 1,
+  "newProjectRoot": "projects",
+  "projects": {
+    "paperless-ui": {
+      "projectType": "application",
+      "schematics": {
+        "@schematics/angular:component": {
+          "style": "scss"
+        }
+      },
+      "root": "",
+      "sourceRoot": "src",
+      "prefix": "app",
+      "i18n": {
+        "sourceLocale": "en-US",
+        "locales": {
+          "ar-AR": "src/locale/messages.ar_AR.xlf",
+          "be-BY": "src/locale/messages.be_BY.xlf",
+          "cs-CZ": "src/locale/messages.cs_CZ.xlf",
+          "da-DK": "src/locale/messages.da_DK.xlf",
+          "de-DE": "src/locale/messages.de_DE.xlf",
+          "en-GB": "src/locale/messages.en_GB.xlf",
+          "es-ES": "src/locale/messages.es_ES.xlf",
+          "fr-FR": "src/locale/messages.fr_FR.xlf",
+          "it-IT": "src/locale/messages.it_IT.xlf",
+          "lb-LU": "src/locale/messages.lb_LU.xlf",
+          "nl-NL": "src/locale/messages.nl_NL.xlf",
+          "pl-PL": "src/locale/messages.pl_PL.xlf",
+          "pt-BR": "src/locale/messages.pt_BR.xlf",
+          "pt-PT": "src/locale/messages.pt_PT.xlf",
+          "ro-RO": "src/locale/messages.ro_RO.xlf",
+          "ru-RU": "src/locale/messages.ru_RU.xlf",
+          "sl-SI": "src/locale/messages.sl_SI.xlf",
+          "sr-CS": "src/locale/messages.sr_CS.xlf",
+          "sv-SE": "src/locale/messages.sv_SE.xlf",
+          "tr-TR": "src/locale/messages.tr_TR.xlf",
+          "zh-CN": "src/locale/messages.zh_CN.xlf"
+        }
+      },
+      "architect": {
+        "build": {
+          "builder": "@angular-devkit/build-angular:browser",
+          "options": {
+            "outputPath": "dist/paperless-ui",
+            "outputHashing": "none",
+            "index": "src/index.html",
+            "main": "src/main.ts",
+            "polyfills": "src/polyfills.ts",
+            "tsConfig": "tsconfig.app.json",
+            "localize": true,
+            "assets": [
+              "src/favicon.ico",
+              "src/apple-touch-icon.png",
+              "src/assets",
+              "src/manifest.webmanifest",
+              {
+                "glob": "pdf.worker.min.js",
+                "input": "node_modules/pdfjs-dist/build/",
+                "output": "/assets/js/"
+              }
+            ],
+            "styles": [
+              "src/styles.scss"
+            ],
+            "scripts": [],
+            "allowedCommonJsDependencies": [
+              "ng2-pdf-viewer"
+            ],
+            "vendorChunk": true,
+            "extractLicenses": false,
+            "buildOptimizer": false,
+            "sourceMap": true,
+            "optimization": false,
+            "namedChunks": true
+          },
+          "configurations": {
+            "production": {
+              "fileReplacements": [
+                {
+                  "replace": "src/environments/environment.ts",
+                  "with": "src/environments/environment.prod.ts"
+                }
+              ],
+              "outputPath": "../src/documents/static/frontend/",
+              "optimization": true,
+              "outputHashing": "none",
+              "sourceMap": false,
+              "namedChunks": false,
+              "extractLicenses": true,
+              "vendorChunk": false,
+              "buildOptimizer": true,
+              "budgets": [
+                {
+                  "type": "initial",
+                  "maximumWarning": "2mb",
+                  "maximumError": "5mb"
+                },
+                {
+                  "type": "anyComponentStyle",
+                  "maximumWarning": "6kb",
+                  "maximumError": "10kb"
+                }
+              ]
+            },
+            "en-US": {
+              "localize": [
+                "en-US"
+              ]
+            }
+          },
+          "defaultConfiguration": ""
+        },
+        "serve": {
+          "builder": "@angular-devkit/build-angular:dev-server",
+          "options": {
+            "browserTarget": "paperless-ui:build:en-US"
+          },
+          "configurations": {
+            "production": {
+              "browserTarget": "paperless-ui:build:production"
+            }
+          }
+        },
+        "extract-i18n": {
+          "builder": "@angular-devkit/build-angular:extract-i18n",
+          "options": {
+            "browserTarget": "paperless-ui:build"
+          }
+        },
+        "test": {
+          "builder": "@angular-builders/jest:run",
+          "options": {
+            "tsConfig": "tsconfig.spec.json",
+            "assets": [
+              "src/favicon.ico",
+              "src/apple-touch-icon.png",
+              "src/assets",
+              "src/manifest.webmanifest"
+            ],
+            "styles": [
+              "src/styles.scss"
+            ],
+            "scripts": []
+          }
+        },
+        "e2e": {
+          "builder": "@cypress/schematic:cypress",
+          "options": {
+            "devServerTarget": "paperless-ui:serve",
+            "watch": true,
+            "headless": false
+          },
+          "configurations": {
+            "production": {
+              "devServerTarget": "paperless-ui:serve:production"
+            }
+          }
+        },
+        "cypress-run": {
+          "builder": "@cypress/schematic:cypress",
+          "options": {
+            "devServerTarget": "paperless-ui:serve"
+          },
+          "configurations": {
+            "production": {
+              "devServerTarget": "paperless-ui:serve:production"
+            }
+          }
+        },
+        "cypress-open": {
+          "builder": "@cypress/schematic:cypress",
+          "options": {
+            "watch": true,
+            "headless": false
+          }
+        },
+        "lint": {
+          "builder": "@angular-eslint/builder:lint",
+          "options": {
+            "lintFilePatterns": [
+              "src/**/*.ts",
+              "src/**/*.html"
+            ]
+          }
+        }
+      }
+    }
+  },
+  "defaultProject": "paperless-ui",
+  "cli": {
+    "schematicCollections": [
+      "@angular-eslint/schematics"
+    ],
+    "analytics": false
+  },
+  "schematics": {
+    "@angular-eslint/schematics:application": {
+      "setParserOptionsProject": true
+    },
+    "@angular-eslint/schematics:library": {
+      "setParserOptionsProject": true
+    }
+  }
 }

src-ui/cypress/e2e/documents/document-detail.cy.ts

@@ -44,7 +44,7 @@ describe('document-detail', () => {
     })
 
     cy.viewport(1024, 1024)
-    cy.visit('/documents/1/')
+    cy.visit('/documents/1/').wait('@ui-settings')
   })
 
   it('should activate / deactivate save button when changes are saved', () => {
@@ -66,8 +66,21 @@ describe('document-detail', () => {
     cy.contains('You have unsaved changes').should('not.exist')
   })
 
+  it('should show a mobile preview', () => {
+    cy.viewport(440, 1000)
+    cy.get('a')
+      .contains('Preview')
+      .scrollIntoView({ offset: { top: 150, left: 0 } })
+      .click()
+    cy.get('pdf-viewer').should('be.visible')
+  })
+
   it('should show a list of comments', () => {
-    cy.wait(1000).get('a').contains('Comments').click().wait(1000)
+    cy.wait(1000)
+      .get('a')
+      .contains('Comments')
+      .click({ force: true })
+      .wait(1000)
     cy.get('app-document-comments').find('.card').its('length').should('eq', 3)
   })
 

src-ui/cypress/e2e/documents/documents-list.cy.ts

@@ -52,6 +52,10 @@ describe('documents-list', () => {
 
         req.reply(response)
       })
+
+      cy.intercept('http://localhost:8000/api/documents/selection_data/', {
+        fixture: 'documents/selection_data.json',
+      }).as('selection-data')
     })
 
     cy.viewport(1280, 1024)
@@ -76,6 +80,28 @@ describe('documents-list', () => {
     cy.get('app-document-card-large')
   })
 
+  it('should show partial tag selection', () => {
+    cy.get('app-document-card-small:nth-child(1)').click()
+    cy.get('app-document-card-small:nth-child(4)').click()
+    cy.get('app-bulk-editor button')
+      .contains('Tags')
+      .click()
+      .wait('@selection-data')
+    cy.get('svg.bi-dash').should('be.visible')
+    cy.get('svg.bi-check').should('be.visible')
+  })
+
+  it('should allow bulk removal', () => {
+    cy.get('app-document-card-small:nth-child(1)').click()
+    cy.get('app-document-card-small:nth-child(4)').click()
+    cy.get('app-bulk-editor').within(() => {
+      cy.get('button').contains('Tags').click().wait('@selection-data')
+      cy.get('button').contains('Another Sample Tag').click()
+      cy.get('button').contains('Apply').click()
+    })
+    cy.contains('operation will remove the tag')
+  })
+
   it('should filter tags', () => {
     cy.get('app-filter-editor app-filterable-dropdown[title="Tags"]').within(
       () => {

src-ui/cypress/e2e/settings/settings.cy.ts

@@ -37,6 +37,58 @@ describe('settings', () => {
         ).as('savedViews')
       })
 
+      this.newMailAccounts = []
+
+      cy.intercept(
+        'POST',
+        'http://localhost:8000/api/mail_accounts/',
+        (req) => {
+          const newRule = req.body
+          newRule.id = 3
+          this.newMailAccounts.push(newRule) // store this for later
+          req.reply({ result: 'OK' })
+        }
+      ).as('saveAccount')
+
+      cy.fixture('mail_accounts/mail_accounts.json').then(
+        (mailAccountsJson) => {
+          cy.intercept(
+            'GET',
+            'http://localhost:8000/api/mail_accounts/*',
+            (req) => {
+              console.log(req, this.newMailAccounts)
+
+              let response = { ...mailAccountsJson }
+              if (this.newMailAccounts.length) {
+                response.results = response.results.concat(this.newMailAccounts)
+              }
+
+              req.reply(response)
+            }
+          ).as('getAccounts')
+        }
+      )
+
+      this.newMailRules = []
+
+      cy.intercept('POST', 'http://localhost:8000/api/mail_rules/', (req) => {
+        const newRule = req.body
+        newRule.id = 2
+        this.newMailRules.push(newRule) // store this for later
+        req.reply({ result: 'OK' })
+      }).as('saveRule')
+
+      cy.fixture('mail_rules/mail_rules.json').then((mailRulesJson) => {
+        cy.intercept('GET', 'http://localhost:8000/api/mail_rules/*', (req) => {
+          let response = { ...mailRulesJson }
+          if (this.newMailRules.length) {
+            response.results = response.results.concat(this.newMailRules)
+          }
+
+          req.reply(response)
+        }).as('getRules')
+      })
+
       cy.fixture('documents/documents.json').then((documentsJson) => {
         cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
           let response = { ...documentsJson }
@@ -48,7 +100,6 @@ describe('settings', () => {
 
     cy.viewport(1024, 1600)
     cy.visit('/settings')
-    cy.wait('@savedViews')
   })
 
   it('should activate / deactivate save button when settings change and are saved', () => {
@@ -64,7 +115,7 @@ describe('settings', () => {
     cy.contains('a', 'Dashboard').click()
     cy.contains('You have unsaved changes')
     cy.contains('button', 'Cancel').click()
-    cy.contains('button', 'Save').click().wait('@savedViews')
+    cy.contains('button', 'Save').click().wait('@savedViews').wait(2000)
     cy.contains('a', 'Dashboard').click()
     cy.contains('You have unsaved changes').should('not.exist')
   })
@@ -77,17 +128,55 @@ describe('settings', () => {
   })
 
   it('should remove saved view from sidebar when unset', () => {
-    cy.contains('a', 'Saved views').click()
+    cy.contains('a', 'Saved views').click().wait(2000)
     cy.get('#show_in_sidebar_1').click()
-    cy.contains('button', 'Save').click().wait('@savedViews')
+    cy.contains('button', 'Save').click().wait('@savedViews').wait(2000)
     cy.contains('li', 'Inbox').should('not.exist')
   })
 
   it('should remove saved view from dashboard when unset', () => {
     cy.contains('a', 'Saved views').click()
     cy.get('#show_on_dashboard_1').click()
-    cy.contains('button', 'Save').click().wait('@savedViews')
+    cy.contains('button', 'Save').click().wait('@savedViews').wait(2000)
     cy.visit('/dashboard')
     cy.get('app-saved-view-widget').contains('Inbox').should('not.exist')
   })
+
+  it('should show a list of mail accounts & rules & support creation', () => {
+    cy.contains('a', 'Mail').click()
+    cy.get('app-settings .tab-content ul li').its('length').should('eq', 5) // 2 headers, 2 accounts, 1 rule
+    cy.contains('button', 'Add Account').click()
+    cy.contains('Create new mail account')
+    cy.get('app-input-text[formcontrolname="name"]').type(
+      'Example Mail Account'
+    )
+    cy.get('app-input-text[formcontrolname="imap_server"]').type(
+      'mail.example.com'
+    )
+    cy.get('app-input-text[formcontrolname="imap_port"]').type('993')
+    cy.get('app-input-text[formcontrolname="username"]').type('username')
+    cy.get('app-input-password[formcontrolname="password"]').type('pass')
+    cy.contains('app-mail-account-edit-dialog button', 'Save')
+      .click()
+      .wait('@saveAccount')
+      .wait('@getAccounts')
+    cy.contains('Saved account')
+
+    cy.wait(1000)
+    cy.contains('button', 'Add Rule').click()
+    cy.contains('Create new mail rule')
+    cy.get('app-input-text[formcontrolname="name"]').type('Example Rule')
+    cy.get('app-input-select[formcontrolname="account"]').type('Example{enter}')
+    cy.get('app-input-number[formcontrolname="maximum_age"]').type('30')
+    cy.get('app-input-text[formcontrolname="filter_subject"]').type(
+      '[paperless]'
+    )
+    cy.contains('app-mail-rule-edit-dialog button', 'Save')
+      .click()
+      .wait('@saveRule')
+      .wait('@getRules')
+    cy.contains('Saved rule').wait(1000)
+
+    cy.get('app-settings .tab-content ul li').its('length').should('eq', 7)
+  })
 })

src-ui/cypress/fixtures/documents/selection_data.json

@@ -0,0 +1,293 @@
+{
+    "selected_correspondents": [
+        {
+            "id": 62,
+            "document_count": 0
+        },
+        {
+            "id": 75,
+            "document_count": 0
+        },
+        {
+            "id": 55,
+            "document_count": 0
+        },
+        {
+            "id": 56,
+            "document_count": 0
+        },
+        {
+            "id": 73,
+            "document_count": 0
+        },
+        {
+            "id": 58,
+            "document_count": 0
+        },
+        {
+            "id": 44,
+            "document_count": 0
+        },
+        {
+            "id": 42,
+            "document_count": 0
+        },
+        {
+            "id": 74,
+            "document_count": 0
+        },
+        {
+            "id": 54,
+            "document_count": 0
+        },
+        {
+            "id": 29,
+            "document_count": 0
+        },
+        {
+            "id": 71,
+            "document_count": 0
+        },
+        {
+            "id": 68,
+            "document_count": 0
+        },
+        {
+            "id": 82,
+            "document_count": 0
+        },
+        {
+            "id": 34,
+            "document_count": 0
+        },
+        {
+            "id": 41,
+            "document_count": 0
+        },
+        {
+            "id": 51,
+            "document_count": 0
+        },
+        {
+            "id": 46,
+            "document_count": 0
+        },
+        {
+            "id": 40,
+            "document_count": 0
+        },
+        {
+            "id": 43,
+            "document_count": 0
+        },
+        {
+            "id": 80,
+            "document_count": 0
+        },
+        {
+            "id": 70,
+            "document_count": 0
+        },
+        {
+            "id": 52,
+            "document_count": 0
+        },
+        {
+            "id": 67,
+            "document_count": 0
+        },
+        {
+            "id": 53,
+            "document_count": 0
+        },
+        {
+            "id": 32,
+            "document_count": 0
+        },
+        {
+            "id": 63,
+            "document_count": 0
+        },
+        {
+            "id": 35,
+            "document_count": 0
+        },
+        {
+            "id": 45,
+            "document_count": 0
+        },
+        {
+            "id": 38,
+            "document_count": 0
+        },
+        {
+            "id": 79,
+            "document_count": 0
+        },
+        {
+            "id": 48,
+            "document_count": 0
+        },
+        {
+            "id": 72,
+            "document_count": 0
+        },
+        {
+            "id": 78,
+            "document_count": 0
+        },
+        {
+            "id": 39,
+            "document_count": 0
+        },
+        {
+            "id": 57,
+            "document_count": 0
+        },
+        {
+            "id": 61,
+            "document_count": 0
+        },
+        {
+            "id": 81,
+            "document_count": 0
+        },
+        {
+            "id": 77,
+            "document_count": 0
+        },
+        {
+            "id": 69,
+            "document_count": 0
+        },
+        {
+            "id": 36,
+            "document_count": 3
+        },
+        {
+            "id": 31,
+            "document_count": 0
+        },
+        {
+            "id": 30,
+            "document_count": 0
+        },
+        {
+            "id": 50,
+            "document_count": 0
+        },
+        {
+            "id": 49,
+            "document_count": 0
+        },
+        {
+            "id": 60,
+            "document_count": 0
+        },
+        {
+            "id": 47,
+            "document_count": 0
+        },
+        {
+            "id": 66,
+            "document_count": 0
+        },
+        {
+            "id": 37,
+            "document_count": 0
+        },
+        {
+            "id": 28,
+            "document_count": 0
+        },
+        {
+            "id": 59,
+            "document_count": 0
+        },
+        {
+            "id": 33,
+            "document_count": 0
+        },
+        {
+            "id": 76,
+            "document_count": 0
+        }
+    ],
+    "selected_tags": [
+        {
+            "id": 4,
+            "document_count": 2
+        },
+        {
+            "id": 7,
+            "document_count": 0
+        },
+        {
+            "id": 5,
+            "document_count": 1
+        },
+        {
+            "id": 6,
+            "document_count": 0
+        },
+        {
+            "id": 3,
+            "document_count": 0
+        },
+        {
+            "id": 2,
+            "document_count": 1
+        },
+        {
+            "id": 1,
+            "document_count": 0
+        },
+        {
+            "id": 8,
+            "document_count": 0
+        }
+    ],
+    "selected_document_types": [
+        {
+            "id": 4,
+            "document_count": 0
+        },
+        {
+            "id": 10,
+            "document_count": 0
+        },
+        {
+            "id": 2,
+            "document_count": 0
+        },
+        {
+            "id": 11,
+            "document_count": 0
+        },
+        {
+            "id": 9,
+            "document_count": 0
+        },
+        {
+            "id": 7,
+            "document_count": 2
+        },
+        {
+            "id": 3,
+            "document_count": 0
+        },
+        {
+            "id": 1,
+            "document_count": 0
+        },
+        {
+            "id": 5,
+            "document_count": 0
+        },
+        {
+            "id": 8,
+            "document_count": 1
+        }
+    ],
+    "selected_storage_paths": []
+}

src-ui/cypress/fixtures/mail_accounts/mail_accounts.json

@@ -0,0 +1,27 @@
+{
+    "count": 2,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 1,
+            "name": "IMAP Server",
+            "imap_server": "imap.example.com",
+            "imap_port": 993,
+            "imap_security": 2,
+            "username": "inbox@example.com",
+            "password": "pass",
+            "character_set": "UTF-8"
+        },
+        {
+            "id": 2,
+            "name": "Gmail",
+            "imap_server": "imap.gmail.com",
+            "imap_port": 993,
+            "imap_security": 2,
+            "username": "user@gmail.com",
+            "password": "pass",
+            "character_set": "UTF-8"
+        }
+    ]
+}

src-ui/cypress/fixtures/mail_rules/mail_rules.json

@@ -0,0 +1,30 @@
+{
+    "count": 1,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 1,
+            "name": "Gmail",
+            "account": 2,
+            "folder": "INBOX",
+            "filter_from": null,
+            "filter_subject": "[paperless]",
+            "filter_body": null,
+            "filter_attachment_filename": null,
+            "maximum_age": 30,
+            "action": 3,
+            "action_parameter": null,
+            "assign_title_from": 1,
+            "assign_tags": [
+                9
+            ],
+            "assign_correspondent_from": 1,
+            "assign_correspondent": 2,
+            "assign_document_type": null,
+            "order": 0,
+            "attachment_type": 2,
+            "consumption_scope": 1
+        }
+    ]
+}

src-ui/cypress/fixtures/saved_views/savedviews.json

@@ -1 +1,44 @@
-{"count":3,"next":null,"previous":null,"results":[{"id":1,"name":"Inbox","show_on_dashboard":true,"show_in_sidebar":true,"sort_field":"created","sort_reverse":true,"filter_rules":[{"rule_type":6,"value":"18"}]},{"id":2,"name":"Recently Added","show_on_dashboard":true,"show_in_sidebar":false,"sort_field":"created","sort_reverse":true,"filter_rules":[]},{"id":11,"name":"Taxes","show_on_dashboard":false,"show_in_sidebar":true,"sort_field":"created","sort_reverse":true,"filter_rules":[{"rule_type":6,"value":"39"}]}]}
+{
+    "count": 3,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 1,
+            "name": "Inbox",
+            "show_on_dashboard": true,
+            "show_in_sidebar": true,
+            "sort_field": "created",
+            "sort_reverse": true,
+            "filter_rules": [
+                {
+                    "rule_type": 6,
+                    "value": "18"
+                }
+            ]
+        },
+        {
+            "id": 2,
+            "name": "Recently Added",
+            "show_on_dashboard": true,
+            "show_in_sidebar": false,
+            "sort_field": "created",
+            "sort_reverse": true,
+            "filter_rules": []
+        },
+        {
+            "id": 11,
+            "name": "Taxes",
+            "show_on_dashboard": false,
+            "show_in_sidebar": true,
+            "sort_field": "created",
+            "sort_reverse": true,
+            "filter_rules": [
+                {
+                    "rule_type": 6,
+                    "value": "39"
+                }
+            ]
+        }
+    ]
+}

src-ui/cypress/support/e2e.ts

@@ -3,7 +3,7 @@
 beforeEach(() => {
   cy.intercept('http://localhost:8000/api/ui_settings/', {
     fixture: 'ui_settings/settings.json',
-  })
+  }).as('ui-settings')
 
   cy.intercept('http://localhost:8000/api/remote_version/', {
     fixture: 'remote_version/remote_version.json',
@@ -29,6 +29,10 @@ beforeEach(() => {
     fixture: 'storage_paths/storage_paths.json',
   })
 
+  cy.intercept('http://localhost:8000/api/tasks/', {
+    fixture: 'tasks/tasks.json',
+  })
+
   cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
     fixture: 'documents/1/metadata.json',
   })

src-ui/package.json

@@ -13,44 +13,50 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/common": "~14.2.8",
-    "@angular/compiler": "~14.2.8",
-    "@angular/core": "~14.2.8",
-    "@angular/forms": "~14.2.8",
-    "@angular/localize": "~14.2.8",
-    "@angular/platform-browser": "~14.2.8",
-    "@angular/platform-browser-dynamic": "~14.2.8",
-    "@angular/router": "~14.2.8",
-    "@ng-bootstrap/ng-bootstrap": "^13.0.0",
-    "@ng-select/ng-select": "^9.0.2",
+    "@angular/common": "~15.1.2",
+    "@angular/compiler": "~15.1.2",
+    "@angular/core": "~15.1.2",
+    "@angular/forms": "~15.1.2",
+    "@angular/localize": "~15.1.2",
+    "@angular/platform-browser": "~15.1.2",
+    "@angular/platform-browser-dynamic": "~15.1.2",
+    "@angular/router": "~15.1.2",
+    "@ng-bootstrap/ng-bootstrap": "^14.0.1",
+    "@ng-select/ng-select": "^10.0.1",
     "@ngneat/dirty-check-forms": "^3.0.3",
     "@popperjs/core": "^2.11.6",
-    "bootstrap": "^5.2.1",
+    "bootstrap": "^5.2.3",
     "file-saver": "^2.0.5",
     "ng2-pdf-viewer": "^9.1.2",
     "ngx-color": "^8.0.3",
-    "ngx-cookie-service": "^14.0.1",
+    "ngx-cookie-service": "^15.0.0",
     "ngx-file-drop": "^14.0.2",
-    "ngx-ui-tour-ng-bootstrap": "^11.1.0",
-    "rxjs": "~7.5.7",
+    "ngx-ui-tour-ng-bootstrap": "^12.0.0",
+    "rxjs": "^7.8.0",
     "tslib": "^2.4.1",
     "uuid": "^9.0.0",
     "zone.js": "~0.11.8"
   },
   "devDependencies": {
-    "@angular-builders/jest": "14.1.0",
-    "@angular-devkit/build-angular": "~14.2.7",
-    "@angular/cli": "~14.2.7",
-    "@angular/compiler-cli": "~14.2.8",
+    "@angular-builders/jest": "15.0.0",
+    "@angular-devkit/build-angular": "~15.1.4",
+    "@angular-eslint/builder": "15.2.0",
+    "@angular-eslint/eslint-plugin": "15.2.0",
+    "@angular-eslint/eslint-plugin-template": "15.2.0",
+    "@angular-eslint/schematics": "15.2.0",
+    "@angular-eslint/template-parser": "15.2.0",
+    "@angular/cli": "~15.1.4",
+    "@angular/compiler-cli": "~15.1.2",
     "@types/jest": "28.1.6",
     "@types/node": "^18.7.23",
-    "codelyzer": "^6.0.2",
+    "@typescript-eslint/eslint-plugin": "^5.43.0",
+    "@typescript-eslint/parser": "^5.50.0",
     "concurrently": "7.4.0",
+    "eslint": "^8.31.0",
     "jest": "28.1.3",
     "jest-environment-jsdom": "^29.2.2",
     "jest-preset-angular": "^12.2.3",
     "ts-node": "~10.9.1",
-    "tslint": "~6.1.3",
     "typescript": "~4.8.4",
     "wait-on": "~6.0.1"
   },

src-ui/src/app/app-routing.module.ts

@@ -47,6 +47,11 @@ const routes: Routes = [
         component: SettingsComponent,
         canDeactivate: [DirtyFormGuard],
       },
+      {
+        path: 'settings/:section',
+        component: SettingsComponent,
+        canDeactivate: [DirtyFormGuard],
+      },
       { path: 'tasks', component: TasksComponent },
     ],
   },
@@ -56,7 +61,7 @@ const routes: Routes = [
 ]
 
 @NgModule({
-  imports: [RouterModule.forRoot(routes, { relativeLinkResolution: 'legacy' })],
+  imports: [RouterModule.forRoot(routes)],
   exports: [RouterModule],
 })
 export class AppRoutingModule {}

src-ui/src/app/app.component.ts

@@ -191,21 +191,13 @@ export class AppComponent implements OnInit, OnDestroy {
       },
       {
         anchorId: 'tour.settings',
-        content: $localize`Check out the settings for various tweaks to the web app or to toggle settings for saved views.`,
+        content: $localize`Check out the settings for various tweaks to the web app, toggle settings for saved views or setup e-mail checking.`,
         route: '/settings',
         enableBackdrop: true,
         prevBtnTitle,
         nextBtnTitle,
         endBtnTitle,
       },
-      {
-        anchorId: 'tour.admin',
-        content: $localize`The Admin area contains more advanced controls as well as the settings for automatic e-mail fetching.`,
-        enableBackdrop: true,
-        prevBtnTitle,
-        nextBtnTitle,
-        endBtnTitle,
-      },
       {
         anchorId: 'tour.outro',
         title: $localize`Thank you! 🙏`,

src-ui/src/app/app.module.ts

@@ -24,7 +24,7 @@ import { CorrespondentEditDialogComponent } from './components/common/edit-dialo
 import { TagEditDialogComponent } from './components/common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
 import { DocumentTypeEditDialogComponent } from './components/common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
 import { TagComponent } from './components/common/tag/tag.component'
-import { ClearableBadge } from './components/common/clearable-badge/clearable-badge.component'
+import { ClearableBadgeComponent } from './components/common/clearable-badge/clearable-badge.component'
 import { PageHeaderComponent } from './components/common/page-header/page-header.component'
 import { AppFrameComponent } from './components/app-frame/app-frame.component'
 import { ToastsComponent } from './components/common/toasts/toasts.component'
@@ -39,6 +39,7 @@ import { NgxFileDropModule } from 'ngx-file-drop'
 import { TextComponent } from './components/common/input/text/text.component'
 import { SelectComponent } from './components/common/input/select/select.component'
 import { CheckComponent } from './components/common/input/check/check.component'
+import { PasswordComponent } from './components/common/input/password/password.component'
 import { SaveViewConfigDialogComponent } from './components/document-list/save-view-config-dialog/save-view-config-dialog.component'
 import { TagsComponent } from './components/common/input/tags/tags.component'
 import { SortableDirective } from './directives/sortable.directive'
@@ -76,7 +77,10 @@ import { StoragePathEditDialogComponent } from './components/common/edit-dialog/
 import { SettingsService } from './services/settings.service'
 import { TasksComponent } from './components/manage/tasks/tasks.component'
 import { TourNgBootstrapModule } from 'ngx-ui-tour-ng-bootstrap'
+import { MailAccountEditDialogComponent } from './components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component'
+import { MailRuleEditDialogComponent } from './components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component'
 
+import localeAr from '@angular/common/locales/ar'
 import localeBe from '@angular/common/locales/be'
 import localeCs from '@angular/common/locales/cs'
 import localeDa from '@angular/common/locales/da'
@@ -97,6 +101,7 @@ import localeSv from '@angular/common/locales/sv'
 import localeTr from '@angular/common/locales/tr'
 import localeZh from '@angular/common/locales/zh'
 
+registerLocaleData(localeAr)
 registerLocaleData(localeBe)
 registerLocaleData(localeCs)
 registerLocaleData(localeDa)
@@ -143,7 +148,7 @@ function initializeApp(settings: SettingsService) {
     DocumentTypeEditDialogComponent,
     StoragePathEditDialogComponent,
     TagComponent,
-    ClearableBadge,
+    ClearableBadgeComponent,
     PageHeaderComponent,
     AppFrameComponent,
     ToastsComponent,
@@ -157,6 +162,7 @@ function initializeApp(settings: SettingsService) {
     TextComponent,
     SelectComponent,
     CheckComponent,
+    PasswordComponent,
     SaveViewConfigDialogComponent,
     TagsComponent,
     SortableDirective,
@@ -180,6 +186,8 @@ function initializeApp(settings: SettingsService) {
     DocumentAsnComponent,
     DocumentCommentsComponent,
     TasksComponent,
+    MailAccountEditDialogComponent,
+    MailRuleEditDialogComponent,
   ],
   imports: [
     BrowserModule,
@@ -192,7 +200,7 @@ function initializeApp(settings: SettingsService) {
     PdfViewerModule,
     NgSelectModule,
     ColorSliderModule,
-    TourNgBootstrapModule.forRoot(),
+    TourNgBootstrapModule,
   ],
   providers: [
     {

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -174,13 +174,6 @@
               </svg><span>&nbsp;<ng-container i18n>Settings</ng-container></span>
             </a>
           </li>
-          <li class="nav-item" tourAnchor="tour.admin">
-            <a class="nav-link" href="admin/" ngbPopover="Admin" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
-              <svg class="sidebaricon" fill="currentColor">
-                <use xlink:href="assets/bootstrap-icons.svg#toggles"/>
-              </svg><span>&nbsp;<ng-container i18n>Admin</ng-container></span>
-            </a>
-          </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-auto pt-4 mb-1 text-muted">

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -220,6 +220,12 @@ main {
   font-size: 1rem;
 }
 
+@media screen and (min-width: 768px) {
+  .navbar-brand.slim {
+    max-width: 50px;
+  }
+}
+
 .dropdown.show .dropdown-toggle,
 .dropdown-toggle:hover {
   opacity: 0.7;

src-ui/src/app/components/common/clearable-badge/clearable-badge.component.ts

@@ -5,7 +5,7 @@ import { Component, Input, Output, EventEmitter } from '@angular/core'
   templateUrl: './clearable-badge.component.html',
   styleUrls: ['./clearable-badge.component.scss'],
 })
-export class ClearableBadge {
+export class ClearableBadgeComponent {
   constructor() {}
 
   @Input()

src-ui/src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.ts

@@ -5,7 +5,6 @@ import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-
 import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
 import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
-import { ToastService } from 'src/app/services/toast.service'
 
 @Component({
   selector: 'app-correspondent-edit-dialog',
@@ -13,12 +12,8 @@ import { ToastService } from 'src/app/services/toast.service'
   styleUrls: ['./correspondent-edit-dialog.component.scss'],
 })
 export class CorrespondentEditDialogComponent extends EditDialogComponent<PaperlessCorrespondent> {
-  constructor(
-    service: CorrespondentService,
-    activeModal: NgbActiveModal,
-    toastService: ToastService
-  ) {
-    super(service, activeModal, toastService)
+  constructor(service: CorrespondentService, activeModal: NgbActiveModal) {
+    super(service, activeModal)
   }
 
   getCreateTitle() {

src-ui/src/app/components/common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component.ts

@@ -5,7 +5,6 @@ import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-
 import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
 import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
-import { ToastService } from 'src/app/services/toast.service'
 
 @Component({
   selector: 'app-document-type-edit-dialog',
@@ -13,12 +12,8 @@ import { ToastService } from 'src/app/services/toast.service'
   styleUrls: ['./document-type-edit-dialog.component.scss'],
 })
 export class DocumentTypeEditDialogComponent extends EditDialogComponent<PaperlessDocumentType> {
-  constructor(
-    service: DocumentTypeService,
-    activeModal: NgbActiveModal,
-    toastService: ToastService
-  ) {
-    super(service, activeModal, toastService)
+  constructor(service: DocumentTypeService, activeModal: NgbActiveModal) {
+    super(service, activeModal)
   }
 
   getCreateTitle() {

src-ui/src/app/components/common/edit-dialog/edit-dialog.component.ts

@@ -2,11 +2,9 @@ import { Directive, EventEmitter, Input, OnInit, Output } from '@angular/core'
 import { FormGroup } from '@angular/forms'
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
 import { Observable } from 'rxjs'
-import { map } from 'rxjs/operators'
 import { MATCHING_ALGORITHMS, MATCH_AUTO } from 'src/app/data/matching-model'
 import { ObjectWithId } from 'src/app/data/object-with-id'
 import { AbstractPaperlessService } from 'src/app/services/rest/abstract-paperless-service'
-import { ToastService } from 'src/app/services/toast.service'
 
 @Directive()
 export abstract class EditDialogComponent<T extends ObjectWithId>
@@ -14,8 +12,7 @@ export abstract class EditDialogComponent<T extends ObjectWithId>
 {
   constructor(
     private service: AbstractPaperlessService<T>,
-    private activeModal: NgbActiveModal,
-    private toastService: ToastService
+    private activeModal: NgbActiveModal
   ) {}
 
   @Input()
@@ -25,7 +22,7 @@ export abstract class EditDialogComponent<T extends ObjectWithId>
   object: T
 
   @Output()
-  success = new EventEmitter()
+  succeeded = new EventEmitter()
 
   networkActive = false
 
@@ -95,16 +92,16 @@ export abstract class EditDialogComponent<T extends ObjectWithId>
         break
     }
     this.networkActive = true
-    serverResponse.subscribe(
-      (result) => {
+    serverResponse.subscribe({
+      next: (result) => {
         this.activeModal.close()
-        this.success.emit(result)
+        this.succeeded.emit(result)
       },
-      (error) => {
+      error: (error) => {
         this.error = error.error
         this.networkActive = false
-      }
-    )
+      },
+    })
   }
 
   cancel() {

src-ui/src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.html

@@ -0,0 +1,26 @@
+<form [formGroup]="objectForm" (ngSubmit)="save()">
+  <div class="modal-header">
+    <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
+    <button type="button" [disabled]="!closeEnabled" class="btn-close" aria-label="Close" (click)="cancel()">
+    </button>
+  </div>
+  <div class="modal-body">
+    <div class="row">
+      <div class="col">
+        <app-input-text i18n-title title="Name" formControlName="name" [error]="error?.name"></app-input-text>
+        <app-input-text i18n-title title="IMAP Server" formControlName="imap_server" [error]="error?.imap_server"></app-input-text>
+        <app-input-text i18n-title title="IMAP Port" formControlName="imap_port" [error]="error?.imap_port"></app-input-text>
+        <app-input-select i18n-title title="IMAP Security" [items]="imapSecurityOptions" formControlName="imap_security"></app-input-select>
+      </div>
+      <div class="col">
+        <app-input-text i18n-title title="Username" formControlName="username" [error]="error?.username"></app-input-text>
+        <app-input-password i18n-title title="Password" formControlName="password" [error]="error?.password"></app-input-password>
+        <app-input-text i18n-title title="Character Set" formControlName="character_set" [error]="error?.character_set"></app-input-text>
+      </div>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <button type="button" class="btn btn-outline-secondary" (click)="cancel()" i18n [disabled]="networkActive">Cancel</button>
+    <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive">Save</button>
+  </div>
+</form>

src-ui/src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.ts

@@ -0,0 +1,50 @@
+import { Component } from '@angular/core'
+import { FormControl, FormGroup } from '@angular/forms'
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
+import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component'
+import {
+  IMAPSecurity,
+  PaperlessMailAccount,
+} from 'src/app/data/paperless-mail-account'
+import { MailAccountService } from 'src/app/services/rest/mail-account.service'
+
+const IMAP_SECURITY_OPTIONS = [
+  { id: IMAPSecurity.None, name: $localize`No encryption` },
+  { id: IMAPSecurity.SSL, name: $localize`SSL` },
+  { id: IMAPSecurity.STARTTLS, name: $localize`STARTTLS` },
+]
+
+@Component({
+  selector: 'app-mail-account-edit-dialog',
+  templateUrl: './mail-account-edit-dialog.component.html',
+  styleUrls: ['./mail-account-edit-dialog.component.scss'],
+})
+export class MailAccountEditDialogComponent extends EditDialogComponent<PaperlessMailAccount> {
+  constructor(service: MailAccountService, activeModal: NgbActiveModal) {
+    super(service, activeModal)
+  }
+
+  getCreateTitle() {
+    return $localize`Create new mail account`
+  }
+
+  getEditTitle() {
+    return $localize`Edit mail account`
+  }
+
+  getForm(): FormGroup {
+    return new FormGroup({
+      name: new FormControl(null),
+      imap_server: new FormControl(null),
+      imap_port: new FormControl(null),
+      imap_security: new FormControl(IMAPSecurity.SSL),
+      username: new FormControl(null),
+      password: new FormControl(null),
+      character_set: new FormControl('UTF-8'),
+    })
+  }
+
+  get imapSecurityOptions() {
+    return IMAP_SECURITY_OPTIONS
+  }
+}

src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html

@@ -0,0 +1,41 @@
+<form [formGroup]="objectForm" (ngSubmit)="save()">
+  <div class="modal-header">
+    <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
+    <button type="button" [disabled]="!closeEnabled" class="btn-close" aria-label="Close" (click)="cancel()">
+    </button>
+  </div>
+  <div class="modal-body">
+    <div class="row">
+      <div class="col">
+        <app-input-text i18n-title title="Name" formControlName="name" [error]="error?.name"></app-input-text>
+        <app-input-select i18n-title title="Account" [items]="accounts" formControlName="account"></app-input-select>
+        <app-input-text i18n-title title="Folder" formControlName="folder" i18n-hint hint="Subfolders must be separated by a delimiter, often a dot ('.') or slash ('/'), but it varies by mail server." [error]="error?.folder"></app-input-text>
+        <app-input-number i18n-title title="Maximum age (days)" formControlName="maximum_age" [showAdd]="false" [error]="error?.maximum_age"></app-input-number>
+        <app-input-select i18n-title title="Attachment type" [items]="attachmentTypeOptions" formControlName="attachment_type"></app-input-select>
+        <app-input-select i18n-title title="Consumption scope" [items]="consumptionScopeOptions" formControlName="consumption_scope" i18n-hint hint="See docs for .eml processing requirements"></app-input-select>
+        <app-input-number i18n-title title="Rule order" formControlName="order" [showAdd]="false" [error]="error?.order"></app-input-number>
+      </div>
+      <div class="col">
+        <p class="small" i18n>Paperless will only process mails that match <em>all</em> of the filters specified below.</p>
+        <app-input-text i18n-title title="Filter from" formControlName="filter_from" [error]="error?.filter_from"></app-input-text>
+        <app-input-text i18n-title title="Filter subject" formControlName="filter_subject" [error]="error?.filter_subject"></app-input-text>
+        <app-input-text i18n-title title="Filter body" formControlName="filter_body" [error]="error?.filter_body"></app-input-text>
+        <app-input-text i18n-title title="Filter attachment filename" formControlName="filter_attachment_filename" i18n-hint hint="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_attachment_filename"></app-input-text>
+      </div>
+      <div class="col">
+        <app-input-select i18n-title title="Action" [items]="actionOptions" formControlName="action" i18n-hint hint="Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched."></app-input-select>
+        <app-input-text i18n-title title="Action parameter" *ngIf="showActionParamField" formControlName="action_parameter" [error]="error?.action_parameter"></app-input-text>
+        <app-input-select i18n-title title="Assign title from" [items]="metadataTitleOptions" formControlName="assign_title_from"></app-input-select>
+        <app-input-tags [allowCreate]="false" formControlName="assign_tags"></app-input-tags>
+        <app-input-select i18n-title title="Assign document type" [items]="documentTypes" [allowNull]="true" formControlName="assign_document_type"></app-input-select>
+        <app-input-select i18n-title title="Assign correspondent from" [items]="metadataCorrespondentOptions" formControlName="assign_correspondent_from"></app-input-select>
+        <app-input-select *ngIf="showCorrespondentField" i18n-title title="Assign correspondent" [items]="correspondents" [allowNull]="true" formControlName="assign_correspondent"></app-input-select>
+      </div>
+    </div>
+  </div>
+  <div class="modal-footer">
+    <span class="text-danger" *ngIf="error?.non_field_errors"><ng-container i18n>Error</ng-container>: {{error.non_field_errors}}</span>
+    <button type="button" class="btn btn-outline-secondary" (click)="cancel()" i18n [disabled]="networkActive">Cancel</button>
+    <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive">Save</button>
+  </div>
+</form>

src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.ts

@@ -0,0 +1,202 @@
+import { Component } from '@angular/core'
+import { FormControl, FormGroup } from '@angular/forms'
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
+import { first } from 'rxjs'
+import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component'
+import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
+import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
+import { PaperlessMailAccount } from 'src/app/data/paperless-mail-account'
+import {
+  MailAction,
+  MailFilterAttachmentType,
+  MailMetadataCorrespondentOption,
+  MailMetadataTitleOption,
+  PaperlessMailRule,
+  MailRuleConsumptionScope,
+} from 'src/app/data/paperless-mail-rule'
+import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
+import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
+import { MailAccountService } from 'src/app/services/rest/mail-account.service'
+import { MailRuleService } from 'src/app/services/rest/mail-rule.service'
+
+const ATTACHMENT_TYPE_OPTIONS = [
+  {
+    id: MailFilterAttachmentType.Attachments,
+    name: $localize`Only process attachments`,
+  },
+  {
+    id: MailFilterAttachmentType.Everything,
+    name: $localize`Process all files, including 'inline' attachments`,
+  },
+]
+
+const CONSUMPTION_SCOPE_OPTIONS = [
+  {
+    id: MailRuleConsumptionScope.Attachments,
+    name: $localize`Only process attachments`,
+  },
+  {
+    id: MailRuleConsumptionScope.Email_Only,
+    name: $localize`Process message as .eml`,
+  },
+  {
+    id: MailRuleConsumptionScope.Everything,
+    name: $localize`Process message as .eml and attachments separately`,
+  },
+]
+
+const ACTION_OPTIONS = [
+  {
+    id: MailAction.Delete,
+    name: $localize`Delete`,
+  },
+  {
+    id: MailAction.Move,
+    name: $localize`Move to specified folder`,
+  },
+  {
+    id: MailAction.MarkRead,
+    name: $localize`Mark as read, don't process read mails`,
+  },
+  {
+    id: MailAction.Flag,
+    name: $localize`Flag the mail, don't process flagged mails`,
+  },
+  {
+    id: MailAction.Tag,
+    name: $localize`Tag the mail with specified tag, don't process tagged mails`,
+  },
+]
+
+const METADATA_TITLE_OPTIONS = [
+  {
+    id: MailMetadataTitleOption.FromSubject,
+    name: $localize`Use subject as title`,
+  },
+  {
+    id: MailMetadataTitleOption.FromFilename,
+    name: $localize`Use attachment filename as title`,
+  },
+]
+
+const METADATA_CORRESPONDENT_OPTIONS = [
+  {
+    id: MailMetadataCorrespondentOption.FromNothing,
+    name: $localize`Do not assign a correspondent`,
+  },
+  {
+    id: MailMetadataCorrespondentOption.FromEmail,
+    name: $localize`Use mail address`,
+  },
+  {
+    id: MailMetadataCorrespondentOption.FromName,
+    name: $localize`Use name (or mail address if not available)`,
+  },
+  {
+    id: MailMetadataCorrespondentOption.FromCustom,
+    name: $localize`Use correspondent selected below`,
+  },
+]
+
+@Component({
+  selector: 'app-mail-rule-edit-dialog',
+  templateUrl: './mail-rule-edit-dialog.component.html',
+  styleUrls: ['./mail-rule-edit-dialog.component.scss'],
+})
+export class MailRuleEditDialogComponent extends EditDialogComponent<PaperlessMailRule> {
+  accounts: PaperlessMailAccount[]
+  correspondents: PaperlessCorrespondent[]
+  documentTypes: PaperlessDocumentType[]
+
+  constructor(
+    service: MailRuleService,
+    activeModal: NgbActiveModal,
+    accountService: MailAccountService,
+    correspondentService: CorrespondentService,
+    documentTypeService: DocumentTypeService
+  ) {
+    super(service, activeModal)
+
+    accountService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.accounts = result.results))
+
+    correspondentService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.correspondents = result.results))
+
+    documentTypeService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.documentTypes = result.results))
+  }
+
+  getCreateTitle() {
+    return $localize`Create new mail rule`
+  }
+
+  getEditTitle() {
+    return $localize`Edit mail rule`
+  }
+
+  getForm(): FormGroup {
+    return new FormGroup({
+      name: new FormControl(null),
+      account: new FormControl(null),
+      folder: new FormControl('INBOX'),
+      filter_from: new FormControl(null),
+      filter_subject: new FormControl(null),
+      filter_body: new FormControl(null),
+      filter_attachment_filename: new FormControl(null),
+      maximum_age: new FormControl(null),
+      attachment_type: new FormControl(MailFilterAttachmentType.Attachments),
+      consumption_scope: new FormControl(MailRuleConsumptionScope.Attachments),
+      order: new FormControl(null),
+      action: new FormControl(MailAction.MarkRead),
+      action_parameter: new FormControl(null),
+      assign_title_from: new FormControl(MailMetadataTitleOption.FromSubject),
+      assign_tags: new FormControl([]),
+      assign_document_type: new FormControl(null),
+      assign_correspondent_from: new FormControl(
+        MailMetadataCorrespondentOption.FromNothing
+      ),
+      assign_correspondent: new FormControl(null),
+    })
+  }
+
+  get showCorrespondentField(): boolean {
+    return (
+      this.objectForm?.get('assign_correspondent_from')?.value ==
+      MailMetadataCorrespondentOption.FromCustom
+    )
+  }
+
+  get showActionParamField(): boolean {
+    return (
+      this.objectForm?.get('action')?.value == MailAction.Move ||
+      this.objectForm?.get('action')?.value == MailAction.Tag
+    )
+  }
+
+  get attachmentTypeOptions() {
+    return ATTACHMENT_TYPE_OPTIONS
+  }
+
+  get actionOptions() {
+    return ACTION_OPTIONS
+  }
+
+  get metadataTitleOptions() {
+    return METADATA_TITLE_OPTIONS
+  }
+
+  get metadataCorrespondentOptions() {
+    return METADATA_CORRESPONDENT_OPTIONS
+  }
+
+  get consumptionScopeOptions() {
+    return CONSUMPTION_SCOPE_OPTIONS
+  }
+}

src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html

@@ -6,7 +6,7 @@
   </div>
   <div class="modal-body">
 
-    <p *ngIf="this.dialogMode == 'edit'" i18n>
+    <p *ngIf="this.dialogMode === 'edit'" i18n>
       <em>Note that editing a path does not apply changes to stored files until you have run the 'document_renamer' utility. See the <a target="_blank" href="https://docs.paperless-ngx.com/administration/#renamer">documentation</a>.</em>
     </p>
 

src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts

@@ -5,7 +5,6 @@ import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-
 import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
 import { StoragePathService } from 'src/app/services/rest/storage-path.service'
-import { ToastService } from 'src/app/services/toast.service'
 
 @Component({
   selector: 'app-storage-path-edit-dialog',
@@ -13,12 +12,8 @@ import { ToastService } from 'src/app/services/toast.service'
   styleUrls: ['./storage-path-edit-dialog.component.scss'],
 })
 export class StoragePathEditDialogComponent extends EditDialogComponent<PaperlessStoragePath> {
-  constructor(
-    service: StoragePathService,
-    activeModal: NgbActiveModal,
-    toastService: ToastService
-  ) {
-    super(service, activeModal, toastService)
+  constructor(service: StoragePathService, activeModal: NgbActiveModal) {
+    super(service, activeModal)
   }
 
   get pathHint() {

src-ui/src/app/components/common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component.ts

@@ -4,7 +4,6 @@ import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
 import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component'
 import { PaperlessTag } from 'src/app/data/paperless-tag'
 import { TagService } from 'src/app/services/rest/tag.service'
-import { ToastService } from 'src/app/services/toast.service'
 import { randomColor } from 'src/app/utils/color'
 import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 
@@ -14,12 +13,8 @@ import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
   styleUrls: ['./tag-edit-dialog.component.scss'],
 })
 export class TagEditDialogComponent extends EditDialogComponent<PaperlessTag> {
-  constructor(
-    service: TagService,
-    activeModal: NgbActiveModal,
-    toastService: ToastService
-  ) {
-    super(service, activeModal, toastService)
+  constructor(service: TagService, activeModal: NgbActiveModal) {
+    super(service, activeModal)
   }
 
   getCreateTitle() {

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html

@@ -4,7 +4,7 @@
       <use attr.xlink:href="assets/bootstrap-icons.svg#{{icon}}" />
     </svg>
     <div class="d-none d-sm-inline">&nbsp;{{title}}</div>
-    <ng-container *ngIf="!editing && selectionModel.selectionSize() > 0">
+    <ng-container *ngIf="!editing && selectionModel.totalCount > 0">
       <app-clearable-badge [number]="multiple ? selectionModel.totalCount : undefined" [selected]="!multiple && selectionModel.selectionSize() > 0" (cleared)="reset()"></app-clearable-badge>
     </ng-container>
   </button>

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts

@@ -321,7 +321,7 @@ export class FilterableDropdownComponent {
   apply = new EventEmitter<ChangedItems>()
 
   @Output()
-  open = new EventEmitter()
+  opened = new EventEmitter()
 
   get operatorToggleEnabled(): boolean {
     return (
@@ -356,7 +356,7 @@ export class FilterableDropdownComponent {
       if (this.editing) {
         this.selectionModel.reset()
       }
-      this.open.next(this)
+      this.opened.next(this)
     } else {
       this.filterText = ''
       if (this.applyOnClose && this.selectionModel.isDirty()) {

src-ui/src/app/components/common/input/number/number.component.html

@@ -2,7 +2,7 @@
   <label class="form-label" [for]="inputId">{{title}}</label>
   <div class="input-group" [class.is-invalid]="error">
     <input type="number" class="form-control" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)" [class.is-invalid]="error">
-    <button class="btn btn-outline-secondary" type="button" id="button-addon1" (click)="nextAsn()" [disabled]="value">+1</button>
+    <button *ngIf="showAdd" class="btn btn-outline-secondary" type="button" id="button-addon1" (click)="nextAsn()" [disabled]="value">+1</button>
   </div>
   <div class="invalid-feedback">
     {{error}}

src-ui/src/app/components/common/input/number/number.component.ts

@@ -1,4 +1,4 @@
-import { Component, forwardRef } from '@angular/core'
+import { Component, forwardRef, Input } from '@angular/core'
 import { NG_VALUE_ACCESSOR } from '@angular/forms'
 import { FILTER_ASN_ISNULL } from 'src/app/data/filter-rule-type'
 import { DocumentService } from 'src/app/services/rest/document.service'
@@ -17,6 +17,9 @@ import { AbstractInputComponent } from '../abstract-input'
   styleUrls: ['./number.component.scss'],
 })
 export class NumberComponent extends AbstractInputComponent<number> {
+  @Input()
+  showAdd: boolean = true
+
   constructor(private documentService: DocumentService) {
     super()
   }

src-ui/src/app/components/common/input/password/password.component.html

@@ -0,0 +1,8 @@
+<div class="mb-3">
+  <label class="form-label" [for]="inputId">{{title}}</label>
+  <input #inputField type="password" class="form-control" [class.is-invalid]="error" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)">
+  <small *ngIf="hint" class="form-text text-muted" [innerHTML]="hint | safeHtml"></small>
+  <div class="invalid-feedback">
+    {{error}}
+  </div>
+</div>