v1.8.0 version strings

[Beta] Paperless-ngx v1.8.0 Release Candidate 1

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -63,9 +63,11 @@ body:
     attributes:
       label: Installation method
       options:
-        - Docker
+        - Docker - official image
+        - Docker - linuxserver.io image
         - Bare metal
         - Other (please describe above)
+      description: Note there are significant differences from the official image and linuxserver.io, please check if your issue is specific to the third-party image.
     validations:
       required: true
   - type: input

.github/scripts/cleanup-tags.py

@@ -0,0 +1,254 @@
+import logging
+import os
+from argparse import ArgumentParser
+from typing import Final
+from typing import List
+from urllib.parse import quote
+
+import requests
+from common import get_log_level
+
+logger = logging.getLogger("cleanup-tags")
+
+
+class GithubContainerRegistry:
+    def __init__(
+        self,
+        session: requests.Session,
+        token: str,
+        owner_or_org: str,
+    ):
+        self._session: requests.Session = session
+        self._token = token
+        self._owner_or_org = owner_or_org
+        # https://docs.github.com/en/rest/branches/branches
+        self._BRANCHES_ENDPOINT = "https://api.github.com/repos/{OWNER}/{REPO}/branches"
+        if self._owner_or_org == "paperless-ngx":
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-an-organization
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-package-version-for-an-organization
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+        else:
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-the-authenticated-user
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+
+    def __enter__(self):
+        self._session.headers.update(
+            {
+                "Accept": "application/vnd.github.v3+json",
+                "Authorization": f"token {self._token}",
+            },
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if "Accept" in self._session.headers:
+            del self._session.headers["Accept"]
+        if "Authorization" in self._session.headers:
+            del self._session.headers["Authorization"]
+
+    def _read_all_pages(self, endpoint):
+        internal_data = []
+
+        while True:
+            resp = self._session.get(endpoint)
+            if resp.status_code == 200:
+                internal_data += resp.json()
+                if "next" in resp.links:
+                    endpoint = resp.links["next"]["url"]
+                else:
+                    logger.debug("Exiting pagination loop")
+                    break
+            else:
+                logger.warning(f"Request to {endpoint} return HTTP {resp.status_code}")
+                break
+
+        return internal_data
+
+    def get_branches(self, repo: str):
+        endpoint = self._BRANCHES_ENDPOINT.format(OWNER=self._owner_or_org, REPO=repo)
+        internal_data = self._read_all_pages(endpoint)
+        return internal_data
+
+    def filter_branches_by_name_pattern(self, branch_data, pattern: str):
+        matches = {}
+
+        for branch in branch_data:
+            if branch["name"].startswith(pattern):
+                matches[branch["name"]] = branch
+
+        return matches
+
+    def get_package_versions(
+        self,
+        package_name: str,
+        package_type: str = "container",
+    ) -> List:
+        package_name = quote(package_name, safe="")
+        endpoint = self._PACKAGES_VERSIONS_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_type,
+            PACKAGE_NAME=package_name,
+        )
+
+        internal_data = self._read_all_pages(endpoint)
+
+        return internal_data
+
+    def filter_packages_by_tag_pattern(self, package_data, pattern: str):
+        matches = {}
+
+        for package in package_data:
+            if "metadata" in package and "container" in package["metadata"]:
+                container_metadata = package["metadata"]["container"]
+                if "tags" in container_metadata:
+                    container_tags = container_metadata["tags"]
+                    for tag in container_tags:
+                        if tag.startswith(pattern):
+                            matches[tag] = package
+                            break
+
+        return matches
+
+    def filter_packages_untagged(self, package_data):
+        matches = {}
+
+        for package in package_data:
+            if "metadata" in package and "container" in package["metadata"]:
+                container_metadata = package["metadata"]["container"]
+                if "tags" in container_metadata:
+                    container_tags = container_metadata["tags"]
+                    if not len(container_tags):
+                        matches[package["name"]] = package
+
+        return matches
+
+    def delete_package_version(self, package_name, package_data):
+        package_name = quote(package_name, safe="")
+        endpoint = self._PACKAGE_VERSION_DELETE_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_data["metadata"]["package_type"],
+            PACKAGE_NAME=package_name,
+            PACKAGE_VERSION_ID=package_data["id"],
+        )
+        resp = self._session.delete(endpoint)
+        if resp.status_code != 204:
+            logger.warning(
+                f"Request to delete {endpoint} returned HTTP {resp.status_code}",
+            )
+
+
+def _main():
+    parser = ArgumentParser(
+        description="Using the GitHub API locate and optionally delete container"
+        " tags which no longer have an associated feature branch",
+    )
+
+    parser.add_argument(
+        "--delete",
+        action="store_true",
+        default=False,
+        help="If provided, actually delete the container tags",
+    )
+
+    # TODO There's a lot of untagged images, do those need to stay for anything?
+    parser.add_argument(
+        "--untagged",
+        action="store_true",
+        default=False,
+        help="If provided, delete untagged containers as well",
+    )
+
+    parser.add_argument(
+        "--loglevel",
+        default="info",
+        help="Configures the logging level",
+    )
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=get_log_level(args),
+        datefmt="%Y-%m-%d %H:%M:%S",
+        format="%(asctime)s %(levelname)-8s %(message)s",
+    )
+
+    repo_owner: Final[str] = os.environ["GITHUB_REPOSITORY_OWNER"]
+    repo: Final[str] = os.environ["GITHUB_REPOSITORY"]
+    gh_token: Final[str] = os.environ["GITHUB_TOKEN"]
+
+    with requests.session() as sess:
+        with GithubContainerRegistry(sess, gh_token, repo_owner) as gh_api:
+            all_branches = gh_api.get_branches("paperless-ngx")
+            logger.info(f"Located {len(all_branches)} branches of {repo_owner}/{repo} ")
+
+            feature_branches = gh_api.filter_branches_by_name_pattern(
+                all_branches,
+                "feature-",
+            )
+            logger.info(f"Located {len(feature_branches)} feature branches")
+
+            for package_name in ["paperless-ngx", "paperless-ngx/builder/cache/app"]:
+
+                all_package_versions = gh_api.get_package_versions(package_name)
+                logger.info(
+                    f"Located {len(all_package_versions)} versions of package {package_name}",
+                )
+
+                packages_tagged_feature = gh_api.filter_packages_by_tag_pattern(
+                    all_package_versions,
+                    "feature-",
+                )
+                logger.info(
+                    f'Located {len(packages_tagged_feature)} versions of package {package_name} tagged "feature-"',
+                )
+
+                untagged_packages = gh_api.filter_packages_untagged(
+                    all_package_versions,
+                )
+                logger.info(
+                    f"Located {len(untagged_packages)} untagged versions of package {package_name}",
+                )
+
+                to_delete = list(
+                    set(packages_tagged_feature.keys()) - set(feature_branches.keys()),
+                )
+                logger.info(
+                    f"Located {len(to_delete)} versions of package {package_name} to delete",
+                )
+
+                for tag_to_delete in to_delete:
+                    package_version_info = packages_tagged_feature[tag_to_delete]
+
+                    if args.delete:
+                        logger.info(
+                            f"Deleting {tag_to_delete} (id {package_version_info['id']})",
+                        )
+                        gh_api.delete_package_version(
+                            package_name,
+                            package_version_info,
+                        )
+
+                    else:
+                        logger.info(
+                            f"Would delete {tag_to_delete} (id {package_version_info['id']})",
+                        )
+
+                if args.untagged:
+                    logger.info(f"Deleting untagged packages of {package_name}")
+                    for to_delete_name in untagged_packages:
+                        to_delete_version = untagged_packages[to_delete_name]
+                        logger.info(f"Deleting id {to_delete_version['id']}")
+                        if args.delete:
+                            gh_api.delete_package_version(
+                                package_name,
+                                to_delete_version,
+                            )
+                else:
+                    logger.info("Leaving untagged images untouched")
+
+
+if __name__ == "__main__":
+    _main()

.github/scripts/common.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+import logging
+from argparse import ArgumentError
 
 
 def get_image_tag(
@@ -9,7 +11,7 @@ def get_image_tag(
     """
     Returns a string representing the normal image for a given package
     """
-    return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+    return f"ghcr.io/{repo_name.lower()}/builder/{pkg_name}:{pkg_version}"
 
 
 def get_cache_image_tag(
@@ -24,4 +26,19 @@ def get_cache_image_tag(
     Registry type caching is utilized for the builder images, to allow fast
     rebuilds, generally almost instant for the same version
     """
-    return f"ghcr.io/{repo_name}/builder/cache/{pkg_name}:{pkg_version}"
+    return f"ghcr.io/{repo_name.lower()}/builder/cache/{pkg_name}:{pkg_version}"
+
+
+def get_log_level(args) -> int:
+    levels = {
+        "critical": logging.CRITICAL,
+        "error": logging.ERROR,
+        "warn": logging.WARNING,
+        "warning": logging.WARNING,
+        "info": logging.INFO,
+        "debug": logging.DEBUG,
+    }
+    level = levels.get(args.loglevel.lower())
+    if level is None:
+        level = logging.INFO
+    return level

.github/scripts/get-build-json.py

@@ -50,7 +50,6 @@ def _main():
 
     # Default output values
     version = None
-    git_tag = None
     extra_config = {}
 
     if args.package in pipfile_data["default"]:
@@ -59,12 +58,6 @@ def _main():
         pkg_version = pkg_data["version"].split("==")[-1]
         version = pkg_version
 
-        # Based on the package, generate the expected Git tag name
-        if args.package == "pikepdf":
-            git_tag = f"v{pkg_version}"
-        elif args.package == "psycopg2":
-            git_tag = pkg_version.replace(".", "_")
-
         # Any extra/special values needed
         if args.package == "pikepdf":
             extra_config["qpdf_version"] = build_json["qpdf"]["version"]
@@ -72,8 +65,6 @@ def _main():
     elif args.package in build_json:
         version = build_json[args.package]["version"]
 
-        if "git_tag" in build_json[args.package]:
-            git_tag = build_json[args.package]["git_tag"]
     else:
         raise NotImplementedError(args.package)
 
@@ -81,7 +72,6 @@ def _main():
     output = {
         "name": args.package,
         "version": version,
-        "git_tag": git_tag,
         "image_tag": get_image_tag(repo_name, args.package, version),
         "cache_tag": get_cache_image_tag(
             repo_name,

.github/stale.yml

@@ -1,15 +1,23 @@
 # Number of days of inactivity before an issue becomes stale
 daysUntilStale: 30
+
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
-onlyLabels:
-  - unconfirmed
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: [cant-reproduce]
+
 # Label to use when marking an issue as stale
 staleLabel: stale
+
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
   This issue has been automatically marked as stale because it has not had
   recent activity. It will be closed if no further activity occurs. Thank you
   for your contributions.
+  
 # Comment to post when closing a stale issue. Set to `false` to disable
 closeComment: false
+
+# See https://github.com/marketplace/stale for more info on the app
+# and https://github.com/probot/stale for the configuration docs

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
         run: pipx install pipenv
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
           cache: "pipenv"
@@ -57,17 +57,29 @@ jobs:
     name: Prepare Docker Pipeline Data
     if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-20.04
+    # If the push triggered the installer library workflow, wait for it to
+    # complete here.  This ensures the required versions for the final
+    # image have been built, while not waiting at all if the versions haven't changed
+    concurrency:
+      group: build-installer-library
+      cancel-in-progress: false
     needs:
       - documentation
       - ci-backend
       - ci-frontend
     steps:
+      -
+        name: Set ghcr repository name
+        id: set-ghcr-repository
+        run: |
+          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
+          echo ::set-output name=repository::${ghcr_name}
       -
         name: Checkout
         uses: actions/checkout@v3
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "3.9"
       -
@@ -109,6 +121,8 @@ jobs:
 
     outputs:
 
+      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
+
       qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
 
       pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
@@ -117,55 +131,6 @@ jobs:
 
       jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
 
-  build-qpdf-debs:
-    name: qpdf
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.qpdf
-      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
-      build-args: |
-        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-
-  build-jbig2enc:
-    name: jbig2enc
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.jbig2enc
-      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
-      build-args: |
-        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
-
-  build-psycopg2-wheel:
-    name: psycopg2
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.psycopg2
-      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
-      build-args: |
-        PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
-        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
-
-  build-pikepdf-wheel:
-    name: pikepdf
-    needs:
-      - prepare-docker-build
-      - build-qpdf-debs
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.pikepdf
-      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
-      build-args: |
-        REPO=${{ github.repository }}
-        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-        PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
-        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
-
   # build and push image to docker hub.
   build-docker-image:
     runs-on: ubuntu-20.04
@@ -174,29 +139,31 @@ jobs:
       cancel-in-progress: true
     needs:
       - prepare-docker-build
-      - build-psycopg2-wheel
-      - build-jbig2enc
-      - build-qpdf-debs
-      - build-pikepdf-wheel
     steps:
       -
         name: Check pushing to Docker Hub
         id: docker-hub
-        # Only push to Dockerhub from the main repo
+        # Only push to Dockerhub from the main repo AND the ref is either:
+        #  main
+        #  dev
+        #  beta
+        #  a tag
         # Otherwise forks would require a Docker Hub account and secrets setup
         run: |
-          if [[ ${{ github.repository }} == "paperless-ngx/paperless-ngx" ]] ; then
+          if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
+            echo "Enabling DockerHub image push"
             echo ::set-output name=enable::"true"
           else
+            echo "Not pushing to DockerHub"
             echo ::set-output name=enable::"false"
           fi
       -
         name: Gather Docker metadata
         id: docker-meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: |
-            ghcr.io/${{ github.repository }}
+            ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}
             name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
           tags: |
             # Tag branches with branch name
@@ -210,20 +177,20 @@ jobs:
         uses: actions/checkout@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       -
         name: Login to Github Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         # Don't attempt to login is not pushing to Docker Hub
         if: steps.docker-hub.outputs.enable == 'true'
         with:
@@ -231,7 +198,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./Dockerfile
@@ -247,11 +214,11 @@ jobs:
           # Get cache layers from this branch, then dev, then main
           # This allows new branches to get at least some cache benefits, generally from dev
           cache-from: |
-            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
-            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:dev
-            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:main
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main
           cache-to: |
-            type=registry,mode=max,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
       -
         name: Inspect image
         run: |
@@ -278,7 +245,7 @@ jobs:
         uses: actions/checkout@v3
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
       -
@@ -338,6 +305,10 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
+    outputs:
+      prerelease: ${{ steps.get_version.outputs.prerelease }}
+      changelog: ${{ steps.create-release.outputs.body }}
+      version: ${{ steps.get_version.outputs.version }}
     needs:
       - build-release
     if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
@@ -381,6 +352,13 @@ jobs:
           asset_path: ./paperless-ngx.tar.xz
           asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
           asset_content_type: application/x-xz
+
+  append-changelog:
+    runs-on: ubuntu-20.04
+    needs:
+      - publish-release
+    if: needs.publish-release.outputs.prerelease == 'false'
+    steps:
       -
         name: Checkout
         uses: actions/checkout@v3
@@ -391,11 +369,33 @@ jobs:
         id: append-Changelog
         working-directory: docs
         run: |
-          echo -e "# Changelog\n\n${{ steps.create-release.outputs.body }}\n" > changelog-new.md
+          git branch ${{ needs.publish-release.outputs.version }}-changelog
+          git checkout ${{ needs.publish-release.outputs.version }}-changelog
+          echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md
           CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
           echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
           mv changelog-new.md changelog.md
           git config --global user.name "github-actions"
           git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git commit -am "Changelog ${{ steps.get_version.outputs.version }} - GHA"
-          git push origin HEAD:main
+          git push origin ${{ needs.publish-release.outputs.version }}-changelog
+      -
+        name: Create Pull Request
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const { repo, owner } = context.repo;
+            const result = await github.rest.pulls.create({
+              title: '[Documentation] Add ${{ needs.publish-release.outputs.version }} changelog',
+              owner,
+              repo,
+              head: '${{ needs.publish-release.outputs.version }}-changelog',
+              base: 'main',
+              body: 'This PR is auto-generated by CI.'
+            });
+            github.rest.issues.addLabels({
+              owner,
+              repo,
+              issue_number: result.data.number,
+              labels: ['documentation']
+            });

.github/workflows/cleanup-tags.yml

@@ -0,0 +1,48 @@
+name: Cleanup Image Tags
+
+on:
+  schedule:
+    - cron: '0 0 * * SAT'
+  delete:
+  pull_request:
+    types:
+      - closed
+  push:
+    paths:
+      - ".github/workflows/cleanup-tags.yml"
+      - ".github/scripts/cleanup-tags.py"
+      - ".github/scripts/common.py"
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  cleanup:
+    name: Cleanup Image Tags
+    runs-on: ubuntu-20.04
+    permissions:
+      packages: write
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+      -
+        name: Install requests
+        run: |
+          python -m pip install requests
+      -
+        name: Cleanup feature tags
+        run: |
+          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --delete

.github/workflows/installer-library.yml

@@ -0,0 +1,147 @@
+# This workflow will run to update the installer library of
+# Docker images.  These are the images which provide updated wheels
+# .deb installation packages or maybe just some compiled library
+
+name: Build Image Library
+
+on:
+  push:
+    # Must match one of these branches AND one of the paths
+    # to be triggered
+    branches:
+      - "main"
+      - "dev"
+      - "library-*"
+      - "feature-*"
+    paths:
+      # Trigger the workflow if a Dockerfile changed
+      - "docker-builders/**"
+      # Trigger if a package was updated
+      - ".build-config.json"
+      - "Pipfile.lock"
+      # Also trigger on workflow changes related to the library
+      - ".github/workflows/installer-library.yml"
+      - ".github/workflows/reusable-workflow-builder.yml"
+      - ".github/scripts/**"
+
+# Set a workflow level concurrency group so primary workflow
+# can wait for this to complete if needed
+# DO NOT CHANGE without updating main workflow group
+concurrency:
+  group: build-installer-library
+  cancel-in-progress: false
+
+jobs:
+  prepare-docker-build:
+    name: Prepare Docker Image Version Data
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Set ghcr repository name
+        id: set-ghcr-repository
+        run: |
+          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
+          echo ::set-output name=repository::${ghcr_name}
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      -
+        name: Setup qpdf image
+        id: qpdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=qpdf-json::${build_json}
+      -
+        name: Setup psycopg2 image
+        id: psycopg2-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo ::set-output name=psycopg2-json::${build_json}
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=pikepdf-json::${build_json}
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo ::set-output name=jbig2enc-json::${build_json}
+
+    outputs:
+
+      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+
+  build-qpdf-debs:
+    name: qpdf
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+      build-args: |
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+  build-jbig2enc:
+    name: jbig2enc
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.jbig2enc
+      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+      build-args: |
+        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+  build-psycopg2-wheel:
+    name: psycopg2
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.psycopg2
+      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+      build-args: |
+        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+  build-pikepdf-wheel:
+    name: pikepdf
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.pikepdf
+      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+      build-args: |
+        REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }}
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}

.github/workflows/reusable-ci-backend.yml

@@ -65,7 +65,7 @@ jobs:
         run: pipx install pipenv
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "${{ matrix.python-version }}"
           cache: "pipenv"
@@ -74,7 +74,7 @@ jobs:
         name: Install system dependencies
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
       -
         name: Install Python dependencies
         run: |
@@ -87,7 +87,7 @@ jobs:
       -
         name: Get changed files
         id: changed-files-specific
-        uses: tj-actions/changed-files@v19
+        uses: tj-actions/changed-files@v23.1
         with:
           files: |
             src/**
@@ -106,3 +106,24 @@ jobs:
         run: |
           cd src/
           pipenv run coveralls --service=github
+
+  dockerfile-lint:
+    name: "Lint ${{ matrix.dockerfile }}"
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        dockerfile:
+          - Dockerfile
+          - docker-builders/Dockerfile.qpdf
+          - docker-builders/Dockerfile.jbig2enc
+          - docker-builders/Dockerfile.psycopg2
+          - docker-builders/Dockerfile.pikepdf
+      fail-fast: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        uses: hadolint/hadolint-action@v2.1.0
+        with:
+          dockerfile: ${{ matrix.dockerfile }}

.github/workflows/reusable-workflow-builder.yml

@@ -28,20 +28,20 @@ jobs:
         uses: actions/checkout@v3
       -
         name: Login to Github Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       -
         name: Build ${{ fromJSON(inputs.build-json).name }}
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ${{ inputs.dockerfile }}

.gitignore

@@ -63,11 +63,14 @@ target/
 
 # VS Code
 .vscode
+/src-ui/.vscode
+/docs/.vscode
 
 # Other stuff that doesn't belong
 .virtualenv
 virtualenv
 /venv
+.venv/
 /docker-compose.env
 /docker-compose.yml
 
@@ -84,8 +87,9 @@ scripts/nuke
 /paperless.conf
 /consume/
 /export/
-/src-ui/.vscode
 
 # this is where the compiled frontend is moved to.
 /src/documents/static/frontend/
-/docs/.vscode/settings.json
+
+# mac os
+.DS_Store

.hadolint.yml

@@ -0,0 +1,8 @@
+failure-threshold: warning
+ignored:
+  # https://github.com/hadolint/hadolint/wiki/DL3008
+  - DL3008
+  # https://github.com/hadolint/hadolint/wiki/DL3013
+  - DL3013
+  # https://github.com/hadolint/hadolint/wiki/DL3003
+  - DL3003

.pre-commit-config.yaml

@@ -5,7 +5,7 @@
 repos:
   # General hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v4.3.0
     hooks:
       - id: check-docstring-first
       - id: check-json
@@ -27,7 +27,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.6.2"
+    rev: "v2.7.1"
     hooks:
       - id: prettier
         types_or:
@@ -37,7 +37,7 @@ repos:
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.1.0
+    rev: v3.8.1
     hooks:
       - id: reorder-python-imports
         exclude: "(migrations)"
@@ -59,11 +59,11 @@ repos:
         args:
           - "--config=./src/setup.cfg"
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.6.0
     hooks:
       - id: black
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.1
+    rev: v2.37.1
     hooks:
       - id: pyupgrade
         exclude: "(migrations)"
@@ -74,13 +74,6 @@ repos:
     rev: v2.10.0
     hooks:
       - id: hadolint
-        args:
-          - --ignore
-          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
-          - --ignore
-          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
-          - --ignore
-          - DL3003      # https://github.com/hadolint/hadolint/wiki/DL3003  (seems excessive to use WORKDIR so much)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1

Dockerfile

@@ -26,7 +26,7 @@ COPY ./src-ui /src/src-ui
 WORKDIR /src/src-ui
 RUN set -eux \
   && npm update npm -g \
-  && npm ci --no-optional
+  && npm ci --omit=optional
 RUN set -eux \
   && ./node_modules/.bin/ng build --configuration production
 
@@ -77,15 +77,12 @@ ARG RUNTIME_PACKAGES="\
   libraqm0 \
   libgnutls30 \
   libjpeg62-turbo \
-  optipng \
   python3 \
   python3-pip \
   python3-setuptools \
   postgresql-client \
   # For Numpy
   libatlas3-base \
-  # thumbnail size reduction
-  pngquant \
   # OCRmyPDF dependencies
   tesseract-ocr \
   tesseract-ocr-eng \
@@ -93,6 +90,10 @@ ARG RUNTIME_PACKAGES="\
   tesseract-ocr-fra \
   tesseract-ocr-ita \
   tesseract-ocr-spa \
+  # Suggested for OCRmyPDF
+  pngquant \
+  # Suggested for pikepdf
+  jbig2dec \
   tzdata \
   unpaper \
   # Mime type detection
@@ -122,20 +123,33 @@ COPY gunicorn.conf.py .
 # These change sometimes, but rarely
 WORKDIR /usr/src/paperless/src/docker/
 
-RUN --mount=type=bind,readwrite,source=docker,target=./ \
-  set -eux \
+COPY [ \
+  "docker/imagemagick-policy.xml", \
+  "docker/supervisord.conf", \
+  "docker/docker-entrypoint.sh", \
+  "docker/docker-prepare.sh", \
+  "docker/paperless_cmd.sh", \
+  "docker/wait-for-redis.py", \
+  "docker/management_script.sh", \
+  "docker/install_management_commands.sh", \
+  "/usr/src/paperless/src/docker/" \
+]
+
+RUN set -eux \
   && echo "Configuring ImageMagick" \
-    && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+    && mv imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
   && echo "Configuring supervisord" \
     && mkdir /var/log/supervisord /var/run/supervisord \
-    && cp supervisord.conf /etc/supervisord.conf \
+    && mv supervisord.conf /etc/supervisord.conf \
   && echo "Setting up Docker scripts" \
-    && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
+    && mv docker-entrypoint.sh /sbin/docker-entrypoint.sh \
     && chmod 755 /sbin/docker-entrypoint.sh \
-    && cp docker-prepare.sh /sbin/docker-prepare.sh \
+    && mv docker-prepare.sh /sbin/docker-prepare.sh \
     && chmod 755 /sbin/docker-prepare.sh \
-    && cp wait-for-redis.py /sbin/wait-for-redis.py \
+    && mv wait-for-redis.py /sbin/wait-for-redis.py \
     && chmod 755 /sbin/wait-for-redis.py \
+    && mv paperless_cmd.sh /usr/local/bin/paperless_cmd.sh \
+    && chmod 755 /usr/local/bin/paperless_cmd.sh \
   && echo "Installing managment commands" \
     && chmod +x install_management_commands.sh \
     && ./install_management_commands.sh
@@ -151,15 +165,15 @@ RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
     && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \
     && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
   && echo "Installing pikepdf and dependencies" \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/packaging*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/lxml*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/Pillow*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pyparsing*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pikepdf*.whl \
-    && python -m pip list \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \
+    && python3 -m pip list \
   && echo "Installing psycopg2" \
-    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/psycopg2/wheels/psycopg2*.whl \
-    && python -m pip list
+    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
+    && python3 -m pip list
 
 # Python dependencies
 # Change pretty frequently
@@ -169,6 +183,7 @@ COPY requirements.txt ../
 # dependencies
 ARG BUILD_PACKAGES="\
   build-essential \
+  git \
   python3-dev"
 
 RUN set -eux \
@@ -213,4 +228,4 @@ ENTRYPOINT ["/sbin/docker-entrypoint.sh"]
 
 EXPOSE 8000
 
-CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf"]
+CMD ["/usr/local/bin/paperless_cmd.sh"]

Pipfile

@@ -13,29 +13,26 @@ dateparser = "~=1.1"
 django = "~=4.0"
 django-cors-headers = "*"
 django-extensions = "*"
-django-filter = "~=21.1"
-django-q = "~=1.3"
+django-filter = "~=22.1"
+django-q = {editable = true, ref = "paperless-main", git = "https://github.com/paperless-ngx/django-q.git"}
 djangorestframework = "~=3.13"
 filelock = "*"
 fuzzywuzzy = {extras = ["speedup"], version = "*"}
 gunicorn = "*"
-imap-tools = "~=0.54.0"
+imap-tools = "*"
 langdetect = "*"
 pathvalidate = "*"
-pillow = "~=9.1"
-# Any version update to pikepdf requires a base image update
+pillow = "~=9.2"
 pikepdf = "~=5.1"
 python-gnupg = "*"
 python-dotenv = "*"
 python-dateutil = "*"
 python-magic = "*"
-# Any version update to psycopg2 requires a base image update
 psycopg2 = "*"
 redis = "*"
-# Pinned because aarch64 wheels and updates cause warnings when loading the classifier model.
-scikit-learn="==1.0.2"
-whitenoise = "~=6.0.0"
-watchdog = "~=2.1.0"
+scikit-learn="~=1.1"
+whitenoise = "~=6.2.0"
+watchdog = "~=2.1.9"
 whoosh="~=2.7.4"
 inotifyrecursive = "~=0.3"
 ocrmypdf = "~=13.4"
@@ -64,9 +61,10 @@ pytest-django = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-sphinx = "~=4.5.0"
+sphinx = "~=5.0.2"
 sphinx_rtd_theme = "*"
 tox = "*"
 black = "*"
 pre-commit = "*"
+sphinx-autobuild = "*"
 myst-parser = "*"

README.md

@@ -102,7 +102,7 @@ For bugs please [open an issue](https://github.com/paperless-ngx/paperless-ngx/i
 
 Paperless has been around a while now, and people are starting to build stuff on top of it. If you're one of those people, we can add your project to this list:
 
-- [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ngx.
+- [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ng.
 - [Paperless Share](https://github.com/qcasey/paperless_share). Share any files from your Android application with paperless. Very simple, but works with all of the mobile scanning apps out there that allow you to share scanned documents.
 - [Scan to Paperless](https://github.com/sbrunner/scan-to-paperless): Scan and prepare (crop, deskew, OCR, ...) your documents for Paperless.
 

build-docker-image.sh

@@ -34,6 +34,7 @@ branch_name=$(git rev-parse --abbrev-ref HEAD)
 export DOCKER_BUILDKIT=1
 
 docker build --file "$1" \
+	--progress=plain \
 	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
 	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
 	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \

docker-builders/Dockerfile.frontend

@@ -9,6 +9,6 @@ COPY ./src-ui /src/src-ui
 WORKDIR /src/src-ui
 RUN set -eux \
   && npm update npm -g \
-  && npm ci --no-optional
+  && npm ci --omit=optional
 RUN set -eux \
   && ./node_modules/.bin/ng build --configuration production

docker-builders/Dockerfile.pikepdf

@@ -2,8 +2,7 @@
 # Inputs:
 #    - REPO - Docker repository to pull qpdf from
 #    - QPDF_VERSION - The image qpdf version to copy .deb files from
-#    - PIKEPDF_GIT_TAG - The Git tag to clone and build from
-#    - PIKEPDF_VERSION - Used to force the built pikepdf version to match
+#    - PIKEPDF_VERSION - Version of pikepdf to build wheel for
 
 # Default to pulling from the main repo registry when manually building
 ARG REPO="paperless-ngx/paperless-ngx"
@@ -23,7 +22,6 @@ ARG BUILD_PACKAGES="\
   build-essential \
   python3-dev \
   python3-pip \
-  git \
   # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers
   libgnutls28-dev \
   # lxml requrements - https://lxml.de/installation.html
@@ -72,21 +70,19 @@ RUN set -eux \
 # For better caching, seperate the basic installs from
 # the building
 
-ARG PIKEPDF_GIT_TAG
 ARG PIKEPDF_VERSION
 
 RUN set -eux \
-  && echo "building pikepdf wheel" \
-  # Note the v in the tag name here
-  && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
-  && cd pikepdf \
-  # pikepdf seems to specifciy either a next version when built OR
-  # a post release tag.
-  # In either case, this won't match what we want from requirements.txt
-  # Directly modify the setup.py to set the version we just checked out of Git
-  && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \
-  # https://github.com/pikepdf/pikepdf/issues/323
-  && rm pyproject.toml \
+  && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \
   && mkdir wheels \
-  && python3 -m pip wheel . --wheel-dir wheels \
+  && python3 -m pip wheel \
+    # Build the package at the required version
+    pikepdf==${PIKEPDF_VERSION} \
+    # Output the *.whl into this directory
+    --wheel-dir wheels \
+    # Do not use a binary packge for the package being built
+    --no-binary=pikepdf \
+    # Do use binary packages for dependencies
+    --prefer-binary \
+    --no-cache-dir \
   && ls -ahl wheels

docker-builders/Dockerfile.psycopg2

@@ -1,7 +1,6 @@
 # This Dockerfile builds the psycopg2 wheel
 # Inputs:
-#    - PSYCOPG2_GIT_TAG - The Git tag to clone and build from
-#    - PSYCOPG2_VERSION - Unused, kept for future possible usage
+#    - PSYCOPG2_VERSION - Version to build
 
 FROM python:3.9-slim-bullseye as main
 
@@ -11,7 +10,6 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 ARG BUILD_PACKAGES="\
   build-essential \
-  git \
   python3-dev \
   python3-pip \
   # https://www.psycopg.org/docs/install.html#prerequisites
@@ -32,14 +30,20 @@ RUN set -eux \
 # For better caching, seperate the basic installs from
 # the building
 
-ARG PSYCOPG2_GIT_TAG
 ARG PSYCOPG2_VERSION
 
 RUN set -eux \
-  && echo "Building psycopg2 wheel" \
+  && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \
   && cd /usr/src \
-  && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \
-  && cd psycopg2 \
   && mkdir wheels \
-  && python3 -m pip wheel . --wheel-dir wheels \
+  && python3 -m pip wheel \
+    # Build the package at the required version
+    psycopg2==${PSYCOPG2_VERSION} \
+    # Output the *.whl into this directory
+    --wheel-dir wheels \
+    # Do not use a binary packge for the package being built
+    --no-binary=psycopg2 \
+    # Do use binary packages for dependencies
+    --prefer-binary \
+    --no-cache-dir \
   && ls -ahl wheels/

docker/compose/docker-compose.portainer.yml

@@ -31,13 +31,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.postgres-tika.yml

@@ -33,13 +33,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
@@ -77,7 +77,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"

docker/compose/docker-compose.postgres.yml

@@ -29,13 +29,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.sqlite-tika.yml

@@ -33,7 +33,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
@@ -65,7 +65,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"

docker/compose/docker-compose.sqlite.yml

@@ -26,7 +26,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data

docker/docker-entrypoint.sh

@@ -2,6 +2,37 @@
 
 set -e
 
+# Adapted from:
+# https://github.com/docker-library/postgres/blob/master/docker-entrypoint.sh
+# usage: file_env VAR
+#    ie: file_env 'XYZ_DB_PASSWORD' will allow for "$XYZ_DB_PASSWORD_FILE" to
+# fill in the value of "$XYZ_DB_PASSWORD" from a file, especially for Docker's
+# secrets feature
+file_env() {
+	local var="$1"
+	local fileVar="${var}_FILE"
+
+	# Basic validation
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+		exit 1
+	fi
+
+	# Only export var if the _FILE exists
+	if [ "${!fileVar:-}" ]; then
+		# And the file exists
+		if [[ -f ${!fileVar} ]]; then
+			echo "Setting ${var} from file"
+			val="$(< "${!fileVar}")"
+			export "$var"="$val"
+		else
+			echo "File ${!fileVar} doesn't exist"
+			exit 1
+		fi
+	fi
+
+}
+
 # Source: https://github.com/sameersbn/docker-gitlab/
 map_uidgid() {
 	USERMAP_ORIG_UID=$(id -u paperless)
@@ -15,26 +46,56 @@ map_uidgid() {
 	fi
 }
 
+map_folders() {
+	# Export these so they can be used in docker-prepare.sh
+	export DATA_DIR="${PAPERLESS_DATA_DIR:-/usr/src/paperless/data}"
+	export MEDIA_ROOT_DIR="${PAPERLESS_MEDIA_ROOT:-/usr/src/paperless/media}"
+}
+
 initialize() {
+
+	# Setup environment from secrets before anything else
+	for env_var in \
+		PAPERLESS_DBUSER \
+		PAPERLESS_DBPASS \
+		PAPERLESS_SECRET_KEY \
+		PAPERLESS_AUTO_LOGIN_USERNAME \
+		PAPERLESS_ADMIN_USER \
+		PAPERLESS_ADMIN_MAIL \
+		PAPERLESS_ADMIN_PASSWORD; do
+		# Check for a version of this var with _FILE appended
+		# and convert the contents to the env var value
+		file_env ${env_var}
+	done
+
+	# Change the user and group IDs if needed
 	map_uidgid
 
-	for dir in export data data/index media media/documents media/documents/originals media/documents/thumbnails; do
-		if [[ ! -d "../$dir" ]]; then
-			echo "Creating directory ../$dir"
-			mkdir ../$dir
+	# Check for overrides of certain folders
+	map_folders
+
+	local export_dir="/usr/src/paperless/export"
+
+	for dir in "${export_dir}" "${DATA_DIR}" "${DATA_DIR}/index" "${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails"; do
+		if [[ ! -d "${dir}" ]]; then
+			echo "Creating directory ${dir}"
+			mkdir "${dir}"
 		fi
 	done
 
-	echo "Creating directory /tmp/paperless"
-	mkdir -p /tmp/paperless
+	local tmp_dir="/tmp/paperless"
+	echo "Creating directory ${tmp_dir}"
+	mkdir -p "${tmp_dir}"
 
 	set +e
 	echo "Adjusting permissions of paperless files. This may take a while."
-	chown -R paperless:paperless /tmp/paperless
-	find .. -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	chown -R paperless:paperless ${tmp_dir}
+	for dir in "${export_dir}" "${DATA_DIR}" "${MEDIA_ROOT_DIR}"; do
+		find "${dir}" -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	done
 	set -e
 
-	gosu paperless /sbin/docker-prepare.sh
+	${gosu_cmd[@]} /sbin/docker-prepare.sh
 }
 
 install_languages() {
@@ -76,6 +137,11 @@ install_languages() {
 
 echo "Paperless-ngx docker container starting..."
 
+gosu_cmd=(gosu paperless)
+if [ $(id -u) == $(id -u paperless) ]; then
+	gosu_cmd=()
+fi
+
 # Install additional languages if specified
 if [[ -n "$PAPERLESS_OCR_LANGUAGES" ]]; then
 	install_languages "$PAPERLESS_OCR_LANGUAGES"
@@ -85,7 +151,7 @@ initialize
 
 if [[ "$1" != "/"* ]]; then
 	echo Executing management command "$@"
-	exec gosu paperless python3 manage.py "$@"
+	exec ${gosu_cmd[@]} python3 manage.py "$@"
 else
 	echo Executing "$@"
 	exec "$@"

docker/docker-prepare.sh

@@ -3,16 +3,17 @@
 set -e
 
 wait_for_postgres() {
-	attempt_num=1
-	max_attempts=5
+	local attempt_num=1
+	local max_attempts=5
 
 	echo "Waiting for PostgreSQL to start..."
 
-	host="${PAPERLESS_DBHOST:=localhost}"
-	port="${PAPERLESS_DBPORT:=5432}"
+	local host="${PAPERLESS_DBHOST:-localhost}"
+	local port="${PAPERLESS_DBPORT:-5432}"
 
-
-	while [ ! "$(pg_isready -h $host -p $port)" ]; do
+	# Disable warning, host and port can't have spaces
+	# shellcheck disable=SC2086
+	while [ ! "$(pg_isready -h ${host} -p ${port})" ]; do
 
 		if [ $attempt_num -eq $max_attempts ]; then
 			echo "Unable to connect to database."
@@ -43,17 +44,18 @@ migrations() {
 		flock 200
 		echo "Apply database migrations..."
 		python3 manage.py migrate
-	) 200>/usr/src/paperless/data/migration_lock
+	) 200>"${DATA_DIR}/migration_lock"
 }
 
 search_index() {
-	index_version=1
-	index_version_file=/usr/src/paperless/data/.index_version
 
-	if [[ (! -f "$index_version_file") || $(<$index_version_file) != "$index_version" ]]; then
+	local index_version=1
+	local index_version_file=${DATA_DIR}/.index_version
+
+	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
 		echo "Search index out of date. Updating..."
-		python3 manage.py document_index reindex
-		echo $index_version | tee $index_version_file >/dev/null
+		python3 manage.py document_index reindex --no-progress-bar
+		echo ${index_version} | tee "${index_version_file}" >/dev/null
 	fi
 }
 

docker/install_management_commands.sh

@@ -2,7 +2,18 @@
 
 set -eu
 
-for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
+for command in decrypt_documents \
+	document_archiver \
+	document_exporter \
+	document_importer \
+	mail_fetcher \
+	document_create_classifier \
+	document_index \
+	document_renamer \
+	document_retagger \
+	document_thumbnails \
+	document_sanity_checker \
+	manage_superuser;
 do
 	echo "installing $command..."
 	sed "s/management_command/$command/g" management_script.sh > /usr/local/bin/$command

docker/paperless_cmd.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+rootless_args=()
+if [ $(id -u) == $(id -u paperless) ]; then
+	rootless_args=(
+		--user
+		paperless
+		--logfile
+		supervisord.log
+		--pidfile
+		supervisord.pid
+	)
+fi
+
+/usr/local/bin/supervisord -c /etc/supervisord.conf ${rootless_args[@]}

docker/wait-for-redis.py

@@ -26,9 +26,11 @@ if __name__ == "__main__":
             try:
                 client.ping()
                 break
-            except Exception:
+            except Exception as e:
                 print(
-                    f"Redis ping #{attempt} failed, waiting {RETRY_SLEEP_SECONDS}s",
+                    f"Redis ping #{attempt} failed.\n"
+                    f"Error: {str(e)}.\n"
+                    f"Waiting {RETRY_SLEEP_SECONDS}s",
                     flush=True,
                 )
                 time.sleep(RETRY_SLEEP_SECONDS)

docs/Dockerfile

@@ -1,17 +0,0 @@
-FROM python:3.5.1
-
-# Install Sphinx and Pygments
-RUN pip install --no-cache-dir Sphinx Pygments \
-  # Setup directories, copy data
-  && mkdir /build
-
-COPY . /build
-WORKDIR /build/docs
-
-# Build documentation
-RUN make html
-
-# Start webserver
-WORKDIR /build/docs/_build/html
-EXPOSE 8000/tcp
-CMD ["python3", "-m", "http.server"]

docs/Makefile

@@ -24,6 +24,7 @@ I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html       to make standalone HTML files"
+	@echo "  livehtml   to preview changes with live reload in your browser"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
 	@echo "  singlehtml to make a single large HTML file"
 	@echo "  pickle     to make pickle files"
@@ -54,6 +55,9 @@ html:
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
+livehtml:
+	sphinx-autobuild "./" "$(BUILDDIR)" $(O)
+
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo

docs/_static/css/custom.css

@@ -64,6 +64,10 @@ body {
   color: var(--color-text-body);
 }
 
+.rst-content p {
+  word-break: break-word;
+}
+
 h1, h2, h3, h4, h5, h6 {
   font-family: inherit;
 }
@@ -435,7 +439,8 @@ a.image-reference img {
 }
 
 .rst-content code.literal,
-.rst-content tt.literal {
+.rst-content tt.literal,
+html.writer-html5 .rst-content dl.footnote code {
   border-color: var(--color-border);
   background-color: var(--color-border);
   color: var(--color-text-code-inline)

docs/_static/js/darkmode.js

@@ -1,47 +1,47 @@
-let toggleButton;
-let icon;
+let toggleButton
+let icon
 
 function load() {
-	"use strict";
+	'use strict'
 
-	toggleButton = document.createElement("button");
-	toggleButton.setAttribute("title", "Toggle dark mode");
-	toggleButton.classList.add("dark-mode-toggle");
-	icon = document.createElement("i");
-	icon.classList.add("fa", darkModeState ? "fa-sun-o" : "fa-moon-o");
-	toggleButton.appendChild(icon);
-	document.body.prepend(toggleButton);
+	toggleButton = document.createElement('button')
+	toggleButton.setAttribute('title', 'Toggle dark mode')
+	toggleButton.classList.add('dark-mode-toggle')
+	icon = document.createElement('i')
+	icon.classList.add('fa', darkModeState ? 'fa-sun-o' : 'fa-moon-o')
+	toggleButton.appendChild(icon)
+	document.body.prepend(toggleButton)
 
 	// Listen for changes in the OS settings
 	// addListener is used because older versions of Safari don't support addEventListener
 	// prefersDarkQuery set in <head>
 	if (prefersDarkQuery) {
 		prefersDarkQuery.addListener(function (evt) {
-			toggleDarkMode(evt.matches);
-		});
+			toggleDarkMode(evt.matches)
+		})
 	}
 
 	// Initial setting depending on the prefers-color-mode or localstorage
 	// darkModeState should be set in the document <head> to prevent flash
-	if (darkModeState == undefined) darkModeState = false;
-	toggleDarkMode(darkModeState);
+	if (darkModeState == undefined) darkModeState = false
+	toggleDarkMode(darkModeState)
 
 	// Toggles the "dark-mode" class on click and sets localStorage state
-	toggleButton.addEventListener("click", () => {
-		darkModeState = !darkModeState;
+	toggleButton.addEventListener('click', () => {
+		darkModeState = !darkModeState
 
-		toggleDarkMode(darkModeState);
-		localStorage.setItem("dark-mode", darkModeState);
-	});
+		toggleDarkMode(darkModeState)
+		localStorage.setItem('dark-mode', darkModeState)
+	})
 }
 
 function toggleDarkMode(state) {
-	document.documentElement.classList.toggle("dark-mode", state);
-	document.documentElement.classList.toggle("light-mode", !state);
-	icon.classList.remove("fa-sun-o");
-	icon.classList.remove("fa-moon-o");
-	icon.classList.add(state ? "fa-sun-o" : "fa-moon-o");
-	darkModeState = state;
+	document.documentElement.classList.toggle('dark-mode', state)
+	document.documentElement.classList.toggle('light-mode', !state)
+	icon.classList.remove('fa-sun-o')
+	icon.classList.remove('fa-moon-o')
+	icon.classList.add(state ? 'fa-sun-o' : 'fa-moon-o')
+	darkModeState = state
 }
 
-document.addEventListener("DOMContentLoaded", load);
+document.addEventListener('DOMContentLoaded', load)

docs/administration.rst

@@ -118,10 +118,10 @@ Then you can start paperless-ngx with ``-d`` to have it run in the background.
                 image: ghcr.io/paperless-ngx/paperless-ngx:latest
 
     .. note::
-        In version 1.7.1 and onwards, the Docker image can now pinned to a release series.
+        In version 1.7.1 and onwards, the Docker image can now be pinned to a release series.
         This is often combined with automatic updaters such as Watchtower to allow safer
         unattended upgrading to new bugfix releases only.  It is still recommended to always
-        review release notes before upgrading.  To ping your install to a release series, edit
+        review release notes before upgrading.  To pin your install to a release series, edit
         the ``docker-compose.yml`` find the line that says
 
             .. code::
@@ -287,6 +287,10 @@ When you use the provided docker compose script, put the export inside the
 ``export`` folder in your paperless source directory. Specify ``../export``
 as the ``source``.
 
+.. note::
+
+    Importing from a previous version of Paperless may work, but for best results
+    it is suggested to match the versions.
 
 .. _utilities-retagger:
 
@@ -386,8 +390,8 @@ the naming scheme.
 
 .. warning::
 
-    Since this command moves you documents around a lot, it is advised to to
-    a backup before. The renaming logic is robust and will never overwrite
+    Since this command moves your documents, it is advised to do
+    a backup beforehand. The renaming logic is robust and will never overwrite
     or delete a file, but you can't ever be careful enough.
 
 .. code::

docs/advanced_usage.rst

@@ -7,12 +7,12 @@ easier.
 
 .. _advanced-matching:
 
-Matching tags, correspondents and document types
-################################################
+Matching tags, correspondents, document types, and storage paths
+################################################################
 
-Paperless will compare the matching algorithms defined by every tag and
-correspondent already set in your database to see if they apply to the text in
-a document.  In other words, if you defined a tag called ``Home Utility``
+Paperless will compare the matching algorithms defined by every tag, correspondent,
+document type, and storage path in your database to see if they apply to the text
+in a document. In other words, if you define a tag called ``Home Utility``
 that had a ``match`` property of ``bc hydro`` and a ``matching_algorithm`` of
 ``literal``, Paperless will automatically tag your newly-consumed document with
 your ``Home Utility`` tag so long as the text ``bc hydro`` appears in the body
@@ -22,10 +22,10 @@ The matching logic is quite powerful. It supports searching the text of your
 document with different algorithms, and as such, some experimentation may be
 necessary to get things right.
 
-In order to have a tag, correspondent, or type assigned automatically to newly
-consumed documents, assign a match and matching algorithm using the web
-interface. These settings define when to assign correspondents, tags, and types
-to documents.
+In order to have a tag, correspondent, document type, or storage path assigned
+automatically to newly consumed documents, assign a match and matching algorithm
+using the web interface. These settings define when to assign tags, correspondents,
+document types, and storage paths to documents.
 
 The following algorithms are available:
 
@@ -37,7 +37,7 @@ The following algorithms are available:
 * **Literal:** Matches only if the match appears exactly as provided (i.e. preserve ordering) in the PDF.
 * **Regular expression:** Parses the match as a regular expression and tries to
   find a match within the document.
-* **Fuzzy match:** I dont know. Look at the source.
+* **Fuzzy match:** I don't know. Look at the source.
 * **Auto:** Tries to automatically match new documents. This does not require you
   to set a match. See the notes below.
 
@@ -47,9 +47,9 @@ defining a match text of ``"Bank of America" BofA`` using the *any* algorithm,
 will match documents that contain either "Bank of America" or "BofA", but will
 not match documents containing "Bank of South America".
 
-Then just save your tag/correspondent and run another document through the
-consumer.  Once complete, you should see the newly-created document,
-automatically tagged with the appropriate data.
+Then just save your tag, correspondent, document type, or storage path and run
+another document through the consumer.  Once complete, you should see the
+newly-created document, automatically tagged with the appropriate data.
 
 
 .. _advanced-automatic_matching:
@@ -58,9 +58,9 @@ Automatic matching
 ==================
 
 Paperless-ngx comes with a new matching algorithm called *Auto*. This matching
-algorithm tries to assign tags, correspondents, and document types to your
-documents based on how you have already assigned these on existing documents. It
-uses a neural network under the hood.
+algorithm tries to assign tags, correspondents, document types, and storage paths
+to your documents based on how you have already assigned these on existing documents.
+It uses a neural network under the hood.
 
 If, for example, all your bank statements of your account 123 at the Bank of
 America are tagged with the tag "bofa_123" and the matching algorithm of this
@@ -80,20 +80,21 @@ feature:
   that the neural network only learns from documents which you have correctly
   tagged before.
 * The matching algorithm can only work if there is a correlation between the
-  tag, correspondent, or document type and the document itself. Your bank
-  statements usually contain your bank account number and the name of the bank,
-  so this works reasonably well, However, tags such as "TODO" cannot be
-  automatically assigned.
+  tag, correspondent, document type, or storage path and the document itself.
+  Your bank statements usually contain your bank account number and the name
+  of the bank, so this works reasonably well, However, tags such as "TODO"
+  cannot be automatically assigned.
 * The matching algorithm needs a reasonable number of documents to identify when
-  to assign tags, correspondents, and types. If one out of a thousand documents
-  has the correspondent "Very obscure web shop I bought something five years
-  ago", it will probably not assign this correspondent automatically if you buy
-  something from them again. The more documents, the better.
+  to assign tags, correspondents, storage paths, and types. If one out of a
+  thousand documents has the correspondent "Very obscure web shop I bought
+  something five years ago", it will probably not assign this correspondent
+  automatically if you buy something from them again. The more documents, the better.
 * Paperless also needs a reasonable amount of negative examples to decide when
-  not to assign a certain tag, correspondent or type. This will usually be the
-  case as you start filling up paperless with documents. Example: If all your
-  documents are either from "Webshop" and "Bank", paperless will assign one of
-  these correspondents to ANY new document, if both are set to automatic matching.
+  not to assign a certain tag, correspondent, document type, or storage path. This will
+  usually be the case as you start filling up paperless with documents.
+  Example: If all your documents are either from "Webshop" and "Bank", paperless
+  will assign one of these correspondents to ANY new document, if both are set
+  to automatic matching.
 
 Hooking into the consumption process
 ####################################
@@ -120,10 +121,10 @@ Pre-consumption script
 ======================
 
 Executed after the consumer sees a new document in the consumption folder, but
-before any processing of the document is performed. This script receives exactly
-one argument:
+before any processing of the document is performed. This script can access the
+following relevant environment variables set:
 
-* Document file name
+* ``DOCUMENT_SOURCE_PATH``
 
 A simple but common example for this would be creating a simple script like
 this:
@@ -133,7 +134,7 @@ this:
 .. code:: bash
 
     #!/usr/bin/env bash
-    pdf2pdfocr.py -i ${1}
+    pdf2pdfocr.py -i ${DOCUMENT_SOURCE_PATH}
 
 ``/etc/paperless.conf``
 
@@ -156,16 +157,20 @@ Post-consumption script
 =======================
 
 Executed after the consumer has successfully processed a document and has moved it
-into paperless. It receives the following arguments:
+into paperless. It receives the following environment variables:
 
-* Document id
-* Generated file name
-* Source path
-* Thumbnail path
-* Download URL
-* Thumbnail URL
-* Correspondent
-* Tags
+* ``DOCUMENT_ID``
+* ``DOCUMENT_FILE_NAME``
+* ``DOCUMENT_CREATED``
+* ``DOCUMENT_MODIFIED``
+* ``DOCUMENT_ADDED``
+* ``DOCUMENT_SOURCE_PATH``
+* ``DOCUMENT_ARCHIVE_PATH``
+* ``DOCUMENT_THUMBNAIL_PATH``
+* ``DOCUMENT_DOWNLOAD_URL``
+* ``DOCUMENT_THUMBNAIL_URL``
+* ``DOCUMENT_CORRESPONDENT``
+* ``DOCUMENT_TAGS``
 
 The script can be in any language, but for a simple shell script
 example, you can take a look at `post-consumption-example.sh`_ in this project.
@@ -268,6 +273,17 @@ If paperless detects that two documents share the same filename, paperless will
 append ``_01``, ``_02``, etc to the filename. This happens if all the placeholders in a filename
 evaluate to the same value.
 
+.. hint::
+    You can affect how empty placeholders are treated by changing the following setting to
+    `true`.
+
+    .. code::
+
+        PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=True
+
+    Doing this results in all empty placeholders resolving to "" instead of "none" as stated above.
+    Spaces before empty placeholders are removed as well, empty directories are omitted.
+
 .. hint::
 
     Paperless checks the filename of a document whenever it is saved. Therefore,
@@ -290,3 +306,59 @@ evaluate to the same value.
 
     However, keep in mind that inside docker, if files get stored outside of the
     predefined volumes, they will be lost after a restart of paperless.
+
+
+Storage paths
+#############
+
+One of the best things in Paperless is that you can not only access the documents via the
+web interface, but also via the file system.
+
+When as single storage layout is not sufficient for your use case, storage paths come to
+the rescue. Storage paths allow you to configure more precisely where each document is stored
+in the file system.
+
+- Each storage path is a `PAPERLESS_FILENAME_FORMAT` and follows the rules described above
+- Each document is assigned a storage path using the matching algorithms described above, but
+  can be overwritten at any time
+
+For example, you could define the following two storage paths:
+
+1. Normal communications are put into a folder structure sorted by `year/correspondent`
+2. Communications with insurance companies are stored in a flat structure with longer file names,
+   but containing the full date of the correspondence.
+
+.. code::
+
+    By Year = {created_year}/{correspondent}/{title}
+    Insurances = Insurances/{correspondent}/{created_year}-{created_month}-{created_day} {title}
+
+
+If you then map these storage paths to the documents, you might get the following result.
+For simplicity, `By Year` defines the same structure as in the previous example above.
+
+.. code:: text
+
+   2019/                                   # By Year
+      My bank/
+        Statement January.pdf
+        Statement February.pdf
+
+    Insurances/                           # Insurances
+      Healthcare 123/
+        2022-01-01 Statement January.pdf
+        2022-02-02 Letter.pdf
+        2022-02-03 Letter.pdf
+      Dental 456/
+        2021-12-01 New Conditions.pdf
+
+
+.. hint::
+
+    Defining a storage path is optional. If no storage path is defined for a document, the global
+    `PAPERLESS_FILENAME_FORMAT` is applied.
+
+.. caution::
+
+    If you adjust the format of an existing storage path, old documents don't get relocated automatically.
+    You need to run the :ref:`document renamer <utilities-renamer>` to adjust their pathes.

docs/api.rst

@@ -31,7 +31,8 @@ The objects served by the document endpoint contain the following fields:
 *   ``tags``: List of IDs of tags assigned to this document, or empty list.
 *   ``document_type``: Document type of this document, or null.
 *   ``correspondent``:  Correspondent of this document or null.
-*   ``created``: The date at which this document was created.
+*   ``created``: The date time at which this document was created.
+*   ``created_date``: The date (YYYY-MM-DD) at which this document was created. Optional. If also passed with created, this is ignored.
 *   ``modified``: The date at which this document was last edited in paperless. Read-only.
 *   ``added``: The date at which this document was added to paperless. Read-only.
 *   ``archive_serial_number``: The identifier of this document in a physical document archive.
@@ -240,11 +241,13 @@ be instructed to consume the document from there.
 The endpoint supports the following optional form fields:
 
 *   ``title``: Specify a title that the consumer should use for the document.
+*   ``created``: Specify a DateTime where the document was created (e.g. "2016-04-19" or "2016-04-19 06:15:00+02:00").
 *   ``correspondent``: Specify the ID of a correspondent that the consumer should use for the document.
 *   ``document_type``: Similar to correspondent.
 *   ``tags``: Similar to correspondent. Specify this multiple times to have multiple tags added
     to the document.
 
+
 The endpoint will immediately return "OK" if the document consumption process
 was started successfully. No additional status information about the consumption
 process itself is available, since that happens in a different process.

docs/changelog.md

@@ -4,203 +4,203 @@
 
 ### Features
 
-- (chore) Runs pyupgrade to Python 3.8+ @stumpylog (#890)
-- Dockerfile Organization \& Enhancements @stumpylog (#888)
-- mobile friendlier manage pages @shamoon (#873)
-- Use semver for release process @stumpylog (#851)
-- Enable Docker Hub push @stumpylog (#828)
-- Feature barcode tiff support @gador (#766)
-- Updates GHA workflow to rebuild intermediate images on changes @stumpylog (#820)
-- Adds simple Python to wait for Redis broker to be ready @stumpylog (#788)
-- Update GHA workflow to build all Docker images @stumpylog (#761)
+- (chore) Runs pyupgrade to Python 3.8+ [\@stumpylog](https://github.com/stumpylog) ([\#890](https://github.com/paperless-ngx/paperless-ngx/pull/890))
+- Dockerfile Organization \& Enhancements [\@stumpylog](https://github.com/stumpylog) ([\#888](https://github.com/paperless-ngx/paperless-ngx/pull/888))
+- mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873))
+- Use semver for release process [\@stumpylog](https://github.com/stumpylog) ([\#851](https://github.com/paperless-ngx/paperless-ngx/pull/851))
+- Enable Docker Hub push [\@stumpylog](https://github.com/stumpylog) ([\#828](https://github.com/paperless-ngx/paperless-ngx/pull/828))
+- Feature barcode tiff support [\@gador](https://github.com/gador) ([\#766](https://github.com/paperless-ngx/paperless-ngx/pull/766))
+- Updates GHA workflow to rebuild intermediate images on changes [\@stumpylog](https://github.com/stumpylog) ([\#820](https://github.com/paperless-ngx/paperless-ngx/pull/820))
+- Adds simple Python to wait for Redis broker to be ready [\@stumpylog](https://github.com/stumpylog) ([\#788](https://github.com/paperless-ngx/paperless-ngx/pull/788))
+- Update GHA workflow to build all Docker images [\@stumpylog](https://github.com/stumpylog) ([\#761](https://github.com/paperless-ngx/paperless-ngx/pull/761))
 
 ### Bug Fixes
 
-- Feature / fix saved view \& sort field query params @shamoon (#881)
-- mobile friendlier manage pages @shamoon (#873)
-- Add timeout to healthcheck @shamoon (#880)
-- Always accept yyyy-mm-dd date inputs @shamoon (#864)
-- Fix local Docker image building @stumpylog (#849)
-- Fix: show errors on invalid date input @shamoon (#862)
-- Fix: Older dates do not display on frontend @shamoon (#852)
-- Fixes IMAP UTF8 Authenication @stumpylog (#725)
-- Fix password field remains visible @shamoon (#840)
-- Fixes Pillow build for armv7 @stumpylog (#815)
-- Update frontend localization source file @shamoon (#814)
-- Fix install script extra OCR languages format @stumpylog (#777)
+- Feature / fix saved view \& sort field query params [\@shamoon](https://github.com/shamoon) ([\#881](https://github.com/paperless-ngx/paperless-ngx/pull/881))
+- Mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873))
+- Add timeout to healthcheck [\@shamoon](https://github.com/shamoon) ([\#880](https://github.com/paperless-ngx/paperless-ngx/pull/880))
+- Always accept yyyy-mm-dd date inputs [\@shamoon](https://github.com/shamoon) ([\#864](https://github.com/paperless-ngx/paperless-ngx/pull/864))
+- Fix local Docker image building [\@stumpylog](https://github.com/stumpylog) ([\#849](https://github.com/paperless-ngx/paperless-ngx/pull/849))
+- Fix: show errors on invalid date input [\@shamoon](https://github.com/shamoon) ([\#862](https://github.com/paperless-ngx/paperless-ngx/pull/862))
+- Fix: Older dates do not display on frontend [\@shamoon](https://github.com/shamoon) ([\#852](https://github.com/paperless-ngx/paperless-ngx/pull/852))
+- Fixes IMAP UTF8 Authenication [\@stumpylog](https://github.com/stumpylog) ([\#725](https://github.com/paperless-ngx/paperless-ngx/pull/725))
+- Fix password field remains visible [\@shamoon](https://github.com/shamoon) ([\#840](https://github.com/paperless-ngx/paperless-ngx/pull/840))
+- Fixes Pillow build for armv7 [\@stumpylog](https://github.com/stumpylog) ([\#815](https://github.com/paperless-ngx/paperless-ngx/pull/815))
+- Update frontend localization source file [\@shamoon](https://github.com/shamoon) ([\#814](https://github.com/paperless-ngx/paperless-ngx/pull/814))
+- Fix install script extra OCR languages format [\@stumpylog](https://github.com/stumpylog) ([\#777](https://github.com/paperless-ngx/paperless-ngx/pull/777))
 
 ### Documentation
 
-- Use semver for release process @stumpylog (#851)
-- Deployment: Consolidate tika compose files @qcasey (#866)
-- Fix local Docker image building @stumpylog (#849)
+- Use semver for release process [\@stumpylog](https://github.com/stumpylog) ([\#851](https://github.com/paperless-ngx/paperless-ngx/pull/851))
+- Deployment: Consolidate tika compose files [\@qcasey](https://github.com/qcasey) ([\#866](https://github.com/paperless-ngx/paperless-ngx/pull/866))
+- Fix local Docker image building [\@stumpylog](https://github.com/stumpylog) ([\#849](https://github.com/paperless-ngx/paperless-ngx/pull/849))
 
 ### Maintenance
 
-- Dockerfile Organization \& Enhancements @stumpylog (#888)
-- Add timeout to healthcheck @shamoon (#880)
-- Use semver for release process @stumpylog (#851)
-- Deployment: Consolidate tika compose files @qcasey (#866)
-- Fixes Pillow build for armv7 @stumpylog (#815)
-- Update frontend localization source file @shamoon (#814)
-- Fix install script extra OCR languages format @stumpylog (#777)
-- Adds simple Python to wait for Redis broker to be ready @stumpylog (#788)
+- Dockerfile Organization \& Enhancements [\@stumpylog](https://github.com/stumpylog) ([\#888](https://github.com/paperless-ngx/paperless-ngx/pull/888))
+- Add timeout to healthcheck [\@shamoon](https://github.com/shamoon) ([\#880](https://github.com/paperless-ngx/paperless-ngx/pull/880))
+- Use semver for release process [\@stumpylog](https://github.com/stumpylog) ([\#851](https://github.com/paperless-ngx/paperless-ngx/pull/851))
+- Deployment: Consolidate tika compose files [\@qcasey](https://github.com/qcasey) ([\#866](https://github.com/paperless-ngx/paperless-ngx/pull/866))
+- Fixes Pillow build for armv7 [\@stumpylog](https://github.com/stumpylog) ([\#815](https://github.com/paperless-ngx/paperless-ngx/pull/815))
+- Update frontend localization source file [\@shamoon](https://github.com/shamoon) ([\#814](https://github.com/paperless-ngx/paperless-ngx/pull/814))
+- Fix install script extra OCR languages format [\@stumpylog](https://github.com/stumpylog) ([\#777](https://github.com/paperless-ngx/paperless-ngx/pull/777))
+- Adds simple Python to wait for Redis broker to be ready [\@stumpylog](https://github.com/stumpylog) ([\#788](https://github.com/paperless-ngx/paperless-ngx/pull/788))
 
 ### Dependencies
 
 <details>
 <summary>15 changes</summary>
 
-- Bump tj-actions/changed-files from 18.7 to 19 @dependabot (#830)
-- Bump asgiref from 3.5.0 to 3.5.1 @dependabot (#867)
-- Bump jest from 27.5.1 to 28.0.3 in /src-ui @dependabot (#860)
-- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.1.0 to 12.1.1 in /src-ui @dependabot (#861)
-- Bump @<!---->types/node from 17.0.27 to 17.0.29 in /src-ui @dependabot (#833)
-- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.0.2 to 12.1.0 in /src-ui @dependabot (#834)
-- Bump pytest from 7.1.1 to 7.1.2 @dependabot (#806)
-- Bump github/codeql-action from 1 to 2 @dependabot (#792)
-- Bump imap-tools from 0.53.0 to 0.54.0 @dependabot (#758)
-- Bump ocrmypdf from 13.4.2 to 13.4.3 @dependabot (#757)
-- Bump importlib-resources from 5.6.0 to 5.7.1 @dependabot (#756)
-- Bump tox from 3.24.5 to 3.25.0 @dependabot (#692)
-- Bump cypress from 9.5.3 to 9.6.0 in /src-ui @dependabot (#800)
-- Bump angular \& tools to 13.3.4 or 13.3.3 @shamoon (#799)
-- Bump concurrently from 7.0.0 to 7.1.0 in /src-ui @dependabot (#797)
+- Bump tj-actions/changed-files from 18.7 to 19 @dependabot ([\#830](https://github.com/paperless-ngx/paperless-ngx/pull/830))
+- Bump asgiref from 3.5.0 to 3.5.1 @dependabot ([\#867](https://github.com/paperless-ngx/paperless-ngx/pull/867))
+- Bump jest from 27.5.1 to 28.0.3 in /src-ui @dependabot ([\#860](https://github.com/paperless-ngx/paperless-ngx/pull/860))
+- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.1.0 to 12.1.1 in /src-ui @dependabot ([\#861](https://github.com/paperless-ngx/paperless-ngx/pull/861))
+- Bump @<!---->types/node from 17.0.27 to 17.0.29 in /src-ui @dependabot ([\#833](https://github.com/paperless-ngx/paperless-ngx/pull/833))
+- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.0.2 to 12.1.0 in /src-ui @dependabot ([\#834](https://github.com/paperless-ngx/paperless-ngx/pull/834))
+- Bump pytest from 7.1.1 to 7.1.2 @dependabot ([\#806](https://github.com/paperless-ngx/paperless-ngx/pull/806))
+- Bump github/codeql-action from 1 to 2 @dependabot ([\#792](https://github.com/paperless-ngx/paperless-ngx/pull/792))
+- Bump imap-tools from 0.53.0 to 0.54.0 @dependabot ([\#758](https://github.com/paperless-ngx/paperless-ngx/pull/758))
+- Bump ocrmypdf from 13.4.2 to 13.4.3 @dependabot ([\#757](https://github.com/paperless-ngx/paperless-ngx/pull/757))
+- Bump importlib-resources from 5.6.0 to 5.7.1 @dependabot ([\#756](https://github.com/paperless-ngx/paperless-ngx/pull/756))
+- Bump tox from 3.24.5 to 3.25.0 @dependabot ([\#692](https://github.com/paperless-ngx/paperless-ngx/pull/692))
+- Bump cypress from 9.5.3 to 9.6.0 in /src-ui @dependabot ([\#800](https://github.com/paperless-ngx/paperless-ngx/pull/800))
+- Bump angular \& tools to 13.3.4 or 13.3.3 [\@shamoon](https://github.com/shamoon) ([\#799](https://github.com/paperless-ngx/paperless-ngx/pull/799))
+- Bump concurrently from 7.0.0 to 7.1.0 in /src-ui @dependabot ([\#797](https://github.com/paperless-ngx/paperless-ngx/pull/797))
 </details>
 
 ## paperless-ngx 1.7.0
 
-Breaking Changes
+### Breaking Changes
 
 - `PAPERLESS_URL` is now required when using a reverse proxy. See
   [\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674).
 
-Features
+### Features
 
 - Allow setting more than one tag in mail rules
-  [\@jonasc](https://github.com/jonasc) (\#270)
-- global drag\'n\'drop [\@shamoon](https://github.com/shamoon)
-  (\#283).
+  [\@jonasc](https://github.com/jonasc) ([\#270](https://github.com/paperless-ngx/paperless-ngx/pull/270))
+- Global drag\'n\'drop [\@shamoon](https://github.com/shamoon)
+  ([\#283](https://github.com/paperless-ngx/paperless-ngx/pull/283))
 - Fix: download buttons should disable while waiting
-  [\@shamoon](https://github.com/shamoon) (\#630).
-- Update checker [\@shamoon](https://github.com/shamoon) (\#591).
+  [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630))
+- Update checker [\@shamoon](https://github.com/shamoon) ([\#591](https://github.com/paperless-ngx/paperless-ngx/pull/591))
 - Show prompt on password-protected pdfs
-  [\@shamoon](https://github.com/shamoon) (\#564).
+  [\@shamoon](https://github.com/shamoon) ([\#564](https://github.com/paperless-ngx/paperless-ngx/pull/564))
 - Filtering query params aka browser navigation for filtering
-  [\@shamoon](https://github.com/shamoon) (\#540).
+  [\@shamoon](https://github.com/shamoon) ([\#540](https://github.com/paperless-ngx/paperless-ngx/pull/540))
 - Clickable tags in dashboard widgets
-  [\@shamoon](https://github.com/shamoon) (\#515).
+  [\@shamoon](https://github.com/shamoon) ([\#515](https://github.com/paperless-ngx/paperless-ngx/pull/515))
 - Add bottom pagination [\@shamoon](https://github.com/shamoon)
-  (\#372).
+  ([\#372](https://github.com/paperless-ngx/paperless-ngx/pull/372))
 - Feature barcode splitter [\@gador](https://github.com/gador)
-  (\#532).
-- App loading screen [\@shamoon](https://github.com/shamoon) (\#298).
+  ([\#532](https://github.com/paperless-ngx/paperless-ngx/pull/532))
+- App loading screen [\@shamoon](https://github.com/shamoon) ([\#298](https://github.com/paperless-ngx/paperless-ngx/pull/298))
 - Use progress bar for delayed buttons
-  [\@shamoon](https://github.com/shamoon) (\#415).
+  [\@shamoon](https://github.com/shamoon) ([\#415](https://github.com/paperless-ngx/paperless-ngx/pull/415))
 - Add minimum length for documents text filter
-  [\@shamoon](https://github.com/shamoon) (\#401).
+  [\@shamoon](https://github.com/shamoon) ([\#401](https://github.com/paperless-ngx/paperless-ngx/pull/401))
 - Added nav buttons in the document detail view
-  [\@GruberViktor](https://github.com/gruberviktor) (\#273).
+  [\@GruberViktor](https://github.com/gruberviktor) ([\#273](https://github.com/paperless-ngx/paperless-ngx/pull/273))
 - Improve date keyboard input [\@shamoon](https://github.com/shamoon)
-  (\#253).
-- Color theming [\@shamoon](https://github.com/shamoon) (\#243).
+  ([\#253](https://github.com/paperless-ngx/paperless-ngx/pull/253))
+- Color theming [\@shamoon](https://github.com/shamoon) ([\#243](https://github.com/paperless-ngx/paperless-ngx/pull/243))
 - Parse dates when entered without separators
-  [\@GruberViktor](https://github.com/gruberviktor) (\#250).
+  [\@GruberViktor](https://github.com/gruberviktor) ([\#250](https://github.com/paperless-ngx/paperless-ngx/pull/250))
 
-Bug Fixes
+### Bug Fixes
 
-- add \"localhost\" to ALLOWED_HOSTS
-  [\@gador](https://github.com/gador) (\#700).
-- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690).
+- Add \"localhost\" to ALLOWED_HOSTS
+  [\@gador](https://github.com/gador) ([\#700](https://github.com/paperless-ngx/paperless-ngx/pull/700))
+- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690))
 - Adds wait for file before consuming
-  [\@stumpylog](https://github.com/stumpylog) (\#483).
+  [\@stumpylog](https://github.com/stumpylog) ([\#483](https://github.com/paperless-ngx/paperless-ngx/pull/483))
 - Fix: frontend document editing erases time data
-  [\@shamoon](https://github.com/shamoon) (\#654).
+  [\@shamoon](https://github.com/shamoon) ([\#654](https://github.com/paperless-ngx/paperless-ngx/pull/654))
 - Increase length of SavedViewFilterRule
-  [\@stumpylog](https://github.com/stumpylog) (\#612).
+  [\@stumpylog](https://github.com/stumpylog) ([\#612](https://github.com/paperless-ngx/paperless-ngx/pull/612))
 - Fixes attachment filename matching during mail fetching
-  [\@stumpylog](https://github.com/stumpylog) (\#680).
+  [\@stumpylog](https://github.com/stumpylog) ([\#680](https://github.com/paperless-ngx/paperless-ngx/pull/680))
 - Add `PAPERLESS_URL` env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/discussions/674))
 - Fix: download buttons should disable while waiting
-  [\@shamoon](https://github.com/shamoon) (\#630).
+  [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630))
 - Fixes downloaded filename, add more consumer ignore settings
-  [\@stumpylog](https://github.com/stumpylog) (\#599).
+  [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599))
 - FIX BUG: case-sensitive matching was not possible
-  [\@danielBreitlauch](https://github.com/danielbreitlauch) (\#594).
-- uses shutil.move instead of rename
-  [\@gador](https://github.com/gador) (\#617).
+  [\@danielBreitlauch](https://github.com/danielbreitlauch) ([\#594](https://github.com/paperless-ngx/paperless-ngx/pull/594))
+- Uses shutil.move instead of rename
+  [\@gador](https://github.com/gador) ([\#617](https://github.com/paperless-ngx/paperless-ngx/pull/617))
 - Fix npm deps 01.02.22 2 [\@shamoon](https://github.com/shamoon)
-  (\#610).
+  ([\#610](https://github.com/paperless-ngx/paperless-ngx/discussions/610))
 - Fix npm dependencies 01.02.22
-  [\@shamoon](https://github.com/shamoon) (\#600).
-- fix issue 416: implement PAPERLESS_OCR_MAX_IMAGE_PIXELS
-  [\@hacker-h](https://github.com/hacker-h) (\#441).
-- fix: exclude cypress from build in Dockerfile
-  [\@FrankStrieter](https://github.com/FrankStrieter) (\#526).
+  [\@shamoon](https://github.com/shamoon) ([\#600](https://github.com/paperless-ngx/paperless-ngx/pull/600))
+- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
+  [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441))
+- Fix: exclude cypress from build in Dockerfile
+  [\@FrankStrieter](https://github.com/FrankStrieter) ([\#526](https://github.com/paperless-ngx/paperless-ngx/pull/526))
 - Corrections to pass pre-commit hooks
-  [\@schnuffle](https://github.com/schnuffle) (\#454).
+  [\@schnuffle](https://github.com/schnuffle) ([\#454](https://github.com/paperless-ngx/paperless-ngx/pull/454))
 - Fix 311 unable to click checkboxes in document list
-  [\@shamoon](https://github.com/shamoon) (\#313).
+  [\@shamoon](https://github.com/shamoon) ([\#313](https://github.com/paperless-ngx/paperless-ngx/pull/313))
 - Fix imap tools bug [\@stumpylog](https://github.com/stumpylog)
-  (\#393).
+  ([\#393](https://github.com/paperless-ngx/paperless-ngx/pull/393))
 - Fix filterable dropdown buttons arent translated
-  [\@shamoon](https://github.com/shamoon) (\#366).
+  [\@shamoon](https://github.com/shamoon) ([\#366](https://github.com/paperless-ngx/paperless-ngx/pull/366))
 - Fix 224: \"Auto-detected date is day before receipt date\"
-  [\@a17t](https://github.com/a17t) (\#246).
+  [\@a17t](https://github.com/a17t) ([\#246](https://github.com/paperless-ngx/paperless-ngx/pull/246))
 - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon)
-  (\#322).
+  ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322))
 - Fix page links hidden [\@shamoon](https://github.com/shamoon)
-  (\#314).
+  ([\#314](https://github.com/paperless-ngx/paperless-ngx/pull/314))
 - Fix: Include excluded items in dropdown count
-  [\@shamoon](https://github.com/shamoon) (\#263).
+  [\@shamoon](https://github.com/shamoon) ([\#263](https://github.com/paperless-ngx/paperless-ngx/pull/263))
 
-Translation
+### Translation
 
 - [\@miku323](https://github.com/miku323) contributed to Slovenian
-  translation.
+  translation
 - [\@FaintGhost](https://github.com/FaintGhost) contributed to Chinese
-  Simplified translation.
+  Simplified translation
 - [\@DarkoBG79](https://github.com/DarkoBG79) contributed to Serbian
-  translation.
+  translation
 - [Kemal Secer](https://crowdin.com/profile/kemal.secer) contributed
-  to Turkish translation.
+  to Turkish translation
 - [\@Prominence](https://github.com/Prominence) contributed to
-  Belarusian translation.
+  Belarusian translation
 
-Documentation
+### Documentation
 
-- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690).
-- Add [PAPERLESS\_URL]{.title-ref} env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690))
+- Add `PAPERLESS_URL` env variable & CSRF var
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674))
 - Fixes downloaded filename, add more consumer ignore settings
-  [\@stumpylog](https://github.com/stumpylog) (\#599).
-- fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
-  [\@hacker-h](https://github.com/hacker-h) (\#441).
+  [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599))
+- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
+  [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441))
 - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon)
-  (\#322).
+  ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322))
 
-Maintenance
+### Maintenance
 
 - Add `PAPERLESS_URL` env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674))
 - Chore: Implement release-drafter action for Changelogs
-  [\@qcasey](https://github.com/qcasey) (\#669).
-- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) (\#667).
+  [\@qcasey](https://github.com/qcasey) ([\#669](https://github.com/paperless-ngx/paperless-ngx/pull/669))
+- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) ([\#667](https://github.com/paperless-ngx/paperless-ngx/pull/667))
 - Support docker-compose v2 in install
-  [\@stumpylog](https://github.com/stumpylog) (\#611).
+  [\@stumpylog](https://github.com/stumpylog) ([\#611](https://github.com/paperless-ngx/paperless-ngx/pull/611))
 - Add Belarusian localization [\@shamoon](https://github.com/shamoon)
-  (\#588).
+  ([\#588](https://github.com/paperless-ngx/paperless-ngx/pull/588))
 - Add Turkish localization [\@shamoon](https://github.com/shamoon)
-  (\#536).
+  ([\#536](https://github.com/paperless-ngx/paperless-ngx/pull/536))
 - Add Serbian localization [\@shamoon](https://github.com/shamoon)
-  (\#504).
+  ([\#504](https://github.com/paperless-ngx/paperless-ngx/pull/504))
 - Create PULL_REQUEST_TEMPLATE.md
-  [\@shamoon](https://github.com/shamoon) (\#304).
+  [\@shamoon](https://github.com/shamoon) ([\#304](https://github.com/paperless-ngx/paperless-ngx/pull/304))
 - Add Chinese localization [\@shamoon](https://github.com/shamoon)
-  (\#247).
+  ([\#247](https://github.com/paperless-ngx/paperless-ngx/pull/247))
 - Add Slovenian language for frontend
-  [\@shamoon](https://github.com/shamoon) (\#315).
+  [\@shamoon](https://github.com/shamoon) ([\#315](https://github.com/paperless-ngx/paperless-ngx/pull/315))
 
 ## paperless-ngx 1.6.0
 
@@ -216,46 +216,46 @@ include:
 - Updated Python and Angular dependencies.
 - Dropped support for Python 3.7.
 - Dropped support for Ansible playbooks (thanks
-  [\@slankes](https://github.com/slankes) \#109). If someone would
-  like to continue supporting them, please see the [ansible
+  [\@slankes](https://github.com/slankes) [\#109](https://github.com/paperless-ngx/paperless-ngx/pull/109)). If someone would
+  like to continue supporting them, please see our [ansible
   repo](https://github.com/paperless-ngx/paperless-ngx-ansible).
 - Python code is now required to use Black formatting (thanks
-  [\@kpj](https://github.com/kpj) \#168).
+  [\@kpj](https://github.com/kpj) [\#168](https://github.com/paperless-ngx/paperless-ngx/pull/168)).
 - [\@tribut](https://github.com/tribut) added support for a custom SSO
-  logout redirect (jonaswinkler\#1258). See
+  logout redirect ([jonaswinkler\#1258](https://github.com/jonaswinkler/paperless-ng/pull/1258)). See
   `PAPERLESS_LOGOUT_REDIRECT_URL`.
 - [\@shamoon](https://github.com/shamoon) added a loading indicator
-  when document list is reloading (jonaswinkler\#1297).
+  when document list is reloading ([jonaswinkler\#1297](https://github.com/jonaswinkler/paperless-ng/pull/1297)).
 - [\@shamoon](https://github.com/shamoon) improved the PDF viewer on
-  mobile (\#2).
+  mobile ([\#2](https://github.com/paperless-ngx/paperless-ngx/pull/2)).
 - [\@shamoon](https://github.com/shamoon) added \'any\' / \'all\' and
-  \'not\' filtering with tags (\#10).
+  \'not\' filtering with tags ([\#10](https://github.com/paperless-ngx/paperless-ngx/pull/10)).
 - [\@shamoon](https://github.com/shamoon) added warnings for unsaved
-  changes, with smart edit buttons (\#13).
+  changes, with smart edit buttons ([\#13](https://github.com/paperless-ngx/paperless-ngx/pull/13)).
 - [\@benjaminfrank](https://github.com/benjaminfrank) enabled a
-  non-root access to port 80 via systemd (\#18).
+  non-root access to port 80 via systemd ([\#18](https://github.com/paperless-ngx/paperless-ngx/pull/18)).
 - [\@tribut](https://github.com/tribut) added simple \"delete to
-  trash\" functionality (\#24). See `PAPERLESS_TRASH_DIR`.
+  trash\" functionality ([\#24](https://github.com/paperless-ngx/paperless-ngx/pull/24)). See `PAPERLESS_TRASH_DIR`.
 - [\@amenk](https://github.com/amenk) fixed the search box overlay
-  menu on mobile (\#32).
+  menu on mobile ([\#32](https://github.com/paperless-ngx/paperless-ngx/pull/32)).
 - [\@dblitt](https://github.com/dblitt) updated the login form to not
-  auto-capitalize usernames (\#36).
+  auto-capitalize usernames ([\#36](https://github.com/paperless-ngx/paperless-ngx/pull/36)).
 - [\@evilsidekick293](https://github.com/evilsidekick293) made the
-  worker timeout configurable (\#37). See `PAPERLESS_WORKER_TIMEOUT`.
+  worker timeout configurable ([\#37](https://github.com/paperless-ngx/paperless-ngx/pull/37)). See `PAPERLESS_WORKER_TIMEOUT`.
 - [\@Nicarim](https://github.com/Nicarim) fixed downloads of UTF-8
-  formatted documents in Firefox (\#56).
+  formatted documents in Firefox ([\#56](https://github.com/paperless-ngx/paperless-ngx/pull/56)).
 - [\@mweimerskirch](https://github.com/mweimerskirch) sorted the
-  language dropdown by locale (\#78).
+  language dropdown by locale ([\#78](https://github.com/paperless-ngx/paperless-ngx/issues/78)).
 - [\@mweimerskirch](https://github.com/mweimerskirch) enabled the
-  Czech (\#83) and Danish (\#84) translations.
+  Czech ([\#83](https://github.com/paperless-ngx/paperless-ngx/pull/83)) and Danish ([\#84](https://github.com/paperless-ngx/paperless-ngx/pull/84)) translations.
 - [\@cschmatzler](https://github.com/cschmatzler) enabled specifying
-  the webserver port (\#124). See `PAPERLESS_PORT`.
+  the webserver port ([\#124](https://github.com/paperless-ngx/paperless-ngx/pull/124)). See `PAPERLESS_PORT`.
 - [\@muellermartin](https://github.com/muellermartin) fixed an error
-  when uploading transparent PNGs (\#133).
+  when uploading transparent PNGs ([\#133](https://github.com/paperless-ngx/paperless-ngx/pull/133)).
 - [\@shamoon](https://github.com/shamoon) created a slick new logo
-  (\#165).
+  ([\#165](https://github.com/paperless-ngx/paperless-ngx/pull/165)).
 - [\@tim-vogel](https://github.com/tim-vogel) fixed exports missing
-  groups (\#193).
+  groups ([\#193](https://github.com/paperless-ngx/paperless-ngx/pull/193)).
 
 Known issues:
 

docs/configuration.rst

@@ -31,7 +31,7 @@ PAPERLESS_REDIS=<url>
 
 PAPERLESS_DBHOST=<hostname>
     By default, sqlite is used as the database backend. This can be changed here.
-    Set PAPERLESS_DBHOST and PostgreSQL will be used instead of mysql.
+    Set PAPERLESS_DBHOST and PostgreSQL will be used instead of sqlite.
 
 PAPERLESS_DBPORT=<port>
     Adjust port if necessary.
@@ -60,6 +60,13 @@ PAPERLESS_DBSSLMODE=<mode>
 
     Default is ``prefer``.
 
+PAPERLESS_DB_TIMEOUT=<float>
+    Amount of time for a database connection to wait for the database to unlock.
+    Mostly applicable for an sqlite based installation, consider changing to postgresql
+    if you need to increase this.
+
+    Defaults to unset, keeping the Django defaults.
+
 Paths and folders
 #################
 
@@ -111,6 +118,14 @@ PAPERLESS_FILENAME_FORMAT=<format>
 
     Default is none, which disables this feature.
 
+PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=<bool>
+    Tells paperless to replace placeholders in `PAPERLESS_FILENAME_FORMAT` that would resolve
+    to 'none' to be omitted from the resulting filename. This also holds true for directory
+    names.
+    See :ref:`advanced-file_name_handling` for details.
+
+    Defaults to `false` which disables this feature.
+
 PAPERLESS_LOGGING_DIR=<path>
     This is where paperless will store log files.
 
@@ -416,14 +431,23 @@ PAPERLESS_OCR_IMAGE_DPI=<num>
     the produced PDF documents are A4 sized.
 
 PAPERLESS_OCR_MAX_IMAGE_PIXELS=<num>
-    Paperless will not OCR images that have more pixels than this limit.
-    This is intended to prevent decompression bombs from overloading paperless.
-    Increasing this limit is desired if you face a DecompressionBombError despite
-    the concerning file not being malicious; this could e.g. be caused by invalidly
-    recognized metadata.
-    If you have enough resources or if you are certain that your uploaded files
-    are not malicious you can increase this value to your needs.
-    The default value is 256000000, an image with more pixels than that would not be parsed.
+    Paperless will raise a warning when OCRing images which are over this limit and
+    will not OCR images which are more than twice this limit.  Note this does not
+    prevent the document from being consumed, but could result in missing text content.
+
+    If unset, will default to the value determined by
+    `Pillow <https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.MAX_IMAGE_PIXELS>`_.
+
+    .. note::
+
+        Increasing this limit could cause Paperless to consume additional resources
+        when consuming a file.  Be sure you have sufficient system resources.
+
+    .. caution::
+
+        The limit is intended to prevent malicious files from consuming system resources
+        and causing crashes and other errors.  Only increase this value if you are certain
+        your documents are not malicious and you need the text which was not OCRed
 
 PAPERLESS_OCR_USER_ARGS=<json>
     OCRmyPDF offers many more options. Use this parameter to specify any
@@ -519,6 +543,8 @@ PAPERLESS_TASK_WORKERS=<num>
     maintain the automatic matching algorithm, check emails, consume documents,
     etc. This variable specifies how many things it will do in parallel.
 
+    Defaults to 1
+
 
 PAPERLESS_THREADS_PER_WORKER=<num>
     Furthermore, paperless uses multiple threads when consuming documents to
@@ -590,6 +616,28 @@ PAPERLESS_CONSUMER_POLLING=<num>
 
     Defaults to 0, which disables polling and uses filesystem notifications.
 
+PAPERLESS_CONSUMER_POLLING_RETRY_COUNT=<num>
+    If consumer polling is enabled, sets the number of times paperless will check for a
+    file to remain unmodified.
+
+    Defaults to 5.
+
+PAPERLESS_CONSUMER_POLLING_DELAY=<num>
+    If consumer polling is enabled, sets the delay in seconds between each check (above) paperless
+    will do while waiting for a file to remain unmodified.
+
+    Defaults to 5.
+
+.. _configuration-inotify:
+
+PAPERLESS_CONSUMER_INOTIFY_DELAY=<num>
+    Sets the time in seconds the consumer will wait for additional events
+    from inotify before the consumer will consider a file ready and begin consumption.
+    Certain scanners or network setups may generate multiple events for a single file,
+    leading to multiple consumers working on the same file.  Configure this to
+    prevent that.
+
+    Defaults to 0.5 seconds.
 
 PAPERLESS_CONSUMER_DELETE_DUPLICATES=<bool>
     When the consumer detects a duplicate document, it will not touch the
@@ -650,7 +698,6 @@ PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
 
   Defaults to "PATCHT"
 
-
 PAPERLESS_CONVERT_MEMORY_LIMIT=<num>
     On smaller systems, or even in the case of Very Large Documents, the consumer
     may explode, complaining about how it's "unable to extend pixel cache".  In
@@ -674,13 +721,6 @@ PAPERLESS_CONVERT_TMPDIR=<path>
 
     Default is none, which disables the temporary directory.
 
-PAPERLESS_OPTIMIZE_THUMBNAILS=<bool>
-    Use optipng to optimize thumbnails. This usually reduces the size of
-    thumbnails by about 20%, but uses considerable compute time during
-    consumption.
-
-    Defaults to true.
-
 PAPERLESS_POST_CONSUME_SCRIPT=<filename>
     After a document is consumed, Paperless can trigger an arbitrary script if
     you like.  This script will be passed a number of arguments for you to work
@@ -696,6 +736,9 @@ PAPERLESS_FILENAME_DATE_ORDER=<format>
     The filename will be checked first, and if nothing is found, the document
     text will be checked as normal.
 
+    A date in a filename must have some separators (`.`, `-`, `/`, etc)
+    for it to be parsed.
+
     Defaults to none, which disables this feature.
 
 PAPERLESS_THUMBNAIL_FONT_NAME=<filename>
@@ -713,10 +756,7 @@ PAPERLESS_IGNORE_DATES=<string>
     this process. This is useful for special dates (like date of birth) that appear
     in documents regularly but are very unlikely to be the documents creation date.
 
-    You may specify dates in a multitude of formats supported by dateparser (see
-    https://dateparser.readthedocs.io/en/latest/#popular-formats) but as the dates
-    need to be comma separated, the options are limited.
-    Example: "2020-12-02,22.04.1999"
+    The date is parsed using the order specified in PAPERLESS_DATE_ORDER
 
     Defaults to an empty string to not ignore any dates.
 
@@ -751,9 +791,6 @@ PAPERLESS_CONVERT_BINARY=<path>
 PAPERLESS_GS_BINARY=<path>
     Defaults to "/usr/bin/gs".
 
-PAPERLESS_OPTIPNG_BINARY=<path>
-    Defaults to "/usr/bin/optipng".
-
 
 .. _configuration-docker:
 
@@ -769,9 +806,7 @@ PAPERLESS_WEBSERVER_WORKERS=<num>
     also loads the entire application into memory separately, so increasing this value
     will increase RAM usage.
 
-    Consider configuring this to 1 on low power devices with limited amount of RAM.
-
-    Defaults to 2.
+    Defaults to 1.
 
 PAPERLESS_PORT=<port>
     The port number the webserver will listen on inside the container. There are

docs/requirements.txt

@@ -0,0 +1 @@
+myst-parser==0.17.2

docs/scanners.rst

@@ -13,43 +13,45 @@ that works right for you based on recommendations from other Paperless users.
 Physical scanners
 =================
 
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brand   | Model          | Supports                                 | Recommended By |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-|         |                | FTP | SFTP | NFS | SMB | SMTP | API [1]_ |                |
-+=========+================+=====+======+=====+=====+======+==========+================+
-| Brother | `ADS-1700W`_   | yes |      |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-1600W`_   | yes |      |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-1500W`_   | yes |      |     | yes | yes  |          |`danielquinn`_  |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-1100W`_   | yes |      |     |     |      |          |`ytzelf`_       |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-2800W`_   | yes | yes  |     | yes | yes  |          |`philpagel`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-J6930DW`_ | yes |      |     |     |      |          |`ayounggun`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-L5850DW`_ | yes |      |     |     | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-L2750DW`_ | yes |      |     | yes | yes  |          |`muued`_        |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-J5910DW`_ | yes |      |     |     |      |          |`bmsleight`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-8950DW`_  | yes |      |     | yes | yes  |          |`philpagel`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-9142CDN`_ | yes |      |     | yes |      |          |`REOLDEV`_      |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Fujitsu | `ix500`_       | yes |      |     | yes |      |          |`eonist`_       |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Epson   | `ES-580W`_     | yes |      |     | yes | yes  |          |`fignew`_       |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Epson   | `WF-7710DWF`_  | yes |      |     | yes |      |          |`Skylinar`_     |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Fujitsu | `S1300i`_      | yes |      |     | yes |      |          |`jonaswinkler`_ |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Doxie   | `Q2`_          |     |      |     |     |      | yes      |`Unkn0wnCat`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brand   | Model             | Supports                                      | Recommended By |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+|         |                   | FTP | SFTP | NFS | SMB      | SMTP | API [1]_ |                |
++=========+===================+=====+======+=====+==========+======+==========+================+
+| Brother | `ADS-1700W`_      | yes | yes  |     | yes      | yes  |          |`holzhannes`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-1600W`_      | yes |      |     | yes      | yes  |          |`holzhannes`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-1500W`_      | yes |      |     | yes      | yes  |          |`danielquinn`_  |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-1100W`_      | yes |      |     |          |      |          |`ytzelf`_       |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-2800W`_      | yes | yes  |     | yes      | yes  |          |`philpagel`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-J6930DW`_    | yes |      |     |          |      |          |`ayounggun`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-L5850DW`_    | yes |      |     |          | yes  |          |`holzhannes`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-L2750DW`_    | yes |      |     | yes      | yes  |          |`muued`_        |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-J5910DW`_    | yes |      |     |          |      |          |`bmsleight`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-8950DW`_     | yes |      |     | yes      | yes  |          |`philpagel`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-9142CDN`_    | yes |      |     | yes      |      |          |`REOLDEV`_      |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Canon   | `Maxify MB 5350`_ |     |      |     | yes [2]_ | yes  |          |`eingemaischt`_ |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Fujitsu | `ix500`_          | yes |      |     | yes      |      |          |`eonist`_       |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Epson   | `ES-580W`_        | yes |      |     | yes      | yes  |          |`fignew`_       |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Epson   | `WF-7710DWF`_     | yes |      |     | yes      |      |          |`Skylinar`_     |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Fujitsu | `S1300i`_         | yes |      |     | yes      |      |          |`jonaswinkler`_ |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Doxie   | `Q2`_             |     |      |     |          |      | yes      |`Unkn0wnCat`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
 
 .. _MFC-L5850DW: https://www.brother-usa.com/products/mfcl5850dw
 .. _MFC-L2750DW: https://www.brother.de/drucker/laserdrucker/mfc-l2750dw
@@ -58,6 +60,7 @@ Physical scanners
 .. _ADS-1500W: https://www.brother.ca/en/p/ads1500w
 .. _ADS-1100W: https://support.brother.com/g/b/downloadtop.aspx?c=fr&lang=fr&prod=ads1100w_eu_as_cn
 .. _ADS-2800W: https://www.brother-usa.com/products/ads2800w
+.. _Maxify MB 5350: https://www.canon.de/printers/inkjet/maxify/maxify_mb5350/specification.html
 .. _MFC-J6930DW: https://www.brother.ca/en/p/MFCJ6930DW
 .. _MFC-J5910DW: https://www.brother.co.uk/printers/inkjet-printers/mfcj5910dw
 .. _MFC-8950DW: https://www.brother-usa.com/products/mfc8950dw
@@ -81,8 +84,11 @@ Physical scanners
 .. _Unkn0wnCat: https://github.com/Unkn0wnCat
 .. _muued: https://github.com/muued
 .. _philpagel: https://github.com/philpagel
+.. _eingemaischt: https://github.com/eingemaischt
 
 .. [1] Scanners with API Integration allow to push scanned documents directly to :ref:`Paperless API <api-file_uploads>`, sometimes referred to as Webhook or Document POST.
+.. [2] Canon Multi Function Printers show strange behavior over SMB. They close and reopen the file after every page. It's recommended to tune the
+       :ref:`polling <configuration-polling>` and :ref:`inotify <configuration-inotify>` configuration values for your scanner. The scanner timeout is 3 minutes, so ``180`` is a good starting point.
 
 Mobile phone software
 =====================
@@ -105,6 +111,9 @@ You can use your phone to "scan" documents. The regular camera app will work, bu
 
 On Android, you can use these applications in combination with one of the :ref:`Paperless-ngx compatible apps <usage-mobile_upload>` to "Share" the documents produced by these scanner apps with paperless. On iOS, you can share the scanned documents via iOS-Sharing to other mail, WebDav or FTP apps.
 
+There is also an iOS Shortcut that allows you to directly upload text, PDF and image documents available here: https://www.icloud.com/shortcuts/d234abc0885040129d9d75fa45fe1154
+Please note this only works for documents downloaded to iCloud / the device, in other words not directly from a URL.
+
 .. _Office Lens: https://play.google.com/store/apps/details?id=com.microsoft.office.officelens
 .. _Genius Scan: https://play.google.com/store/apps/details?id=com.thegrizzlylabs.geniusscan.free
 .. _OCR Scanner - QuickScan: https://apps.apple.com/us/app/quickscan-scanner-text-ocr/id1513790291

docs/setup.rst

@@ -184,6 +184,25 @@ Install Paperless from Docker Hub
     port 8000. Modifying the part before the colon will map requests on another
     port to the webserver running on the default port.
 
+    **Rootless**
+
+    If you want to run Paperless as a rootless container, you will need to do the
+    following in your ``docker-compose.yml``:
+
+    - set the ``user`` running the container to map to the ``paperless`` user in the
+      container.
+      This value (``user_id`` below), should be the same id that ``USERMAP_UID`` and
+      ``USERMAP_GID`` are set to in the next step.
+      See ``USERMAP_UID`` and ``USERMAP_GID`` :ref:`here <configuration-docker>`.
+
+    Your entry for Paperless should contain something like:
+
+     .. code::
+
+        webserver:
+          image: ghcr.io/paperless-ngx/paperless-ngx:latest
+          user: <user_id>
+
 5.  Modify ``docker-compose.env``, following the comments in the file. The
     most important change is to set ``USERMAP_UID`` and ``USERMAP_GID``
     to the uid and gid of your user on the host system. Use ``id -u`` and
@@ -200,6 +219,19 @@ Install Paperless from Docker Hub
         You can copy any setting from the file ``paperless.conf.example`` and paste it here.
         Have a look at :ref:`configuration` to see what's available.
 
+    .. note::
+
+        You can utilize Docker secrets for some configuration settings by
+        appending `_FILE` to some configuration values.  This is supported currently
+        only by:
+          * PAPERLESS_DBUSER
+          * PAPERLESS_DBPASS
+          * PAPERLESS_SECRET_KEY
+          * PAPERLESS_AUTO_LOGIN_USERNAME
+          * PAPERLESS_ADMIN_USER
+          * PAPERLESS_ADMIN_MAIL
+          * PAPERLESS_ADMIN_PASSWORD
+
     .. caution::
 
         Some file systems such as NFS network shares don't support file system
@@ -286,7 +318,6 @@ writing. Windows is not and will never be supported.
 
     *   ``fonts-liberation`` for generating thumbnails for plain text files
     *   ``imagemagick`` >= 6 for PDF conversion
-    *   ``optipng`` for optimizing thumbnails
     *   ``gnupg`` for handling encrypted documents
     *   ``libpq-dev`` for PostgreSQL
     *   ``libmagic-dev`` for mime type detection
@@ -298,7 +329,7 @@ writing. Windows is not and will never be supported.
 
     .. code::
 
-        python3 python3-pip python3-dev imagemagick fonts-liberation optipng gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils
+        python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils
 
     These dependencies are required for OCRmyPDF, which is used for text recognition.
 
@@ -308,7 +339,7 @@ writing. Windows is not and will never be supported.
     *   ``qpdf``
     *   ``liblept5``
     *   ``libxml2``
-    *   ``pngquant``
+    *   ``pngquant`` (suggested for certain PDF image optimizations)
     *   ``zlib1g``
     *   ``tesseract-ocr`` >= 4.0.0 for OCR
     *   ``tesseract-ocr`` language packs (``tesseract-ocr-eng``, ``tesseract-ocr-deu``, etc)
@@ -332,6 +363,12 @@ writing. Windows is not and will never be supported.
 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
     to use PostgreSQL, SQLite is available as well.
 
+    .. note::
+
+        On bare-metal installations using SQLite, ensure the
+        `JSON1 extension <https://code.djangoproject.com/wiki/JSON1Extension>`_ is enabled. This is
+        usually the case, but not always.
+
 4.  Get the release archive from `<https://github.com/paperless-ngx/paperless-ngx/releases>`_.
     If you clone the git repo as it is, you also have to compile the front end by yourself.
     Extract the archive to a place from where you wish to execute it, such as ``/opt/paperless``.
@@ -724,8 +761,6 @@ configuring some options in paperless can help improve performance immensely:
 *   If you want to perform OCR on the device, consider using ``PAPERLESS_OCR_CLEAN=none``.
     This will speed up OCR times and use less memory at the expense of slightly worse
     OCR results.
-*   Set ``PAPERLESS_OPTIMIZE_THUMBNAILS`` to 'false' if you want faster consumption
-    times. Thumbnails will be about 20% larger.
 *   If using docker, consider setting ``PAPERLESS_WEBSERVER_WORKERS`` to
     1. This will save some memory.
 

docs/troubleshooting.rst

@@ -235,3 +235,85 @@ You might find messages like these in your log files:
 This indicates that paperless failed to read PDF metadata from one of your documents. This happens when you
 open the affected documents in paperless for editing. Paperless will continue to work, and will simply not
 show the invalid metadata.
+
+Consumer fails with a FileNotFoundError
+#######################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.consumer] Error while consuming document SCN_0001.pdf: FileNotFoundError: [Errno 2] No such file or directory: '/tmp/ocrmypdf.io.yhk3zbv0/origin.pdf'
+    Traceback (most recent call last):
+      File "/app/paperless/src/paperless_tesseract/parsers.py", line 261, in parse
+        ocrmypdf.ocr(**args)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/api.py", line 337, in ocr
+        return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 385, in run_pipeline
+        exec_concurrent(context, executor)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 302, in exec_concurrent
+        pdf = post_process(pdf, context, executor)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 235, in post_process
+        pdf_out = metadata_fixup(pdf_out, context)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_pipeline.py", line 798, in metadata_fixup
+        with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
+      File "/usr/local/lib/python3.8/dist-packages/pikepdf/_methods.py", line 923, in open
+        pdf = Pdf._open(
+    FileNotFoundError: [Errno 2] No such file or directory: '/tmp/ocrmypdf.io.yhk3zbv0/origin.pdf'
+
+This probably indicates paperless tried to consume the same file twice.  This can happen for a number of reasons,
+depending on how documents are placed into the consume folder.  If paperless is using inotify (the default) to
+check for documents, try adjusting the :ref:`inotify configuration <configuration-inotify>`.  If polling is enabled,
+try adjusting the :ref:`polling configuration <configuration-polling>`.
+
+Consumer fails waiting for file to remain unmodified.
+#####################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.management.consumer] Timeout while waiting on file /usr/src/paperless/src/../consume/SCN_0001.pdf to remain unmodified.
+
+This indicates paperless timed out while waiting for the file to be completely written to the consume folder.
+Adjusting :ref:`polling configuration <configuration-polling>` values should resolve the issue.
+
+.. note::
+
+    The user will need to manually move the file out of the consume folder and
+    back in, for the initial failing file to be consumed.
+
+Consumer fails reporting "OS reports file as busy still".
+#########################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [WARNING] [paperless.management.consumer] Not consuming file /usr/src/paperless/src/../consume/SCN_0001.pdf: OS reports file as busy still
+
+This indicates paperless was unable to open the file, as the OS reported the file as still being in use.  To prevent a
+crash, paperless did not try to consume the file.  If paperless is using inotify (the default) to
+check for documents, try adjusting the :ref:`inotify configuration <configuration-inotify>`.  If polling is enabled,
+try adjusting the :ref:`polling configuration <configuration-polling>`.
+
+.. note::
+
+    The user will need to manually move the file out of the consume folder and
+    back in, for the initial failing file to be consumed.
+
+Log reports "Creating PaperlessTask failed".
+#########################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.management.consumer] Creating PaperlessTask failed: db locked
+
+You are likely using an sqlite based installation, with an increased number of workers and are running into sqlite's concurrency limitations.
+Uploading or consuming multiple files at once results in many workers attempting to access the database simultaneously.
+
+Consider changing to the PostgreSQL database if you will be processing many documents at once often.  Otherwise,
+try tweaking the ``PAPERLESS_DB_TIMEOUT`` setting to allow more time for the database to unlock.  This may have
+minor performance implications.

docs/usage_overview.rst

@@ -161,6 +161,9 @@ These are as follows:
     will not consume flagged mails.
 *   **Move to folder:** Moves consumed mails out of the way so that paperless wont
     consume them again.
+*   **Add custom Tag:** Adds a custom tag to mails with consumed documents (the IMAP
+    standard calls these "keywords"). Paperless will not consume mails already tagged.
+    Not all mail servers support this feature!
 
 .. caution::
 

gunicorn.conf.py

@@ -1,7 +1,7 @@
 import os
 
-bind = f'0.0.0.0:{os.getenv("PAPERLESS_PORT", 8000)}'
-workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 2))
+bind = f'[::]:{os.getenv("PAPERLESS_PORT", 8000)}'
+workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 1))
 worker_class = "paperless.workers.ConfigurableWorker"
 timeout = 120
 

paperless.conf.example

@@ -23,6 +23,7 @@
 #PAPERLESS_MEDIA_ROOT=../media
 #PAPERLESS_STATICDIR=../static
 #PAPERLESS_FILENAME_FORMAT=
+#PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=
 
 # Security and hosting
 
@@ -64,7 +65,6 @@
 #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
 #PAPERLESS_CONSUMER_ENABLE_BARCODES=false
 #PAPERLESS_CONSUMER_ENABLE_BARCODES=PATCHT
-#PAPERLESS_OPTIMIZE_THUMBNAILS=true
 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_FILENAME_DATE_ORDER=YMD
@@ -83,4 +83,3 @@
 
 #PAPERLESS_CONVERT_BINARY=/usr/bin/convert
 #PAPERLESS_GS_BINARY=/usr/bin/gs
-#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng

requirements.txt

@@ -1,43 +1,37 @@
-#
-# These requirements were autogenerated by pipenv
-# To regenerate from the project's Pipfile, run:
-#
-#    pipenv lock --requirements
-#
-
 -i https://pypi.python.org/simple
 --extra-index-url https://www.piwheels.org/simple
 aioredis==1.3.1
-anyio==3.5.0; python_full_version >= '3.6.2'
+anyio==3.6.1; python_full_version >= '3.6.2'
 arrow==1.2.2; python_version >= '3.6'
-asgiref==3.5.1; python_version >= '3.7'
+asgiref==3.5.2; python_version >= '3.7'
 async-timeout==4.0.2; python_version >= '3.6'
 attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-autobahn==22.3.2; python_version >= '3.7'
+autobahn==22.6.1; python_version >= '3.7'
 automat==20.2.0
 backports.zoneinfo==0.2.1; python_version < '3.9'
 blessed==1.19.1; python_version >= '2.7'
-certifi==2021.10.8
-cffi==1.15.0
-channels-redis==3.4.0
-channels==3.0.4
-chardet==4.0.0; python_version >= '3.1'
-charset-normalizer==2.0.12; python_version >= '3'
+certifi==2022.6.15; python_version >= '3.6'
+cffi==1.15.1
+channels==3.0.5
+channels-redis==3.4.1
+charset-normalizer==2.1.0; python_version >= '3.6'
 click==8.1.3; python_version >= '3.7'
 coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 concurrent-log-handler==0.9.20
 constantly==15.1.0
-cryptography==37.0.1; python_version >= '3.6'
+cryptography==37.0.4; python_version >= '3.6'
 daphne==3.0.2; python_version >= '3.6'
 dateparser==1.1.1
-django-cors-headers==3.11.0
-django-extensions==3.1.5
-django-filter==21.1
-django-picklefield==3.0.1; python_version >= '3'
-django-q==1.3.9
-django==4.0.4
+deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+deprecation==2.1.0
+django==4.0.6
+django-cors-headers==3.13.0
+django-extensions==3.2.0
+django-filter==22.1
+django-picklefield==3.1; python_version >= '3'
+-e git+https://github.com/paperless-ngx/django-q.git@bf20d57f859a7d872d5979cd8879fac9c9df981c#egg=django-q
 djangorestframework==3.13.1
-filelock==3.6.0
+filelock==3.7.1
 fuzzywuzzy[speedup]==0.18.0
 gunicorn==20.1.0
 h11==0.13.0; python_version >= '3.6'
@@ -45,50 +39,50 @@ hiredis==2.0.0; python_version >= '3.6'
 httptools==0.4.0
 humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 hyperlink==21.0.0
-idna==3.3; python_version >= '3'
-imap-tools==0.54.0
+idna==3.3; python_version >= '3.5'
+imap-tools==0.56.0
 img2pdf==0.4.4
-importlib-resources==5.7.1; python_version < '3.9'
+importlib-resources==5.8.0; python_version < '3.9'
 incremental==21.3.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
 joblib==1.1.0; python_version >= '3.6'
 langdetect==1.0.9
-lxml==4.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-msgpack==1.0.3
-numpy==1.22.3; python_version >= '3.8'
-ocrmypdf==13.4.3
+lxml==4.9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+msgpack==1.0.4
+numpy==1.23.1; python_version >= '3.8'
+ocrmypdf==13.6.1
 packaging==21.3; python_version >= '3.6'
 pathvalidate==2.5.0
 pdf2image==1.16.0
-pdfminer.six==20220319
-pikepdf==5.1.2
-pillow==9.1.0
+pdfminer.six==20220524
+pikepdf==5.4.0
+pillow==9.2.0
 pluggy==1.0.0; python_version >= '3.6'
-portalocker==2.4.0; python_version >= '3'
+portalocker==2.5.1; python_version >= '3'
 psycopg2==2.9.3
-pyasn1-modules==0.2.8
 pyasn1==0.4.8
+pyasn1-modules==0.2.8
 pycparser==2.21
 pyopenssl==22.0.0
-pyparsing==3.0.8; python_full_version >= '3.6.8'
+pyparsing==3.0.9; python_full_version >= '3.6.8'
 python-dateutil==2.8.2
 python-dotenv==0.20.0
-python-gnupg==0.4.8
+python-gnupg==0.4.9
 python-levenshtein==0.12.2
-python-magic==0.4.25
-pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+python-magic==0.4.27
 pytz==2022.1
+pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 pyyaml==6.0
 pyzbar==0.1.9
-redis==3.5.3
+redis==4.3.4
 regex==2022.3.2; python_version >= '3.6'
-reportlab==3.6.9; python_version >= '3.7' and python_version < '4'
-requests==2.27.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-scikit-learn==1.0.2
-scipy==1.8.0; python_version < '3.11' and python_version >= '3.8'
+reportlab==3.6.11; python_version >= '3.7' and python_version < '4'
+requests==2.28.1; python_version >= '3.7' and python_version < '4'
+scikit-learn==1.1.1
+scipy==1.8.1; python_version < '3.11' and python_version >= '3.8'
 service-identity==21.1.0
-setuptools==62.1.0; python_version >= '3.7'
+setuptools==63.2.0; python_version >= '3.7'
 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sniffio==1.2.0; python_version >= '3.5'
 sqlparse==0.4.2; python_version >= '3.5'
@@ -97,17 +91,18 @@ tika==1.24
 tqdm==4.64.0
 twisted[tls]==22.4.0; python_full_version >= '3.6.7'
 txaio==22.2.1; python_version >= '3.6'
-typing-extensions==4.2.0; python_version >= '3.7'
+typing-extensions==4.3.0; python_version >= '3.7'
 tzdata==2022.1; python_version >= '3.6'
 tzlocal==4.2; python_version >= '3.6'
-urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-uvicorn[standard]==0.17.6
+urllib3==1.26.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
+uvicorn[standard]==0.18.2
 uvloop==0.16.0
-watchdog==2.1.7
-watchgod==0.8.2
+watchdog==2.1.9
+watchfiles==0.16.0
 wcwidth==0.2.5
 websockets==10.3
-whitenoise==6.0.0
+whitenoise==6.2.0
 whoosh==2.7.4
-zipp==3.8.0; python_version < '3.9'
+wrapt==1.14.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+zipp==3.8.1; python_version < '3.9'
 zope.interface==5.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

scripts/post-consumption-example.sh

@@ -1,21 +1,16 @@
 #!/usr/bin/env bash
 
-DOCUMENT_ID=${1}
-DOCUMENT_FILE_NAME=${2}
-DOCUMENT_SOURCE_PATH=${3}
-DOCUMENT_THUMBNAIL_PATH=${4}
-DOCUMENT_DOWNLOAD_URL=${5}
-DOCUMENT_THUMBNAIL_URL=${6}
-DOCUMENT_CORRESPONDENT=${7}
-DOCUMENT_TAGS=${8}
-
 echo "
 
 A document with an id of ${DOCUMENT_ID} was just consumed.  I know the
 following additional information about it:
 
 * Generated File Name: ${DOCUMENT_FILE_NAME}
+* Archive Path: ${DOCUMENT_ARCHIVE_PATH}
 * Source Path: ${DOCUMENT_SOURCE_PATH}
+* Created: ${DOCUMENT_CREATED}
+* Added: ${DOCUMENT_ADDED}
+* Modified: ${DOCUMENT_MODIFIED}
 * Thumbnail Path: ${DOCUMENT_THUMBNAIL_PATH}
 * Download URL: ${DOCUMENT_DOWNLOAD_URL}
 * Thumbnail URL: ${DOCUMENT_THUMBNAIL_URL}

src-ui/cypress.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from 'cypress'
+
+export default defineConfig({
+  videosFolder: 'cypress/videos',
+  screenshotsFolder: 'cypress/screenshots',
+  fixturesFolder: 'cypress/fixtures',
+  e2e: {
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.ts')(on, config)
+    },
+    baseUrl: 'http://localhost:4200',
+  },
+})

src-ui/cypress.json

@@ -1,9 +0,0 @@
-{
-  "integrationFolder": "cypress/integration",
-  "supportFile": "cypress/support/index.ts",
-  "videosFolder": "cypress/videos",
-  "screenshotsFolder": "cypress/screenshots",
-  "pluginsFile": "cypress/plugins/index.ts",
-  "fixturesFolder": "cypress/fixtures",
-  "baseUrl": "http://localhost:4200"
-}

src-ui/cypress/e2e/documents/document-detail.cy.ts

@@ -1,5 +1,7 @@
 describe('document-detail', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.modifiedDocuments = []
 
     cy.fixture('documents/documents.json').then((documentsJson) => {
@@ -15,30 +17,6 @@ describe('document-detail', () => {
       req.reply({ result: 'OK' })
     }).as('saveDoc')
 
-    cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
-      fixture: 'documents/1/metadata.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
-      fixture: 'documents/1/suggestions.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/saved_views/*', {
-      fixture: 'saved_views/savedviews.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/document_types/*', {
-      fixture: 'document_types/doctypes.json',
-    })
-
     cy.viewport(1024, 1024)
     cy.visit('/documents/1/')
   })

src-ui/cypress/e2e/documents/documents-list.cy.ts

@@ -1,8 +1,9 @@
 describe('documents-list', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.bulkEdits = {}
 
-    // mock API methods
     cy.fixture('documents/documents.json').then((documentsJson) => {
       // bulk edit
       cy.intercept(
@@ -53,40 +54,25 @@ describe('documents-list', () => {
       })
     })
 
-    cy.intercept('http://localhost:8000/api/documents/1/thumb/', {
-      fixture: 'documents/lorem-ipsum.png',
-    })
-
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/document_types/*', {
-      fixture: 'document_types/doctypes.json',
-    })
-
+    cy.viewport(1280, 1024)
     cy.visit('/documents')
   })
 
   it('should show a list of documents rendered as cards with thumbnails', () => {
     cy.contains('3 documents')
-    cy.contains('lorem-ipsum')
+    cy.contains('lorem ipsum')
     cy.get('app-document-card-small:first-of-type img')
       .invoke('attr', 'src')
       .should('eq', 'http://localhost:8000/api/documents/1/thumb/')
   })
 
   it('should change to table "details" view', () => {
-    cy.get('div.btn-group-toggle input[value="details"]').parent().click()
+    cy.get('div.btn-group input[value="details"]').next().click()
     cy.get('table')
   })
 
   it('should change to large cards view', () => {
-    cy.get('div.btn-group-toggle input[value="largeCards"]').parent().click()
+    cy.get('div.btn-group input[value="largeCards"]').next().click()
     cy.get('app-document-card-large')
   })
 

src-ui/cypress/e2e/documents/query-params.cy.ts

@@ -0,0 +1,331 @@
+import { PaperlessDocument } from 'src/app/data/paperless-document'
+
+describe('documents query params', () => {
+  beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
+    cy.fixture('documents/documents.json').then((documentsJson) => {
+      // mock api filtering
+      cy.intercept('GET', 'http://localhost:8000/api/documents/*', (req) => {
+        let response = { ...documentsJson }
+
+        if (req.query.hasOwnProperty('ordering')) {
+          const sort_field = req.query['ordering'].toString().replace('-', '')
+          const reverse = req.query['ordering'].toString().indexOf('-') !== -1
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).sort((docA, docB) => {
+            let result = 0
+            switch (sort_field) {
+              case 'created':
+              case 'added':
+                result =
+                  new Date(docA[sort_field]) < new Date(docB[sort_field])
+                    ? -1
+                    : 1
+                break
+              case 'archive_serial_number':
+                result = docA[sort_field] < docB[sort_field] ? -1 : 1
+                break
+            }
+            if (reverse) result = -result
+            return result
+          })
+        }
+
+        if (req.query.hasOwnProperty('tags__id__in')) {
+          const tag_ids: Array<number> = req.query['tags__id__in']
+            .toString()
+            .split(',')
+            .map((v) => +v)
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) =>
+              d.tags.length > 0 &&
+              d.tags.filter((t) => tag_ids.includes(t)).length > 0
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('tags__id__none')) {
+          const tag_ids: Array<number> = req.query['tags__id__none']
+            .toString()
+            .split(',')
+            .map((v) => +v)
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.tags.filter((t) => tag_ids.includes(t)).length == 0)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('is_tagged') &&
+          req.query['is_tagged'] == '0'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.tags.length == 0)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('document_type__id')) {
+          const doctype_id = +req.query['document_type__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.document_type == doctype_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('document_type__isnull') &&
+          req.query['document_type__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.document_type == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('correspondent__id')) {
+          const correspondent_id = +req.query['correspondent__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.correspondent == correspondent_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('correspondent__isnull') &&
+          req.query['correspondent__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.correspondent == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('storage_path__id')) {
+          const storage_path_id = +req.query['storage_path__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.storage_path == storage_path_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('storage_path__isnull') &&
+          req.query['storage_path__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.storage_path == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('created__date__gt')) {
+          const date = new Date(req.query['created__date__gt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.created) > date)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('created__date__lt')) {
+          const date = new Date(req.query['created__date__lt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.created) < date)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('added__date__gt')) {
+          const date = new Date(req.query['added__date__gt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.added) > date)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('added__date__lt')) {
+          const date = new Date(req.query['added__date__lt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.added) < date)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('title_content')) {
+          const title_content_regexp = new RegExp(
+            req.query['title_content'].toString(),
+            'i'
+          )
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) =>
+              title_content_regexp.test(d.title) ||
+              title_content_regexp.test(d.content)
+          )
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('archive_serial_number')) {
+          const asn = +req.query['archive_serial_number']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.archive_serial_number == asn)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__isnull')) {
+          const isnull = req.query['storage_path__isnull'] == '1'
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) =>
+            isnull
+              ? d.archive_serial_number == undefined
+              : d.archive_serial_number != undefined
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__gt')) {
+          const asn = +req.query['archive_serial_number__gt']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) => d.archive_serial_number > 0 && d.archive_serial_number > asn
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__lt')) {
+          const asn = +req.query['archive_serial_number__lt']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) => d.archive_serial_number > 0 && d.archive_serial_number < asn
+          )
+          response.count = response.results.length
+        }
+
+        req.reply(response)
+      })
+    })
+  })
+
+  it('should show a list of documents sorted by created', () => {
+    cy.visit('/documents?sort=created')
+    cy.get('app-document-card-small').first().contains('No latin title')
+  })
+
+  it('should show a list of documents reverse sorted by created', () => {
+    cy.visit('/documents?sort=created&reverse=true')
+    cy.get('app-document-card-small').first().contains('sit amet')
+  })
+
+  it('should show a list of documents sorted by added', () => {
+    cy.visit('/documents?sort=added')
+    cy.get('app-document-card-small').first().contains('No latin title')
+  })
+
+  it('should show a list of documents reverse sorted by added', () => {
+    cy.visit('/documents?sort=added&reverse=true')
+    cy.get('app-document-card-small').first().contains('sit amet')
+  })
+
+  it('should show a list of documents filtered by any tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&tags__id__in=2,4,5')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by excluded tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&tags__id__none=2,4')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by no tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&is_tagged=0')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by document type', () => {
+    cy.visit('/documents?sort=created&reverse=true&document_type__id=1')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by no document type', () => {
+    cy.visit('/documents?sort=created&reverse=true&document_type__isnull=1')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by correspondent', () => {
+    cy.visit('/documents?sort=created&reverse=true&correspondent__id=9')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by no correspondent', () => {
+    cy.visit('/documents?sort=created&reverse=true&correspondent__isnull=1')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by storage path', () => {
+    cy.visit('/documents?sort=created&reverse=true&storage_path__id=2')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by no storage path', () => {
+    cy.visit('/documents?sort=created&reverse=true&storage_path__isnull=1')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by title or content', () => {
+    cy.visit('/documents?sort=created&reverse=true&title_content=lorem')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by asn', () => {
+    cy.visit('/documents?sort=created&reverse=true&archive_serial_number=12345')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by empty asn', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__isnull=1'
+    )
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by non-empty asn', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__isnull=0'
+    )
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by asn greater than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__gt=12346'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by asn less than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__lt=12346'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by created date greater than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&created__date__gt=2022-03-23'
+    )
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by created date less than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&created__date__lt=2022-03-23'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by added date greater than', () => {
+    cy.visit('/documents?sort=created&reverse=true&added__date__gt=2022-03-24')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by added date less than', () => {
+    cy.visit('/documents?sort=created&reverse=true&added__date__lt=2022-03-24')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by multiple filters', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&document_type__id=1&correspondent__id=9&tags__id__in=4,5'
+    )
+    cy.contains('2 documents')
+  })
+})

src-ui/cypress/e2e/manage/manage.cy.ts

@@ -1,12 +1,5 @@
 describe('manage', () => {
-  beforeEach(() => {
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-  })
+  // also uses global fixtures from cypress/support/e2e.ts
 
   it('should show a list of correspondents with bottom pagination as well', () => {
     cy.visit('/correspondents')

src-ui/cypress/e2e/settings/settings.cy.ts

@@ -1,47 +1,49 @@
 describe('settings', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.modifiedViews = []
 
     // mock API methods
-    cy.fixture('saved_views/savedviews.json').then((savedViewsJson) => {
-      // saved views PATCH
-      cy.intercept(
-        'PATCH',
-        'http://localhost:8000/api/saved_views/*',
-        (req) => {
-          this.modifiedViews.push(req.body) // store this for later
-          req.reply({ result: 'OK' })
-        }
-      )
+    cy.intercept('http://localhost:8000/api/ui_settings/', {
+      fixture: 'ui_settings/settings.json',
+    }).then(() => {
+      cy.fixture('saved_views/savedviews.json').then((savedViewsJson) => {
+        // saved views PATCH
+        cy.intercept(
+          'PATCH',
+          'http://localhost:8000/api/saved_views/*',
+          (req) => {
+            this.modifiedViews.push(req.body) // store this for later
+            req.reply({ result: 'OK' })
+          }
+        )
 
-      cy.intercept('GET', 'http://localhost:8000/api/saved_views/*', (req) => {
-        let response = { ...savedViewsJson }
-        if (this.modifiedViews.length) {
-          response.results = response.results.map((v) => {
-            if (this.modifiedViews.find((mv) => mv.id == v.id))
-              v = this.modifiedViews.find((mv) => mv.id == v.id)
-            return v
-          })
-        }
+        cy.intercept(
+          'GET',
+          'http://localhost:8000/api/saved_views/*',
+          (req) => {
+            let response = { ...savedViewsJson }
+            if (this.modifiedViews.length) {
+              response.results = response.results.map((v) => {
+                if (this.modifiedViews.find((mv) => mv.id == v.id))
+                  v = this.modifiedViews.find((mv) => mv.id == v.id)
+                return v
+              })
+            }
 
-        req.reply(response)
-      }).as('savedViews')
-    })
-
-    cy.fixture('documents/documents.json').then((documentsJson) => {
-      cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
-        let response = { ...documentsJson }
-        response = response.results.find((d) => d.id == 1)
-        req.reply(response)
+            req.reply(response)
+          }
+        ).as('savedViews')
       })
-    })
 
-    cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
-      fixture: 'documents/1/metadata.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
-      fixture: 'documents/1/suggestions.json',
+      cy.fixture('documents/documents.json').then((documentsJson) => {
+        cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
+          let response = { ...documentsJson }
+          response = response.results.find((d) => d.id == 1)
+          req.reply(response)
+        })
+      })
     })
 
     cy.viewport(1024, 1024)

src-ui/cypress/e2e/tasks/tasks.cy.ts

@@ -0,0 +1,60 @@
+describe('tasks', () => {
+  beforeEach(() => {
+    this.dismissedTasks = new Set<number>()
+
+    cy.fixture('tasks/tasks.json').then((tasksViewsJson) => {
+      // acknowledge tasks POST
+      cy.intercept(
+        'POST',
+        'http://localhost:8000/api/acknowledge_tasks/',
+        (req) => {
+          req.body['tasks'].forEach((t) => this.dismissedTasks.add(t)) // store this for later
+          req.reply({ result: 'OK' })
+        }
+      )
+
+      cy.intercept('GET', 'http://localhost:8000/api/tasks/', (req) => {
+        let response = [...tasksViewsJson]
+        if (this.dismissedTasks.size) {
+          response = response.filter((t) => {
+            return !this.dismissedTasks.has(t.id)
+          })
+        }
+
+        req.reply(response)
+      }).as('tasks')
+    })
+
+    cy.visit('/tasks')
+    cy.wait('@tasks')
+  })
+
+  it('should show a list of dismissable tasks in tabs', () => {
+    cy.get('tbody').find('tr:visible').its('length').should('eq', 10) // double because collapsible result tr
+    cy.wait(500) // stabilizes the test, for some reason...
+    cy.get('tbody')
+      .find('button:visible')
+      .contains('Dismiss')
+      .first()
+      .click()
+      .wait('@tasks')
+      .wait(2000)
+      .then(() => {
+        cy.get('tbody').find('tr:visible').its('length').should('eq', 8) // double because collapsible result tr
+      })
+  })
+
+  it('should allow toggling all tasks in list and warn on dismiss', () => {
+    cy.get('thead').find('input[type="checkbox"]').first().click()
+    cy.get('body').find('button').contains('Dismiss selected').first().click()
+    cy.contains('Confirm')
+    cy.get('.modal')
+      .contains('button', 'Dismiss')
+      .click()
+      .wait('@tasks')
+      .wait(2000)
+      .then(() => {
+        cy.get('tbody').find('tr:visible').should('not.exist')
+      })
+  })
+})

src-ui/cypress/fixtures/remote_version/remote_version.json

@@ -0,0 +1 @@
+{"version":"v1.7.1","update_available":false,"feature_is_set":true}

src-ui/cypress/fixtures/storage_paths/storage_paths.json

@@ -0,0 +1,17 @@
+{
+    "count": 1,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 2,
+            "slug": "year-title",
+            "name": "Year - Title",
+            "path": "{created_year}/{title}",
+            "match": "",
+            "matching_algorithm": 6,
+            "is_insensitive": true,
+            "document_count": 1
+        }
+    ]
+}

src-ui/cypress/fixtures/tags/tags.json

@@ -1 +1,103 @@
-{"count":8,"next":null,"previous":null,"results":[{"id":4,"slug":"another-sample-tag","name":"Another Sample Tag","color":"#a6cee3","text_color":"#000000","match":"","matching_algorithm":6,"is_insensitive":true,"is_inbox_tag":false,"document_count":3},{"id":7,"slug":"newone","name":"NewOne","color":"#9e4ad1","text_color":"#ffffff","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":2},{"id":6,"slug":"partial-tag","name":"Partial Tag","color":"#72dba7","text_color":"#000000","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":1},{"id":2,"slug":"tag-2","name":"Tag 2","color":"#612db7","text_color":"#ffffff","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":3},{"id":3,"slug":"tag-3","name":"Tag 3","color":"#b2df8a","text_color":"#000000","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":4},{"id":5,"slug":"tagwithpartial","name":"TagWithPartial","color":"#3b2db4","text_color":"#ffffff","match":"","matching_algorithm":6,"is_insensitive":true,"is_inbox_tag":false,"document_count":2},{"id":8,"slug":"test-another","name":"Test Another","color":"#3ccea5","text_color":"#000000","match":"","matching_algorithm":4,"is_insensitive":true,"is_inbox_tag":false,"document_count":0},{"id":1,"slug":"test-tag","name":"Test Tag","color":"#fb9a99","text_color":"#000000","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":4}]}
+{
+    "count": 8,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 4,
+            "slug": "another-sample-tag",
+            "name": "Another Sample Tag",
+            "color": "#a6cee3",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 6,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 3
+        },
+        {
+            "id": 7,
+            "slug": "newone",
+            "name": "NewOne",
+            "color": "#9e4ad1",
+            "text_color": "#ffffff",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 2
+        },
+        {
+            "id": 6,
+            "slug": "partial-tag",
+            "name": "Partial Tag",
+            "color": "#72dba7",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 1
+        },
+        {
+            "id": 2,
+            "slug": "tag-2",
+            "name": "Tag 2",
+            "color": "#612db7",
+            "text_color": "#ffffff",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 3
+        },
+        {
+            "id": 3,
+            "slug": "tag-3",
+            "name": "Tag 3",
+            "color": "#b2df8a",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 4
+        },
+        {
+            "id": 5,
+            "slug": "tagwithpartial",
+            "name": "TagWithPartial",
+            "color": "#3b2db4",
+            "text_color": "#ffffff",
+            "match": "",
+            "matching_algorithm": 6,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 2
+        },
+        {
+            "id": 8,
+            "slug": "test-another",
+            "name": "Test Another",
+            "color": "#3ccea5",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 4,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 0
+        },
+        {
+            "id": 1,
+            "slug": "test-tag",
+            "name": "Test Tag",
+            "color": "#fb9a99",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 4
+        }
+    ]
+}

src-ui/cypress/fixtures/ui_settings/settings.json

@@ -0,0 +1,34 @@
+{
+    "user_id": 1,
+    "username": "admin",
+    "display_name": "Admin",
+    "settings": {
+        "language": "",
+        "bulk_edit": {
+            "confirmation_dialogs": true,
+            "apply_on_close": false
+        },
+        "documentListSize": 50,
+        "dark_mode": {
+            "use_system": true,
+            "enabled": "false",
+            "thumb_inverted": "true"
+        },
+        "theme": {
+            "color": "#b198e5"
+        },
+        "document_details": {
+            "native_pdf_viewer": false
+        },
+        "date_display": {
+            "date_locale": "",
+            "date_format": "mediumDate"
+        },
+        "notifications": {
+            "consumer_new_documents": true,
+            "consumer_success": true,
+            "consumer_failed": true,
+            "consumer_suppress_on_dashboard": true
+        }
+    }
+}

src-ui/cypress/support/e2e.ts

@@ -0,0 +1,43 @@
+// mock API methods
+
+beforeEach(() => {
+  cy.intercept('http://localhost:8000/api/ui_settings/', {
+    fixture: 'ui_settings/settings.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/remote_version/', {
+    fixture: 'remote_version/remote_version.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/saved_views/*', {
+    fixture: 'saved_views/savedviews.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/tags/*', {
+    fixture: 'tags/tags.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/correspondents/*', {
+    fixture: 'correspondents/correspondents.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/document_types/*', {
+    fixture: 'document_types/doctypes.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/storage_paths/*', {
+    fixture: 'storage_paths/storage_paths.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
+    fixture: 'documents/1/metadata.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
+    fixture: 'documents/1/suggestions.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/documents/1/thumb/', {
+    fixture: 'documents/lorem-ipsum.png',
+  })
+})

src-ui/cypress/support/index.ts

@@ -1,17 +0,0 @@
-// ***********************************************************
-// This example support/index.js is processed and
-// loaded automatically before your test files.
-//
-// This is a great place to put global configuration and
-// behavior that modifies Cypress.
-//
-// You can change the location of this file or turn off
-// automatically serving support files with the
-// 'supportFile' configuration option.
-//
-// You can read more here:
-// https://on.cypress.io/configuration
-// ***********************************************************
-
-// When a command from ./commands is ready to use, import with `import './commands'` syntax
-// import './commands';

src-ui/package.json

@@ -13,48 +13,48 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/common": "~13.3.5",
-    "@angular/compiler": "~13.3.5",
-    "@angular/core": "~13.3.5",
-    "@angular/forms": "~13.3.5",
-    "@angular/localize": "~13.3.5",
-    "@angular/platform-browser": "~13.3.5",
-    "@angular/platform-browser-dynamic": "~13.3.5",
-    "@angular/router": "~13.3.5",
-    "@ng-bootstrap/ng-bootstrap": "^12.1.1",
-    "@ng-select/ng-select": "^8.1.1",
+    "@angular/common": "~14.0.4",
+    "@angular/compiler": "~14.0.4",
+    "@angular/core": "~14.0.4",
+    "@angular/forms": "~14.0.4",
+    "@angular/localize": "~14.0.4",
+    "@angular/platform-browser": "~14.0.4",
+    "@angular/platform-browser-dynamic": "~14.0.4",
+    "@angular/router": "~14.0.4",
+    "@ng-bootstrap/ng-bootstrap": "^13.0.0-beta.1",
+    "@ng-select/ng-select": "^9.0.2",
     "@ngneat/dirty-check-forms": "^3.0.2",
-    "@popperjs/core": "^2.11.4",
+    "@popperjs/core": "^2.11.5",
     "bootstrap": "^5.1.3",
     "file-saver": "^2.0.5",
     "ng2-pdf-viewer": "^9.0.0",
     "ngx-color": "^7.3.3",
-    "ngx-cookie-service": "^13.1.2",
+    "ngx-cookie-service": "^14.0.1",
     "ngx-file-drop": "^13.0.0",
     "rxjs": "~7.5.5",
     "tslib": "^2.3.1",
     "uuid": "^8.3.1",
-    "zone.js": "~0.11.4"
+    "zone.js": "~0.11.6"
   },
   "devDependencies": {
-    "@angular-builders/jest": "13.0.3",
-    "@angular-devkit/build-angular": "~13.3.4",
-    "@angular/cli": "~13.3.4",
-    "@angular/compiler-cli": "~13.3.5",
-    "@types/jest": "27.4.1",
-    "@types/node": "^17.0.30",
+    "@angular-builders/jest": "14.0.0",
+    "@angular-devkit/build-angular": "~14.0.4",
+    "@angular/cli": "~14.0.4",
+    "@angular/compiler-cli": "~14.0.4",
+    "@types/jest": "28.1.4",
+    "@types/node": "^18.0.0",
     "codelyzer": "^6.0.2",
-    "concurrently": "7.1.0",
-    "jest": "28.0.3",
-    "jest-environment-jsdom": "^28.0.2",
-    "jest-preset-angular": "^12.0.0-next.1",
-    "ts-node": "~10.7.0",
+    "concurrently": "7.2.2",
+    "jest": "28.1.2",
+    "jest-environment-jsdom": "^28.1.2",
+    "jest-preset-angular": "^12.1.0",
+    "ts-node": "~10.8.1",
     "tslint": "~6.1.3",
     "typescript": "~4.6.3",
     "wait-on": "~6.0.1"
   },
   "optionalDependencies": {
-    "@cypress/schematic": "^1.6.0",
-    "cypress": "~9.6.0"
+    "@cypress/schematic": "^2.0.0",
+    "cypress": "~10.3.0"
   }
 }

src-ui/src/app/app-routing.module.ts

@@ -12,6 +12,8 @@ import { TagListComponent } from './components/manage/tag-list/tag-list.componen
 import { NotFoundComponent } from './components/not-found/not-found.component'
 import { DocumentAsnComponent } from './components/document-asn/document-asn.component'
 import { DirtyFormGuard } from './guards/dirty-form.guard'
+import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component'
+import { TasksComponent } from './components/manage/tasks/tasks.component'
 
 const routes: Routes = [
   { path: '', redirectTo: 'dashboard', pathMatch: 'full' },
@@ -27,12 +29,14 @@ const routes: Routes = [
       { path: 'tags', component: TagListComponent },
       { path: 'documenttypes', component: DocumentTypeListComponent },
       { path: 'correspondents', component: CorrespondentListComponent },
+      { path: 'storagepaths', component: StoragePathListComponent },
       { path: 'logs', component: LogsComponent },
       {
         path: 'settings',
         component: SettingsComponent,
         canDeactivate: [DirtyFormGuard],
       },
+      { path: 'tasks', component: TasksComponent },
     ],
   },
 

src-ui/src/app/app.component.ts

@@ -1,4 +1,5 @@
-import { SettingsService, SETTINGS_KEYS } from './services/settings.service'
+import { SettingsService } from './services/settings.service'
+import { SETTINGS_KEYS } from './data/paperless-uisettings'
 import { Component, OnDestroy, OnInit } from '@angular/core'
 import { Router } from '@angular/router'
 import { Subscription } from 'rxjs'
@@ -6,6 +7,7 @@ import { ConsumerStatusService } from './services/consumer-status.service'
 import { ToastService } from './services/toast.service'
 import { NgxFileDropEntry } from 'ngx-file-drop'
 import { UploadDocumentsService } from './services/upload-documents.service'
+import { TasksService } from './services/tasks.service'
 
 @Component({
   selector: 'app-root',
@@ -26,7 +28,8 @@ export class AppComponent implements OnInit, OnDestroy {
     private consumerStatusService: ConsumerStatusService,
     private toastService: ToastService,
     private router: Router,
-    private uploadDocumentsService: UploadDocumentsService
+    private uploadDocumentsService: UploadDocumentsService,
+    private tasksService: TasksService
   ) {
     let anyWindow = window as any
     anyWindow.pdfWorkerSrc = 'assets/js/pdf.worker.min.js'
@@ -64,6 +67,7 @@ export class AppComponent implements OnInit, OnDestroy {
     this.successSubscription = this.consumerStatusService
       .onDocumentConsumptionFinished()
       .subscribe((status) => {
+        this.tasksService.reload()
         if (
           this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)
         ) {
@@ -82,6 +86,7 @@ export class AppComponent implements OnInit, OnDestroy {
     this.failedSubscription = this.consumerStatusService
       .onDocumentConsumptionFailed()
       .subscribe((status) => {
+        this.tasksService.reload()
         if (
           this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)
         ) {
@@ -94,6 +99,7 @@ export class AppComponent implements OnInit, OnDestroy {
     this.newDocumentSubscription = this.consumerStatusService
       .onDocumentDetected()
       .subscribe((status) => {
+        this.tasksService.reload()
         if (
           this.showNotification(
             SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT

src-ui/src/app/app.module.ts

@@ -1,5 +1,5 @@
 import { BrowserModule } from '@angular/platform-browser'
-import { NgModule } from '@angular/core'
+import { APP_INITIALIZER, NgModule } from '@angular/core'
 import { AppRoutingModule } from './app-routing.module'
 import { AppComponent } from './app.component'
 import {
@@ -61,7 +61,7 @@ import { SafeUrlPipe } from './pipes/safeurl.pipe'
 import { SafeHtmlPipe } from './pipes/safehtml.pipe'
 import { CustomDatePipe } from './pipes/custom-date.pipe'
 import { DateComponent } from './components/common/input/date/date.component'
-import { ISODateTimeAdapter } from './utils/ngb-iso-date-time-adapter'
+import { ISODateAdapter } from './utils/ngb-iso-date-adapter'
 import { LocalizedDateParserFormatter } from './utils/ngb-date-parser-formatter'
 import { ApiVersionInterceptor } from './interceptors/api-version.interceptor'
 import { ColorSliderModule } from 'ngx-color/slider'
@@ -87,6 +87,10 @@ import localeSr from '@angular/common/locales/sr'
 import localeSv from '@angular/common/locales/sv'
 import localeTr from '@angular/common/locales/tr'
 import localeZh from '@angular/common/locales/zh'
+import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component'
+import { StoragePathEditDialogComponent } from './components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
+import { SettingsService } from './services/settings.service'
+import { TasksComponent } from './components/manage/tasks/tasks.component'
 
 registerLocaleData(localeBe)
 registerLocaleData(localeCs)
@@ -109,6 +113,12 @@ registerLocaleData(localeSv)
 registerLocaleData(localeTr)
 registerLocaleData(localeZh)
 
+function initializeApp(settings: SettingsService) {
+  return () => {
+    return settings.initializeSettings()
+  }
+}
+
 @NgModule({
   declarations: [
     AppComponent,
@@ -118,6 +128,7 @@ registerLocaleData(localeZh)
     TagListComponent,
     DocumentTypeListComponent,
     CorrespondentListComponent,
+    StoragePathListComponent,
     LogsComponent,
     SettingsComponent,
     NotFoundComponent,
@@ -125,6 +136,7 @@ registerLocaleData(localeZh)
     ConfirmDialogComponent,
     TagEditDialogComponent,
     DocumentTypeEditDialogComponent,
+    StoragePathEditDialogComponent,
     TagComponent,
     PageHeaderComponent,
     AppFrameComponent,
@@ -160,6 +172,7 @@ registerLocaleData(localeZh)
     DateComponent,
     ColorComponent,
     DocumentAsnComponent,
+    TasksComponent,
   ],
   imports: [
     BrowserModule,
@@ -174,6 +187,12 @@ registerLocaleData(localeZh)
     ColorSliderModule,
   ],
   providers: [
+    {
+      provide: APP_INITIALIZER,
+      useFactory: initializeApp,
+      deps: [SettingsService],
+      multi: true,
+    },
     DatePipe,
     CookieService,
     {
@@ -188,7 +207,7 @@ registerLocaleData(localeZh)
     },
     FilterPipe,
     DocumentTitlePipe,
-    { provide: NgbDateAdapter, useClass: ISODateTimeAdapter },
+    { provide: NgbDateAdapter, useClass: ISODateAdapter },
     { provide: NgbDateParserFormatter, useClass: LocalizedDateParserFormatter },
   ],
   bootstrap: [AppComponent],

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -21,17 +21,17 @@
   </div>
   <ul ngbNav class="order-sm-3">
     <li ngbDropdown class="nav-item dropdown">
-      <button class="btn text-light" id="userDropdown" ngbDropdownToggle>
-        <span *ngIf="displayName" class="navbar-text small me-2 text-light d-none d-sm-inline">
-          {{displayName}}
+      <button class="btn" id="userDropdown" ngbDropdownToggle>
+        <span class="small me-2 d-none d-sm-inline">
+          {{this.settingsService.displayName}}
         </span>
         <svg width="1.3em" height="1.3em" fill="currentColor">
           <use xlink:href="assets/bootstrap-icons.svg#person-circle"/>
         </svg>
       </button>
       <div ngbDropdownMenu class="dropdown-menu-end shadow me-2" aria-labelledby="userDropdown">
-        <div *ngIf="displayName" class="d-sm-none">
-          <p class="small mb-0 px-3 text-muted" i18n>Logged in as {{displayName}}</p>
+        <div class="d-sm-none">
+          <p class="small mb-0 px-3 text-muted" i18n>Logged in as {{this.settingsService.displayName}}</p>
           <div class="dropdown-divider"></div>
         </div>
         <a ngbDropdownItem class="nav-link" routerLink="settings" (click)="closeMenu()">
@@ -70,8 +70,9 @@
           </li>
         </ul>
 
-        <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.sidebarViews.length > 0'>
+        <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.loading || savedViewService.sidebarViews.length > 0'>
           <ng-container i18n>Saved views</ng-container>
+          <div *ngIf="savedViewService.loading" class="spinner-border spinner-border-sm fw-normal ms-2" role="status"></div>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item w-100" *ngFor="let view of savedViewService.sidebarViews">
@@ -133,6 +134,20 @@
               </svg>&nbsp;<ng-container i18n>Document types</ng-container>
             </a>
           </li>
+          <li class="nav-item">
+            <a class="nav-link" routerLink="storagepaths" routerLinkActive="active" (click)="closeMenu()">
+              <svg class="sidebaricon" fill="currentColor">
+                <use xlink:href="assets/bootstrap-icons.svg#folder"/>
+              </svg>&nbsp;<ng-container i18n>Storage paths</ng-container>
+            </a>
+          </li>
+          <li class="nav-item">
+            <a class="nav-link" routerLink="tasks" routerLinkActive="active" (click)="closeMenu()">
+              <svg class="sidebaricon" fill="currentColor">
+                <use xlink:href="assets/bootstrap-icons.svg#list-task"/>
+              </svg>&nbsp;<ng-container i18n>File Tasks<ng-container *ngIf="tasksService.failedFileTasks.length > 0"><span class="badge bg-danger ms-2">{{tasksService.failedFileTasks.length}}</span></ng-container></ng-container>
+            </a>
+          </li>
           <li class="nav-item">
             <a class="nav-link" routerLink="logs" routerLinkActive="active" (click)="closeMenu()">
               <svg class="sidebaricon" fill="currentColor">
@@ -187,7 +202,7 @@
               <div class="me-3">{{ versionString }}</div>
               <div *ngIf="appRemoteVersion" class="version-check">
                 <ng-template #updateAvailablePopContent>
-                  <span class="small">Paperless-ngx v{{ appRemoteVersion.version }} <ng-container i18n>is available.</ng-container><br/><ng-container i18n>Click to view.</ng-container></span>
+                  <span class="small">Paperless-ngx {{ appRemoteVersion.version }} <ng-container i18n>is available.</ng-container><br/><ng-container i18n>Click to view.</ng-container></span>
                 </ng-template>
                 <ng-template #updateCheckingNotEnabledPopContent>
                   <span class="small"><ng-container i18n>Checking for updates is disabled.</ng-container><br/><ng-container i18n>Click for more information.</ng-container></span>

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -9,6 +9,11 @@
   z-index: 995; /* Behind the navbar */
   padding: 50px 0 0; /* Height of navbar */
   box-shadow: inset -1px 0 0 rgba(0, 0, 0, .1);
+
+  .sidebar-heading .spinner-border {
+    width: 0.8em;
+    height: 0.8em;
+  }
 }
 @media (max-width: 767.98px) {
   .sidebar {

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -1,7 +1,7 @@
 import { Component } from '@angular/core'
 import { FormControl } from '@angular/forms'
 import { ActivatedRoute, Router, Params } from '@angular/router'
-import { from, Observable, Subscription, BehaviorSubject } from 'rxjs'
+import { from, Observable } from 'rxjs'
 import {
   debounceTime,
   distinctUntilChanged,
@@ -15,14 +15,14 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service'
 import { SearchService } from 'src/app/services/rest/search.service'
 import { environment } from 'src/environments/environment'
 import { DocumentDetailComponent } from '../document-detail/document-detail.component'
-import { Meta } from '@angular/platform-browser'
 import { DocumentListViewService } from 'src/app/services/document-list-view.service'
 import { FILTER_FULLTEXT_QUERY } from 'src/app/data/filter-rule-type'
 import {
   RemoteVersionService,
   AppRemoteVersion,
 } from 'src/app/services/rest/remote-version.service'
-import { QueryParamsService } from 'src/app/services/query-params.service'
+import { SettingsService } from 'src/app/services/settings.service'
+import { TasksService } from 'src/app/services/tasks.service'
 
 @Component({
   selector: 'app-app-frame',
@@ -36,16 +36,17 @@ export class AppFrameComponent {
     private openDocumentsService: OpenDocumentsService,
     private searchService: SearchService,
     public savedViewService: SavedViewService,
-    private list: DocumentListViewService,
-    private meta: Meta,
     private remoteVersionService: RemoteVersionService,
-    private queryParamsService: QueryParamsService
+    private list: DocumentListViewService,
+    public settingsService: SettingsService,
+    public tasksService: TasksService
   ) {
     this.remoteVersionService
       .checkForUpdates()
       .subscribe((appRemoteVersion: AppRemoteVersion) => {
         this.appRemoteVersion = appRemoteVersion
       })
+    tasksService.reload()
   }
 
   versionString = `${environment.appTitle} ${environment.version}`
@@ -94,7 +95,7 @@ export class AppFrameComponent {
 
   search() {
     this.closeMenu()
-    this.queryParamsService.navigateWithFilterRules([
+    this.list.quickFilter([
       {
         rule_type: FILTER_FULLTEXT_QUERY,
         value: (this.searchField.value as string).trim(),
@@ -143,17 +144,4 @@ export class AppFrameComponent {
         }
       })
   }
-
-  get displayName() {
-    // TODO: taken from dashboard component, is this the best way to pass around username?
-    let tagFullName = this.meta.getTag('name=full_name')
-    let tagUsername = this.meta.getTag('name=username')
-    if (tagFullName && tagFullName.content) {
-      return tagFullName.content
-    } else if (tagUsername && tagUsername.content) {
-      return tagUsername.content
-    } else {
-      return null
-    }
-  }
 }

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html

@@ -5,7 +5,7 @@
     </div>
     <div class="modal-body">
       <p *ngIf="messageBold"><b>{{messageBold}}</b></p>
-      <p *ngIf="message">{{message}}</p>
+      <p class="mb-0" *ngIf="message" [innerHTML]="message | safeHtml"></p>
     </div>
     <div class="modal-footer">
       <button type="button" class="btn btn-outline-secondary" (click)="cancel()" [disabled]="!buttonsEnabled" i18n>

src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.html

@@ -0,0 +1,24 @@
+<form [formGroup]="objectForm" (ngSubmit)="save()">
+  <div class="modal-header">
+    <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
+    <button type="button" [disabled]="!closeEnabled" class="btn-close" aria-label="Close" (click)="cancel()">
+    </button>
+  </div>
+  <div class="modal-body">
+
+    <p *ngIf="this.dialogMode == 'edit'" i18n>
+      <em>Note that editing a path does not apply changes to stored files until you have run the 'document_renamer' utility. See the <a target="_blank" href="https://paperless-ngx.readthedocs.io/en/latest/administration.html#utilities-renamer">documentation</a>.</em>
+    </p>
+
+    <app-input-text i18n-title title="Name" formControlName="name" [error]="error?.name"></app-input-text>
+    <app-input-text i18n-title title="Path" formControlName="path" [error]="error?.path" [hint]="pathHint"></app-input-text>
+    <app-input-select i18n-title title="Matching algorithm" [items]="getMatchingAlgorithms()" formControlName="matching_algorithm"></app-input-select>
+    <app-input-text *ngIf="patternRequired" i18n-title title="Matching pattern" formControlName="match" [error]="error?.match"></app-input-text>
+    <app-input-check *ngIf="patternRequired" i18n-title title="Case insensitive" formControlName="is_insensitive"></app-input-check>
+
+  </div>
+  <div class="modal-footer">
+    <button type="button" class="btn btn-outline-secondary" (click)="cancel()" i18n [disabled]="networkActive">Cancel</button>
+    <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive">Save</button>
+  </div>
+</form>

src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts

@@ -0,0 +1,50 @@
+import { Component } from '@angular/core'
+import { FormControl, FormGroup } from '@angular/forms'
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
+import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component'
+import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
+import { StoragePathService } from 'src/app/services/rest/storage-path.service'
+import { ToastService } from 'src/app/services/toast.service'
+
+@Component({
+  selector: 'app-storage-path-edit-dialog',
+  templateUrl: './storage-path-edit-dialog.component.html',
+  styleUrls: ['./storage-path-edit-dialog.component.scss'],
+})
+export class StoragePathEditDialogComponent extends EditDialogComponent<PaperlessStoragePath> {
+  constructor(
+    service: StoragePathService,
+    activeModal: NgbActiveModal,
+    toastService: ToastService
+  ) {
+    super(service, activeModal, toastService)
+  }
+
+  get pathHint() {
+    return (
+      $localize`e.g.` +
+      ' <code>{created_year}-{title}</code> ' +
+      $localize`or use slashes to add directories e.g.` +
+      ' <code>{created_year}/{correspondent}/{title}</code>. ' +
+      $localize`See <a target="_blank" href="https://paperless-ngx.readthedocs.io/en/latest/advanced_usage.html#file-name-handling">documentation</a> for full list.`
+    )
+  }
+
+  getCreateTitle() {
+    return $localize`Create new storage path`
+  }
+
+  getEditTitle() {
+    return $localize`Edit storage path`
+  }
+
+  getForm(): FormGroup {
+    return new FormGroup({
+      name: new FormControl(''),
+      path: new FormControl(''),
+      matching_algorithm: new FormControl(1),
+      match: new FormControl(''),
+      is_insensitive: new FormControl(true),
+    })
+  }
+}

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html

@@ -16,13 +16,11 @@
   <div class="dropdown-menu py-0 shadow" ngbDropdownMenu attr.aria-labelledby="dropdown{{title}}">
     <div class="list-group list-group-flush">
       <div *ngIf="!editing && multiple" class="list-group-item d-flex">
-        <div class="btn-group btn-group-xs btn-group-toggle flex-fill" ngbRadioGroup [(ngModel)]="selectionModel.logicalOperator" (change)="selectionModel.toggleOperator()" [disabled]="!operatorToggleEnabled">
-          <label ngbButtonLabel class="btn btn-outline-primary">
-            <input ngbButton type="radio" class="btn-check" name="logicalOperator" value="and" i18n> All
-          </label>
-          <label ngbButtonLabel class="btn btn-outline-primary">
-            <input ngbButton type="radio" class="btn-check" name="logicalOperator" value="or" i18n> Any
-          </label>
+        <div class="btn-group btn-group-xs flex-fill">
+          <input [(ngModel)]="selectionModel.logicalOperator" [disabled]="!operatorToggleEnabled" (ngModelChange)="selectionModel.toggleOperator()" type="radio" class="btn-check" id="logicalOperatorAnd" value="and">
+          <label class="btn btn-outline-primary" for="logicalOperatorAnd" i18n>All</label>
+          <input [(ngModel)]="selectionModel.logicalOperator" [disabled]="!operatorToggleEnabled" (ngModelChange)="selectionModel.toggleOperator()" type="radio" class="btn-check" id="logicalOperatorOr" value="or">
+          <label class="btn btn-outline-primary" for="logicalOperatorOr" i18n>Any</label>
         </div>
       </div>
       <div class="list-group-item">

src-ui/src/app/components/common/input/date/date.component.html

@@ -2,7 +2,7 @@
   <label class="form-label" [for]="inputId">{{title}}</label>
   <div class="input-group" [class.is-invalid]="error">
     <input class="form-control" [class.is-invalid]="error" [placeholder]="placeholder" [id]="inputId" maxlength="10"
-          (dateSelect)="onChange(value)" (change)="onChange(value)" (keypress)="onKeyPress($event)"
+          (dateSelect)="onChange(value)" (change)="onChange(value)" (keypress)="onKeyPress($event)" (paste)="onPaste($event)"
           name="dp" [(ngModel)]="value" ngbDatepicker #datePicker="ngbDatepicker" #datePickerContent="ngModel">
     <button class="btn btn-outline-secondary calendar" (click)="datePicker.toggle()" type="button">
       <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-calendar" viewBox="0 0 16 16">

src-ui/src/app/components/common/input/date/date.component.ts

@@ -1,6 +1,8 @@
 import { Component, forwardRef, OnInit } from '@angular/core'
 import { NG_VALUE_ACCESSOR } from '@angular/forms'
+import { NgbDateParserFormatter } from '@ng-bootstrap/ng-bootstrap'
 import { SettingsService } from 'src/app/services/settings.service'
+import { LocalizedDateParserFormatter } from 'src/app/utils/ngb-date-parser-formatter'
 import { AbstractInputComponent } from '../abstract-input'
 
 @Component({
@@ -19,7 +21,10 @@ export class DateComponent
   extends AbstractInputComponent<string>
   implements OnInit
 {
-  constructor(private settings: SettingsService) {
+  constructor(
+    private settings: SettingsService,
+    private ngbDateParserFormatter: NgbDateParserFormatter
+  ) {
     super()
   }
 
@@ -30,7 +35,20 @@ export class DateComponent
 
   placeholder: string
 
-  // prevent chars other than numbers and separators
+  onPaste(event: ClipboardEvent) {
+    const clipboardData: DataTransfer =
+      event.clipboardData || window['clipboardData']
+    if (clipboardData) {
+      event.preventDefault()
+      let pastedText = clipboardData.getData('text')
+      pastedText = pastedText.replace(/[\sa-z#!$%\^&\*;:{}=\-_`~()]+/g, '')
+      const parsedDate = this.ngbDateParserFormatter.parse(pastedText)
+      const formattedDate = this.ngbDateParserFormatter.format(parsedDate)
+      this.writeValue(formattedDate)
+      this.onChange(formattedDate)
+    }
+  }
+
   onKeyPress(event: KeyboardEvent) {
     if ('Enter' !== event.key && !/[0-9,\.\/-]+/.test(event.key)) {
       event.preventDefault()

src-ui/src/app/components/common/input/select/select.component.html

@@ -9,7 +9,8 @@
         [items]="items"
         [addTag]="allowCreateNew && addItemRef"
         addTagText="Add item"
-        i18n-addTagText="Used for both types and correspondents"
+        i18n-addTagText="Used for both types, correspondents, storage paths"
+        [placeholder]="placeholder"
         bindLabel="name"
         bindValue="id"
         (change)="onChange(value)"

src-ui/src/app/components/common/input/select/select.component.ts

@@ -41,6 +41,9 @@ export class SelectComponent extends AbstractInputComponent<number> {
   @Input()
   suggestions: number[]
 
+  @Input()
+  placeholder: string
+
   @Output()
   createNew = new EventEmitter<string>()
 

src-ui/src/app/components/common/input/text/text.component.html

@@ -1,7 +1,7 @@
 <div class="mb-3">
   <label class="form-label" [for]="inputId">{{title}}</label>
   <input #inputField type="text" class="form-control" [class.is-invalid]="error" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)">
-  <small *ngIf="hint" class="form-text text-muted">{{hint}}</small>
+  <small *ngIf="hint" class="form-text text-muted" [innerHTML]="hint | safeHtml"></small>
   <div class="invalid-feedback">
     {{error}}
   </div>

src-ui/src/app/components/common/tag/tag.component.scss

@@ -1,3 +1,6 @@
 a {
     cursor: pointer;
+    white-space: normal;
+    word-break: break-word;
+    text-align: end;
 }

src-ui/src/app/components/common/toasts/toasts.component.html

@@ -4,5 +4,5 @@
   [class]="toast.classname"
   (hidden)="toastService.closeToast(toast)">
   <p>{{toast.content}}</p>
-  <p *ngIf="toast.action"><button class="btn btn-sm btn-outline-secondary" (click)="toastService.closeToast(toast); toast.action()">{{toast.actionName}}</button></p>
+  <p class="mb-0" *ngIf="toast.action"><button class="btn btn-sm btn-outline-secondary" (click)="toastService.closeToast(toast); toast.action()">{{toast.actionName}}</button></p>
 </ngb-toast>

src-ui/src/app/components/common/toasts/toasts.component.scss

@@ -8,4 +8,4 @@
 
 .toast:not(.show) {
   display: block; // this corrects an ng-bootstrap bug that prevented animations
-}
+}

src-ui/src/app/components/dashboard/dashboard.component.html

@@ -21,9 +21,14 @@
 
 <div class='row'>
   <div class="col-lg-8">
-    <app-welcome-widget *ngIf="savedViews.length == 0"></app-welcome-widget>
+    <ng-container *ngIf="savedViewService.loading">
+      <div class="spinner-border spinner-border-sm me-2" role="status"></div>
+      <ng-container i18n>Loading...</ng-container>
+    </ng-container>
 
-    <ng-container *ngFor="let v of savedViews">
+    <app-welcome-widget *ngIf="!savedViewService.loading && savedViewService.dashboardViews.length == 0"></app-welcome-widget>
+
+    <ng-container *ngFor="let v of savedViewService.dashboardViews">
       <app-saved-view-widget [savedView]="v"></app-saved-view-widget>
     </ng-container>
 

src-ui/src/app/components/dashboard/dashboard.component.ts

@@ -1,43 +1,24 @@
 import { Component, OnInit } from '@angular/core'
 import { Meta } from '@angular/platform-browser'
-import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'
 import { SavedViewService } from 'src/app/services/rest/saved-view.service'
+import { SettingsService } from 'src/app/services/settings.service'
 
 @Component({
   selector: 'app-dashboard',
   templateUrl: './dashboard.component.html',
   styleUrls: ['./dashboard.component.scss'],
 })
-export class DashboardComponent implements OnInit {
-  constructor(private savedViewService: SavedViewService, private meta: Meta) {}
-
-  get displayName() {
-    let tagFullName = this.meta.getTag('name=full_name')
-    let tagUsername = this.meta.getTag('name=username')
-    if (tagFullName && tagFullName.content) {
-      return tagFullName.content
-    } else if (tagUsername && tagUsername.content) {
-      return tagUsername.content
-    } else {
-      return null
-    }
-  }
+export class DashboardComponent {
+  constructor(
+    public savedViewService: SavedViewService,
+    public settingsService: SettingsService
+  ) {}
 
   get subtitle() {
-    if (this.displayName) {
-      return $localize`Hello ${this.displayName}, welcome to Paperless-ngx!`
+    if (this.settingsService.displayName) {
+      return $localize`Hello ${this.settingsService.displayName}, welcome to Paperless-ngx!`
     } else {
       return $localize`Welcome to Paperless-ngx!`
     }
   }
-
-  savedViews: PaperlessSavedView[] = []
-
-  ngOnInit(): void {
-    this.savedViewService.listAll().subscribe((results) => {
-      this.savedViews = results.results.filter(
-        (savedView) => savedView.show_on_dashboard
-      )
-    })
-  }
 }

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html

@@ -1,4 +1,4 @@
-<app-widget-frame [title]="savedView.name">
+<app-widget-frame [title]="savedView.name" [loading]="loading">
 
   <a class="btn-link" header-buttons [routerLink]="[]" (click)="showAll()" i18n>Show all</a>
 
@@ -11,8 +11,8 @@
       </tr>
     </thead>
     <tbody>
-      <tr *ngFor="let doc of documents" [routerLink]="['/', 'documents', doc.id]">
-        <td>{{doc.created | customDate}}</td>
+      <tr *ngFor="let doc of documents" (click)="openDocumentsService.openDocument(doc)">
+        <td>{{doc.created_date | customDate}}</td>
         <td>{{doc.title | documentTitle}}<app-tag [tag]="t" *ngFor="let t of doc.tags$ | async" class="ms-1" (click)="clickTag(t); $event.stopPropagation();"></app-tag></td>
       </tr>
     </tbody>

src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts

@@ -7,7 +7,8 @@ import { ConsumerStatusService } from 'src/app/services/consumer-status.service'
 import { DocumentService } from 'src/app/services/rest/document.service'
 import { PaperlessTag } from 'src/app/data/paperless-tag'
 import { FILTER_HAS_TAGS_ALL } from 'src/app/data/filter-rule-type'
-import { QueryParamsService } from 'src/app/services/query-params.service'
+import { OpenDocumentsService } from 'src/app/services/open-documents.service'
+import { DocumentListViewService } from 'src/app/services/document-list-view.service'
 
 @Component({
   selector: 'app-saved-view-widget',
@@ -15,11 +16,14 @@ import { QueryParamsService } from 'src/app/services/query-params.service'
   styleUrls: ['./saved-view-widget.component.scss'],
 })
 export class SavedViewWidgetComponent implements OnInit, OnDestroy {
+  loading: boolean = true
+
   constructor(
     private documentService: DocumentService,
     private router: Router,
-    private queryParamsService: QueryParamsService,
-    private consumerStatusService: ConsumerStatusService
+    private list: DocumentListViewService,
+    private consumerStatusService: ConsumerStatusService,
+    public openDocumentsService: OpenDocumentsService
   ) {}
 
   @Input()
@@ -43,6 +47,7 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy {
   }
 
   reload() {
+    this.loading = this.documents.length == 0
     this.documentService
       .listFiltered(
         1,
@@ -52,6 +57,7 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy {
         this.savedView.filter_rules
       )
       .subscribe((result) => {
+        this.loading = false
         this.documents = result.results
       })
   }
@@ -67,7 +73,7 @@ export class SavedViewWidgetComponent implements OnInit, OnDestroy {
   }
 
   clickTag(tag: PaperlessTag) {
-    this.queryParamsService.navigateWithFilterRules([
+    this.list.quickFilter([
       { rule_type: FILTER_HAS_TAGS_ALL, value: tag.id.toString() },
     ])
   }

src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html

@@ -1,4 +1,4 @@
-<app-widget-frame title="Statistics" i18n-title>
+<app-widget-frame title="Statistics" [loading]="loading" i18n-title>
   <ng-container content>
     <p class="card-text" i18n *ngIf="statistics?.documents_inbox != null">Documents in inbox: {{statistics?.documents_inbox}}</p>
     <p class="card-text" i18n>Total documents: {{statistics?.documents_total}}</p>

src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts

@@ -15,6 +15,8 @@ export interface Statistics {
   styleUrls: ['./statistics-widget.component.scss'],
 })
 export class StatisticsWidgetComponent implements OnInit, OnDestroy {
+  loading: boolean = true
+
   constructor(
     private http: HttpClient,
     private consumerStatusService: ConsumerStatusService
@@ -29,7 +31,9 @@ export class StatisticsWidgetComponent implements OnInit, OnDestroy {
   }
 
   reload() {
+    this.loading = true
     this.getStatistics().subscribe((statistics) => {
+      this.loading = false
       this.statistics = statistics
     })
   }

src-ui/src/app/components/dashboard/widgets/widget-frame/widget-frame.component.html

@@ -2,6 +2,10 @@
   <div class="card-header">
     <div class="d-flex justify-content-between align-items-center">
       <h5 class="card-title mb-0">{{title}}</h5>
+      <ng-container *ngIf="loading">
+        <div class="spinner-border spinner-border-sm fw-normal ms-2 me-auto" role="status"></div>
+        <div class="visually-hidden" i18n>Loading...</div>
+      </ng-container>
       <ng-content select ="[header-buttons]"></ng-content>
     </div>
 

src-ui/src/app/components/dashboard/widgets/widget-frame/widget-frame.component.ts

@@ -11,5 +11,8 @@ export class WidgetFrameComponent implements OnInit {
   @Input()
   title: string
 
+  @Input()
+  loading: boolean = false
+
   ngOnInit(): void {}
 }

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -8,7 +8,7 @@
     <button type="button" class="btn btn-sm btn-outline-danger me-2 ms-auto" (click)="delete()">
         <svg class="buttonicon" fill="currentColor">
             <use xlink:href="assets/bootstrap-icons.svg#trash" />
-        </svg>&nbsp;<span class="d-none d-lg-inline" i18n>Delete</span>
+        </svg><span class="d-none d-lg-inline ps-1" i18n>Delete</span>
     </button>
 
     <div class="btn-group me-2">
@@ -16,7 +16,7 @@
         <a [href]="downloadUrl" class="btn btn-sm btn-outline-primary">
             <svg class="buttonicon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#download" />
-            </svg>&nbsp;<span class="d-none d-lg-inline" i18n>Download</span>
+            </svg><span class="d-none d-lg-inline ps-1" i18n>Download</span>
         </a>
 
         <div class="btn-group" ngbDropdown role="group" *ngIf="metadata?.has_archive_version">
@@ -28,10 +28,16 @@
 
     </div>
 
+    <button type="button" class="btn btn-sm btn-outline-primary me-2" (click)="redoOcr()">
+        <svg class="buttonicon" fill="currentColor">
+            <use xlink:href="assets/bootstrap-icons.svg#arrow-counterclockwise" />
+        </svg><span class="d-none d-lg-inline ps-1" i18n>Redo OCR</span>
+    </button>
+
     <button type="button" class="btn btn-sm btn-outline-primary me-2" (click)="moreLike()">
         <svg class="buttonicon" fill="currentColor">
-            <use xlink:href="assets/bootstrap-icons.svg#three-dots" />
-        </svg>&nbsp;<span class="d-none d-lg-inline" i18n>More like this</span>
+            <use xlink:href="assets/bootstrap-icons.svg#diagram-3" />
+        </svg><span class="d-none d-lg-inline ps-1" i18n>More like this</span>
     </button>
 
     <button type="button" class="btn btn-sm btn-outline-primary me-2" i18n-title title="Close" (click)="close()">
@@ -68,11 +74,13 @@
 
                         <app-input-text #inputTitle i18n-title title="Title" formControlName="title" (keyup)="titleKeyUp($event)" [error]="error?.title"></app-input-text>
                         <app-input-number i18n-title title="Archive serial number" [error]="error?.archive_serial_number" formControlName='archive_serial_number'></app-input-number>
-                        <app-input-date i18n-title title="Date created" formControlName="created" [error]="error?.created"></app-input-date>
+                        <app-input-date i18n-title title="Date created" formControlName="created_date" [error]="error?.created_date"></app-input-date>
                         <app-input-select [items]="correspondents" i18n-title title="Correspondent" formControlName="correspondent" [allowNull]="true"
                             (createNew)="createCorrespondent($event)" [suggestions]="suggestions?.correspondents"></app-input-select>
                         <app-input-select [items]="documentTypes" i18n-title title="Document type" formControlName="document_type" [allowNull]="true"
                             (createNew)="createDocumentType($event)" [suggestions]="suggestions?.document_types"></app-input-select>
+                        <app-input-select [items]="storagePaths" i18n-title title="Storage path" formControlName="storage_path" [allowNull]="true"
+                            (createNew)="createStoragePath($event)" [suggestions]="suggestions?.storage_paths" i18n-placeholder placeholder="Default"></app-input-select>
                         <app-input-tags formControlName="tags" [suggestions]="suggestions?.tags"></app-input-tags>
 
                     </ng-template>

src-ui/src/app/components/document-detail/document-detail.component.scss

@@ -14,10 +14,14 @@
 }
 
 ::ng-deep .ng2-pdf-viewer-container .page {
-  --page-margin: 1px 0 -8px;
+  --page-margin: 1px 0 10px;
   width: 100% !important;
 }
 
+::ng-deep .ng2-pdf-viewer-container .page:last-child {
+  --page-margin: 1px 0 20px;
+}
+
 .password-prompt {
   position: absolute;
   top: 30%;

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -18,10 +18,7 @@ import { DocumentTypeEditDialogComponent } from '../common/edit-dialog/document-
 import { PDFDocumentProxy } from 'ng2-pdf-viewer'
 import { ToastService } from 'src/app/services/toast.service'
 import { TextComponent } from '../common/input/text/text.component'
-import {
-  SettingsService,
-  SETTINGS_KEYS,
-} from 'src/app/services/settings.service'
+import { SettingsService } from 'src/app/services/settings.service'
 import { dirtyCheck, DirtyComponent } from '@ngneat/dirty-check-forms'
 import { Observable, Subject, BehaviorSubject } from 'rxjs'
 import {
@@ -34,8 +31,10 @@ import {
 } from 'rxjs/operators'
 import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions'
 import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type'
-import { normalizeDateStr } from 'src/app/utils/date'
-import { QueryParamsService } from 'src/app/services/query-params.service'
+import { StoragePathService } from 'src/app/services/rest/storage-path.service'
+import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
+import { StoragePathEditDialogComponent } from '../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
+import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings'
 
 @Component({
   selector: 'app-document-detail',
@@ -68,13 +67,15 @@ export class DocumentDetailComponent
 
   correspondents: PaperlessCorrespondent[]
   documentTypes: PaperlessDocumentType[]
+  storagePaths: PaperlessStoragePath[]
 
   documentForm: FormGroup = new FormGroup({
     title: new FormControl(''),
     content: new FormControl(''),
-    created: new FormControl(),
+    created_date: new FormControl(),
     correspondent: new FormControl(),
     document_type: new FormControl(),
+    storage_path: new FormControl(),
     archive_serial_number: new FormControl(),
     tags: new FormControl([]),
   })
@@ -85,6 +86,7 @@ export class DocumentDetailComponent
   store: BehaviorSubject<any>
   isDirty$: Observable<boolean>
   unsubscribeNotifier: Subject<any> = new Subject()
+  docChangeNotifier: Subject<any> = new Subject()
 
   requiresPassword: boolean = false
   password: string
@@ -116,19 +118,8 @@ export class DocumentDetailComponent
     private documentTitlePipe: DocumentTitlePipe,
     private toastService: ToastService,
     private settings: SettingsService,
-    private queryParamsService: QueryParamsService
-  ) {
-    this.titleSubject
-      .pipe(
-        debounceTime(1000),
-        distinctUntilChanged(),
-        takeUntil(this.unsubscribeNotifier)
-      )
-      .subscribe((titleValue) => {
-        this.title = titleValue
-        this.documentForm.patchValue({ title: titleValue })
-      })
-  }
+    private storagePathService: StoragePathService
+  ) {}
 
   titleKeyUp(event) {
     this.titleSubject.next(event.target?.value)
@@ -147,27 +138,8 @@ export class DocumentDetailComponent
   ngOnInit(): void {
     this.documentForm.valueChanges
       .pipe(takeUntil(this.unsubscribeNotifier))
-      .subscribe((changes) => {
+      .subscribe(() => {
         this.error = null
-        if (this.ogDate) {
-          try {
-            let newDate = new Date(normalizeDateStr(changes['created']))
-            newDate.setHours(
-              this.ogDate.getHours(),
-              this.ogDate.getMinutes(),
-              this.ogDate.getSeconds(),
-              this.ogDate.getMilliseconds()
-            )
-            this.documentForm.patchValue(
-              { created: newDate.toISOString() },
-              { emitEvent: false }
-            )
-          } catch (e) {
-            // catch this before we try to save and simulate an api error
-            this.error = { created: e.message }
-          }
-        }
-
         Object.assign(this.document, this.documentForm.value)
       })
 
@@ -175,15 +147,23 @@ export class DocumentDetailComponent
       .listAll()
       .pipe(first())
       .subscribe((result) => (this.correspondents = result.results))
+
     this.documentTypeService
       .listAll()
       .pipe(first())
       .subscribe((result) => (this.documentTypes = result.results))
 
+    this.storagePathService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.storagePaths = result.results))
+
     this.route.paramMap
       .pipe(
+        takeUntil(this.unsubscribeNotifier),
         switchMap((paramMap) => {
           const documentId = +paramMap.get('id')
+          this.docChangeNotifier.next(documentId)
           return this.documentsService.get(documentId)
         })
       )
@@ -204,29 +184,45 @@ export class DocumentDetailComponent
               this.openDocumentService.getOpenDocument(this.documentId)
             )
           } else {
-            this.openDocumentService.openDocument(doc)
+            this.openDocumentService.openDocument(doc, false)
             this.updateComponent(doc)
           }
 
-          this.ogDate = new Date(normalizeDateStr(doc.created.toString()))
+          this.titleSubject
+            .pipe(
+              debounceTime(1000),
+              distinctUntilChanged(),
+              takeUntil(this.docChangeNotifier),
+              takeUntil(this.unsubscribeNotifier)
+            )
+            .subscribe({
+              next: (titleValue) => {
+                this.title = titleValue
+                this.documentForm.patchValue({ title: titleValue })
+              },
+              complete: () => {
+                // doc changed so we manually check dirty in case title was changed
+                if (
+                  this.store.getValue().title !==
+                  this.documentForm.get('title').value
+                ) {
+                  this.openDocumentService.setDirty(doc.id, true)
+                }
+              },
+            })
 
           // Initialize dirtyCheck
           this.store = new BehaviorSubject({
             title: doc.title,
             content: doc.content,
-            created: this.ogDate.toISOString(),
+            created_date: doc.created_date,
             correspondent: doc.correspondent,
             document_type: doc.document_type,
+            storage_path: doc.storage_path,
             archive_serial_number: doc.archive_serial_number,
             tags: [...doc.tags],
           })
 
-          // start with ISO8601 string
-          this.documentForm.patchValue(
-            { created: this.ogDate.toISOString() },
-            { emitEvent: false }
-          )
-
           this.isDirty$ = dirtyCheck(
             this.documentForm,
             this.store.asObservable()
@@ -235,7 +231,6 @@ export class DocumentDetailComponent
           return this.isDirty$.pipe(map((dirty) => ({ doc, dirty })))
         })
       )
-      .pipe(takeUntil(this.unsubscribeNotifier))
       .subscribe({
         next: ({ doc, dirty }) => {
           this.openDocumentService.setDirty(doc.id, dirty)
@@ -324,6 +319,27 @@ export class DocumentDetailComponent
       })
   }
 
+  createStoragePath(newName: string) {
+    var modal = this.modalService.open(StoragePathEditDialogComponent, {
+      backdrop: 'static',
+    })
+    modal.componentInstance.dialogMode = 'create'
+    if (newName) modal.componentInstance.object = { name: newName }
+    modal.componentInstance.success
+      .pipe(
+        switchMap((newStoragePath) => {
+          return this.storagePathService
+            .listAll()
+            .pipe(map((storagePaths) => ({ newStoragePath, storagePaths })))
+        })
+      )
+      .pipe(takeUntil(this.unsubscribeNotifier))
+      .subscribe(({ newStoragePath, documentTypes: storagePaths }) => {
+        this.storagePaths = storagePaths.results
+        this.documentForm.get('storage_path').setValue(newStoragePath.id)
+      })
+  }
+
   discard() {
     this.documentsService
       .get(this.documentId)
@@ -448,7 +464,7 @@ export class DocumentDetailComponent
   }
 
   moreLike() {
-    this.queryParamsService.navigateWithFilterRules([
+    this.documentListViewService.quickFilter([
       {
         rule_type: FILTER_FULLTEXT_MORELIKE,
         value: this.documentId.toString(),
@@ -456,6 +472,42 @@ export class DocumentDetailComponent
     ])
   }
 
+  redoOcr() {
+    let modal = this.modalService.open(ConfirmDialogComponent, {
+      backdrop: 'static',
+    })
+    modal.componentInstance.title = $localize`Redo OCR confirm`
+    modal.componentInstance.messageBold = $localize`This operation will permanently redo OCR for this document.`
+    modal.componentInstance.message = $localize`This operation cannot be undone.`
+    modal.componentInstance.btnClass = 'btn-danger'
+    modal.componentInstance.btnCaption = $localize`Proceed`
+    modal.componentInstance.confirmClicked.subscribe(() => {
+      modal.componentInstance.buttonsEnabled = false
+      this.documentsService
+        .bulkEdit([this.document.id], 'redo_ocr', {})
+        .subscribe({
+          next: () => {
+            this.toastService.showInfo(
+              $localize`Redo OCR operation will begin in the background.`
+            )
+            if (modal) {
+              modal.close()
+            }
+          },
+          error: (error) => {
+            if (modal) {
+              modal.componentInstance.buttonsEnabled = true
+            }
+            this.toastService.showError(
+              $localize`Error executing operation: ${JSON.stringify(
+                error.error
+              )}`
+            )
+          },
+        })
+    })
+  }
+
   hasNext() {
     return this.documentListViewService.hasNext(this.documentId)
   }

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html

@@ -53,27 +53,43 @@
         [(selectionModel)]="documentTypeSelectionModel"
         (apply)="setDocumentTypes($event)">
       </app-filterable-dropdown>
+      <app-filterable-dropdown class="me-2 me-md-3" title="Storage path" icon="folder-fill" i18n-title
+        filterPlaceholder="Filter storage paths" i18n-filterPlaceholder
+        [items]="storagePaths"
+        [editing]="true"
+        [applyOnClose]="applyOnClose"
+        (open)="openStoragePathDropdown()"
+        [(selectionModel)]="storagePathsSelectionModel"
+        (apply)="setStoragePaths($event)">
+      </app-filterable-dropdown>
     </div>
   </div>
   <div class="col-auto ms-auto mb-2 mb-xl-0 d-flex">
     <div class="btn-group btn-group-sm me-2">
-      <button type="button" [disabled]="awaitingDownload" class="btn btn-outline-primary btn-sm" (click)="downloadSelected()">
-        <svg *ngIf="!awaitingDownload" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor">
-          <use xlink:href="assets/bootstrap-icons.svg#download" />
-        </svg>
-        <div *ngIf="awaitingDownload" class="spinner-border spinner-border-sm" role="status">
-          <span class="visually-hidden">Preparing download...</span>
-        </div>
-        &nbsp;
-        <ng-container i18n>Download</ng-container>
-      </button>
-      <div class="btn-group" ngbDropdown role="group" aria-label="Button group with nested dropdown">
-        <button [disabled]="awaitingDownload" class="btn btn-outline-primary btn-sm dropdown-toggle-split" ngbDropdownToggle></button>
-        <div class="dropdown-menu shadow" ngbDropdownMenu>
-          <button ngbDropdownItem i18n (click)="downloadSelected('originals')">Download originals</button>
+
+      <div ngbDropdown class="me-2 d-flex">
+        <button class="btn btn-sm btn-outline-primary" id="dropdownSelect" ngbDropdownToggle>
+          <svg class="toolbaricon" fill="currentColor">
+            <use xlink:href="assets/bootstrap-icons.svg#three-dots" />
+          </svg>
+          <div class="d-none d-sm-inline">&nbsp;<ng-container i18n>Actions</ng-container></div>
+        </button>
+        <div ngbDropdownMenu aria-labelledby="dropdownSelect" class="shadow">
+          <button ngbDropdownItem [disabled]="awaitingDownload" (click)="downloadSelected()" i18n>
+            Download
+            <div *ngIf="awaitingDownload" class="spinner-border spinner-border-sm" role="status">
+              <span class="visually-hidden">Preparing download...</span>
+            </div>
+          </button>
+          <button ngbDropdownItem [disabled]="awaitingDownload" (click)="downloadSelected('originals')" i18n>
+            Download originals
+            <div *ngIf="awaitingDownload" class="spinner-border spinner-border-sm" role="status">
+              <span class="visually-hidden">Preparing download...</span>
+            </div>
+          </button>
+          <button ngbDropdownItem (click)="redoOcrSelected()" i18n>Redo OCR</button>
         </div>
       </div>
-    </div>
 
     <button type="button" class="btn btn-sm btn-outline-danger" (click)="applyDelete()">
       <svg width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor">

src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.ts

@@ -19,12 +19,12 @@ import {
 } from '../../common/filterable-dropdown/filterable-dropdown.component'
 import { ToggleableItemState } from '../../common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component'
 import { MatchingModel } from 'src/app/data/matching-model'
-import {
-  SettingsService,
-  SETTINGS_KEYS,
-} from 'src/app/services/settings.service'
+import { SettingsService } from 'src/app/services/settings.service'
 import { ToastService } from 'src/app/services/toast.service'
 import { saveAs } from 'file-saver'
+import { StoragePathService } from 'src/app/services/rest/storage-path.service'
+import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
+import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings'
 
 @Component({
   selector: 'app-bulk-editor',
@@ -35,10 +35,12 @@ export class BulkEditorComponent {
   tags: PaperlessTag[]
   correspondents: PaperlessCorrespondent[]
   documentTypes: PaperlessDocumentType[]
+  storagePaths: PaperlessStoragePath[]
 
   tagSelectionModel = new FilterableDropdownSelectionModel()
   correspondentSelectionModel = new FilterableDropdownSelectionModel()
   documentTypeSelectionModel = new FilterableDropdownSelectionModel()
+  storagePathsSelectionModel = new FilterableDropdownSelectionModel()
   awaitingDownload: boolean
 
   constructor(
@@ -50,7 +52,8 @@ export class BulkEditorComponent {
     private modalService: NgbModal,
     private openDocumentService: OpenDocumentsService,
     private settings: SettingsService,
-    private toastService: ToastService
+    private toastService: ToastService,
+    private storagePathService: StoragePathService
   ) {}
 
   applyOnClose: boolean = this.settings.get(
@@ -70,6 +73,9 @@ export class BulkEditorComponent {
     this.documentTypeService
       .listAll()
       .subscribe((result) => (this.documentTypes = result.results))
+    this.storagePathService
+      .listAll()
+      .subscribe((result) => (this.storagePaths = result.results))
   }
 
   private executeBulkOperation(modal, method: string, args) {
@@ -147,6 +153,17 @@ export class BulkEditorComponent {
       })
   }
 
+  openStoragePathDropdown() {
+    this.documentService
+      .getSelectionData(Array.from(this.list.selected))
+      .subscribe((s) => {
+        this.applySelectionData(
+          s.selected_storage_paths,
+          this.storagePathsSelectionModel
+        )
+      })
+  }
+
   private _localizeList(items: MatchingModel[]) {
     if (items.length == 0) {
       return ''
@@ -301,6 +318,42 @@ export class BulkEditorComponent {
     }
   }
 
+  setStoragePaths(changedDocumentPaths: ChangedItems) {
+    if (
+      changedDocumentPaths.itemsToAdd.length == 0 &&
+      changedDocumentPaths.itemsToRemove.length == 0
+    )
+      return
+
+    let storagePath =
+      changedDocumentPaths.itemsToAdd.length > 0
+        ? changedDocumentPaths.itemsToAdd[0]
+        : null
+
+    if (this.showConfirmationDialogs) {
+      let modal = this.modalService.open(ConfirmDialogComponent, {
+        backdrop: 'static',
+      })
+      modal.componentInstance.title = $localize`Confirm storage path assignment`
+      if (storagePath) {
+        modal.componentInstance.message = $localize`This operation will assign the storage path "${storagePath.name}" to ${this.list.selected.size} selected document(s).`
+      } else {
+        modal.componentInstance.message = $localize`This operation will remove the storage path from ${this.list.selected.size} selected document(s).`
+      }
+      modal.componentInstance.btnClass = 'btn-warning'
+      modal.componentInstance.btnCaption = $localize`Confirm`
+      modal.componentInstance.confirmClicked.subscribe(() => {
+        this.executeBulkOperation(modal, 'set_storage_path', {
+          storage_path: storagePath ? storagePath.id : null,
+        })
+      })
+    } else {
+      this.executeBulkOperation(null, 'set_storage_path', {
+        storage_path: storagePath ? storagePath.id : null,
+      })
+    }
+  }
+
   applyDelete() {
     let modal = this.modalService.open(ConfirmDialogComponent, {
       backdrop: 'static',
@@ -326,4 +379,19 @@ export class BulkEditorComponent {
         this.awaitingDownload = false
       })
   }
+
+  redoOcrSelected() {
+    let modal = this.modalService.open(ConfirmDialogComponent, {
+      backdrop: 'static',
+    })
+    modal.componentInstance.title = $localize`Redo OCR confirm`
+    modal.componentInstance.messageBold = $localize`This operation will permanently redo OCR for ${this.list.selected.size} selected document(s).`
+    modal.componentInstance.message = $localize`This operation cannot be undone.`
+    modal.componentInstance.btnClass = 'btn-danger'
+    modal.componentInstance.btnCaption = $localize`Proceed`
+    modal.componentInstance.confirmClicked.subscribe(() => {
+      modal.componentInstance.buttonsEnabled = false
+      this.executeBulkOperation(modal, 'redo_ocr', {})
+    })
+  }
 }