Updates the version strings to 1.10.0

New Crowdin updates

.build-config.json

@@ -1,6 +1,6 @@
 {
   "qpdf": {
-      "version": "10.6.3"
+      "version": "11.1.1"
     },
   "jbig2enc": {
       "version": "0.29",

.editorconfig

@@ -27,6 +27,9 @@ indent_style = space
 [*.md]
 indent_style = space
 
+[Pipfile.lock]
+indent_style = space
+
 # Tests don't get a line width restriction.  It's still a good idea to follow
 # the 79 character rule, but in the interests of clarity, tests often need to
 # violate it.

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -13,6 +13,7 @@ body:
         - [The troubleshooting documentation](https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html).
         - [The installation instructions](https://paperless-ngx.readthedocs.io/en/latest/setup.html#installation).
         - [Existing issues and discussions](https://github.com/paperless-ngx/paperless-ngx/search?q=&type=issues).
+        - Disable any customer container initialization scripts, if using any
 
         If you encounter issues while installing or configuring Paperless-ngx, please post in the ["Support" section of the discussions](https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=support).
   - type: textarea
@@ -41,7 +42,15 @@ body:
     id: logs
     attributes:
       label: Webserver logs
-      description: If available, post any logs from the web server related to your issue.
+      description: Logs from the web server related to your issue.
+      render: bash
+    validations:
+      required: true
+  - type: textarea
+    id: logs_browser
+    attributes:
+      label: Browser logs
+      description: Logs from the web browser related to your issue, if needed
       render: bash
   - type: input
     id: version
@@ -63,9 +72,11 @@ body:
     attributes:
       label: Installation method
       options:
-        - Docker
+        - Docker - official image
+        - Docker - linuxserver.io image
         - Bare metal
         - Other (please describe above)
+      description: Note there are significant differences from the official image and linuxserver.io, please check if your issue is specific to the third-party image.
     validations:
       required: true
   - type: input

.github/release-drafter.yml

@@ -1,3 +1,14 @@
+autolabeler:
+  - label: "bug"
+    branch:
+      - '/^fix/'
+    title:
+      - "/^fix/i"
+  - label: "enhancement"
+    branch:
+      - '/^feature/'
+    title:
+      - "/^feature/i"
 categories:
   - title: 'Breaking Changes'
     labels:
@@ -15,9 +26,15 @@ categories:
       - 'chore'
       - 'deployment'
       - 'translation'
+      - 'ci-cd'
   - title: 'Dependencies'
     collapse-after: 3
     label: 'dependencies'
+  - title: 'All App Changes'
+    labels:
+      - 'frontend'
+      - 'backend'
+    collapse-after: 0
 include-labels:
   - 'enhancement'
   - 'bug'
@@ -25,11 +42,12 @@ include-labels:
   - 'deployment'
   - 'translation'
   - 'dependencies'
-replacers: # Changes "Feature: Update checker" to "Update checker"
-  - search: '/Feature:|Feat:|\[feature\]/gi'
-    replace: ''
+  - 'documentation'
+  - 'frontend'
+  - 'backend'
+  - 'ci-cd'
 category-template: '### $TITLE'
-change-template: '- $TITLE [@$AUTHOR](https://github.com/$AUTHOR) ([#$NUMBER]($URL))'
+change-template: '- $TITLE @$AUTHOR ([#$NUMBER]($URL))'
 change-title-escapes: '\<*_&#@'
 template: |
   ## paperless-ngx $RESOLVED_VERSION

.github/scripts/cleanup-tags.py

@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+import json
+import logging
+import os
+import shutil
+import subprocess
+from argparse import ArgumentParser
+from typing import Dict
+from typing import Final
+from typing import List
+from typing import Optional
+
+from common import get_log_level
+from github import ContainerPackage
+from github import GithubBranchApi
+from github import GithubContainerRegistryApi
+
+logger = logging.getLogger("cleanup-tags")
+
+
+class DockerManifest2:
+    """
+    Data class wrapping the Docker Image Manifest Version 2.
+
+    See https://docs.docker.com/registry/spec/manifest-v2-2/
+    """
+
+    def __init__(self, data: Dict) -> None:
+        self._data = data
+        # This is the sha256: digest string.  Corresponds to GitHub API name
+        # if the package is an untagged package
+        self.digest = self._data["digest"]
+        platform_data_os = self._data["platform"]["os"]
+        platform_arch = self._data["platform"]["architecture"]
+        platform_variant = self._data["platform"].get(
+            "variant",
+            "",
+        )
+        self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
+
+
+class RegistryTagsCleaner:
+    """
+    This is the base class for the image registry cleaning.  Given a package
+    name, it will keep all images which are tagged and all untagged images
+    referred to by a manifest.  This results in only images which have been untagged
+    and cannot be referenced except by their SHA in being removed.  None of these
+    images should be referenced, so it is fine to delete them.
+    """
+
+    def __init__(
+        self,
+        package_name: str,
+        repo_owner: str,
+        repo_name: str,
+        package_api: GithubContainerRegistryApi,
+        branch_api: Optional[GithubBranchApi],
+    ):
+        self.actually_delete = False
+        self.package_api = package_api
+        self.branch_api = branch_api
+        self.package_name = package_name
+        self.repo_owner = repo_owner
+        self.repo_name = repo_name
+        self.tags_to_delete: List[str] = []
+        self.tags_to_keep: List[str] = []
+
+        # Get the information about all versions of the given package
+        # These are active, not deleted, the default returned from the API
+        self.all_package_versions = self.package_api.get_active_package_versions(
+            self.package_name,
+        )
+
+        # Get a mapping from a tag like "1.7.0" or "feature-xyz" to the ContainerPackage
+        # tagged with it.  It makes certain lookups easy
+        self.all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
+        for pkg in self.all_package_versions:
+            for tag in pkg.tags:
+                self.all_pkgs_tags_to_version[tag] = pkg
+        logger.info(
+            f"Located {len(self.all_package_versions)} versions of package {self.package_name}",
+        )
+
+        self.decide_what_tags_to_keep()
+
+    def clean(self):
+        """
+        This method will delete image versions, based on the selected tags to delete
+        """
+        for tag_to_delete in self.tags_to_delete:
+            package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
+
+            if self.actually_delete:
+                logger.info(
+                    f"Deleting {tag_to_delete} (id {package_version_info.id})",
+                )
+                self.package_api.delete_package_version(
+                    package_version_info,
+                )
+
+            else:
+                logger.info(
+                    f"Would delete {tag_to_delete} (id {package_version_info.id})",
+                )
+        else:
+            logger.info("No tags to delete")
+
+    def clean_untagged(self, is_manifest_image: bool):
+        """
+        This method will delete untagged images, that is those which are not named.  It
+        handles if the image tag is actually a manifest, which points to images that look otherwise
+        untagged.
+        """
+
+        def _clean_untagged_manifest():
+            """
+
+            Handles the deletion of untagged images, but where the package is a manifest, ie a multi
+            arch image, which means some "untagged" images need to exist still.
+
+            Ok, bear with me, these are annoying.
+
+            Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
+            These images are untagged, but pointed to, and so should not be removed (or every pull fails).
+
+            So for each image getting kept, parse the manifest to find the digest(s) it points to.  Then
+            remove those from the list of untagged images.  The final result is the untagged, not pointed to
+            version which should be safe to remove.
+
+            Example:
+                Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
+                    amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
+                    armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
+                    arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
+                each of which appears as untagged image, but isn't really.
+
+                So from the list of untagged packages, remove those digests.  Once all tags which
+                are being kept are checked, the remaining untagged packages are actually untagged
+                with no referrals in a manifest to them.
+            """
+            # Simplify the untagged data, mapping name (which is a digest) to the version
+            # At the moment, these are the images which APPEAR untagged.
+            untagged_versions = {}
+            for x in self.all_package_versions:
+                if x.untagged:
+                    untagged_versions[x.name] = x
+
+            skips = 0
+
+            # Parse manifests to locate digests pointed to
+            for tag in sorted(self.tags_to_keep):
+                full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
+                logger.info(f"Checking manifest for {full_name}")
+                try:
+                    proc = subprocess.run(
+                        [
+                            shutil.which("docker"),
+                            "manifest",
+                            "inspect",
+                            full_name,
+                        ],
+                        capture_output=True,
+                    )
+
+                    manifest_list = json.loads(proc.stdout)
+                    for manifest_data in manifest_list["manifests"]:
+                        manifest = DockerManifest2(manifest_data)
+
+                        if manifest.digest in untagged_versions:
+                            logger.info(
+                                f"Skipping deletion of {manifest.digest},"
+                                f" referred to by {full_name}"
+                                f" for {manifest.platform}",
+                            )
+                            del untagged_versions[manifest.digest]
+                            skips += 1
+
+                except Exception as err:
+                    self.actually_delete = False
+                    logger.exception(err)
+                    return
+
+            logger.info(
+                f"Skipping deletion of {skips} packages referred to by a manifest",
+            )
+
+            # Delete the untagged and not pointed at packages
+            logger.info(f"Deleting untagged packages of {self.package_name}")
+            for to_delete_name in untagged_versions:
+                to_delete_version = untagged_versions[to_delete_name]
+
+                if self.actually_delete:
+                    logger.info(
+                        f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
+                    )
+                    self.package_api.delete_package_version(
+                        to_delete_version,
+                    )
+                else:
+                    logger.info(
+                        f"Would delete {to_delete_name} (id {to_delete_version.id})",
+                    )
+
+        def _clean_untagged_non_manifest():
+            """
+            If the package is not a multi-arch manifest, images without tags are safe to delete.
+            """
+
+            for package in self.all_package_versions:
+                if package.untagged:
+                    if self.actually_delete:
+                        logger.info(
+                            f"Deleting id {package.id} named {package.name}",
+                        )
+                        self.package_api.delete_package_version(
+                            package,
+                        )
+                    else:
+                        logger.info(
+                            f"Would delete {package.name} (id {package.id})",
+                        )
+                else:
+                    logger.info(
+                        f"Not deleting tag {package.tags[0]} of package {self.package_name}",
+                    )
+
+        logger.info("Beginning untagged image cleaning")
+
+        if is_manifest_image:
+            _clean_untagged_manifest()
+        else:
+            _clean_untagged_non_manifest()
+
+    def decide_what_tags_to_keep(self):
+        """
+        This method holds the logic to delete what tags to keep and there fore
+        what tags to delete.
+
+        By default, any image with at least 1 tag will be kept
+        """
+        # By default, keep anything which is tagged
+        self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
+
+
+class MainImageTagsCleaner(RegistryTagsCleaner):
+    def decide_what_tags_to_keep(self):
+        """
+        Overrides the default logic for deciding what images to keep.  Images tagged as "feature-"
+        will be removed, if the corresponding branch no longer exists.
+        """
+
+        # Default to everything gets kept still
+        super().decide_what_tags_to_keep()
+
+        # Locate the feature branches
+        feature_branches = {}
+        for branch in self.branch_api.get_branches(
+            repo=self.repo_name,
+        ):
+            if branch.name.startswith("feature-"):
+                logger.debug(f"Found feature branch {branch.name}")
+                feature_branches[branch.name] = branch
+
+        logger.info(f"Located {len(feature_branches)} feature branches")
+
+        if not len(feature_branches):
+            # Our work here is done, delete nothing
+            return
+
+        # Filter to packages which are tagged with feature-*
+        packages_tagged_feature: List[ContainerPackage] = []
+        for package in self.all_package_versions:
+            if package.tag_matches("feature-"):
+                packages_tagged_feature.append(package)
+
+        # Map tags like "feature-xyz" to a ContainerPackage
+        feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
+        for pkg in packages_tagged_feature:
+            for tag in pkg.tags:
+                feature_pkgs_tags_to_versions[tag] = pkg
+
+        logger.info(
+            f'Located {len(feature_pkgs_tags_to_versions)} versions of package {self.package_name} tagged "feature-"',
+        )
+
+        # All the feature tags minus all the feature branches leaves us feature tags
+        # with no corresponding branch
+        self.tags_to_delete = list(
+            set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
+        )
+
+        # All the tags minus the set of going to be deleted tags leaves us the
+        # tags which will be kept around
+        self.tags_to_keep = list(
+            set(self.all_pkgs_tags_to_version.keys()) - set(self.tags_to_delete),
+        )
+        logger.info(
+            f"Located {len(self.tags_to_delete)} versions of package {self.package_name} to delete",
+        )
+
+
+class LibraryTagsCleaner(RegistryTagsCleaner):
+    """
+    Exists for the off change that someday, the installer library images
+    will need their own logic
+    """
+
+    pass
+
+
+def _main():
+    parser = ArgumentParser(
+        description="Using the GitHub API locate and optionally delete container"
+        " tags which no longer have an associated feature branch",
+    )
+
+    # Requires an affirmative command to actually do a delete
+    parser.add_argument(
+        "--delete",
+        action="store_true",
+        default=False,
+        help="If provided, actually delete the container tags",
+    )
+
+    # When a tagged image is updated, the previous version remains, but it no longer tagged
+    # Add this option to remove them as well
+    parser.add_argument(
+        "--untagged",
+        action="store_true",
+        default=False,
+        help="If provided, delete untagged containers as well",
+    )
+
+    # If given, the package is assumed to be a multi-arch manifest.  Cache packages are
+    # not multi-arch, all other types are
+    parser.add_argument(
+        "--is-manifest",
+        action="store_true",
+        default=False,
+        help="If provided, the package is assumed to be a multi-arch manifest following schema v2",
+    )
+
+    # Allows configuration of log level for debugging
+    parser.add_argument(
+        "--loglevel",
+        default="info",
+        help="Configures the logging level",
+    )
+
+    # Get the name of the package being processed this round
+    parser.add_argument(
+        "package",
+        help="The package to process",
+    )
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=get_log_level(args),
+        datefmt="%Y-%m-%d %H:%M:%S",
+        format="%(asctime)s %(levelname)-8s %(message)s",
+    )
+
+    # Must be provided in the environment
+    repo_owner: Final[str] = os.environ["GITHUB_REPOSITORY_OWNER"]
+    repo: Final[str] = os.environ["GITHUB_REPOSITORY"]
+    gh_token: Final[str] = os.environ["TOKEN"]
+
+    # Find all branches named feature-*
+    # Note: Only relevant to the main application, but simpler to
+    # leave in for all packages
+    with GithubBranchApi(gh_token) as branch_api:
+        with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
+            if args.package in {"paperless-ngx", "paperless-ngx/builder/cache/app"}:
+                cleaner = MainImageTagsCleaner(
+                    args.package,
+                    repo_owner,
+                    repo,
+                    container_api,
+                    branch_api,
+                )
+            else:
+                cleaner = LibraryTagsCleaner(
+                    args.package,
+                    repo_owner,
+                    repo,
+                    container_api,
+                    None,
+                )
+
+            # Set if actually doing a delete vs dry run
+            cleaner.actually_delete = args.delete
+
+            # Clean images with tags
+            cleaner.clean()
+
+            # Clean images which are untagged
+            cleaner.clean_untagged(args.is_manifest)
+
+
+if __name__ == "__main__":
+    _main()

.github/scripts/common.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import logging
 
 
 def get_image_tag(
@@ -9,7 +10,7 @@ def get_image_tag(
     """
     Returns a string representing the normal image for a given package
     """
-    return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+    return f"ghcr.io/{repo_name.lower()}/builder/{pkg_name}:{pkg_version}"
 
 
 def get_cache_image_tag(
@@ -24,4 +25,24 @@ def get_cache_image_tag(
     Registry type caching is utilized for the builder images, to allow fast
     rebuilds, generally almost instant for the same version
     """
-    return f"ghcr.io/{repo_name}/builder/cache/{pkg_name}:{pkg_version}"
+    return f"ghcr.io/{repo_name.lower()}/builder/cache/{pkg_name}:{pkg_version}"
+
+
+def get_log_level(args) -> int:
+    """
+    Returns a logging level, based
+    :param args:
+    :return:
+    """
+    levels = {
+        "critical": logging.CRITICAL,
+        "error": logging.ERROR,
+        "warn": logging.WARNING,
+        "warning": logging.WARNING,
+        "info": logging.INFO,
+        "debug": logging.DEBUG,
+    }
+    level = levels.get(args.loglevel.lower())
+    if level is None:
+        level = logging.INFO
+    return level

.github/scripts/get-build-json.py

@@ -50,7 +50,6 @@ def _main():
 
     # Default output values
     version = None
-    git_tag = None
     extra_config = {}
 
     if args.package in pipfile_data["default"]:
@@ -59,12 +58,6 @@ def _main():
         pkg_version = pkg_data["version"].split("==")[-1]
         version = pkg_version
 
-        # Based on the package, generate the expected Git tag name
-        if args.package == "pikepdf":
-            git_tag = f"v{pkg_version}"
-        elif args.package == "psycopg2":
-            git_tag = pkg_version.replace(".", "_")
-
         # Any extra/special values needed
         if args.package == "pikepdf":
             extra_config["qpdf_version"] = build_json["qpdf"]["version"]
@@ -72,8 +65,6 @@ def _main():
     elif args.package in build_json:
         version = build_json[args.package]["version"]
 
-        if "git_tag" in build_json[args.package]:
-            git_tag = build_json[args.package]["git_tag"]
     else:
         raise NotImplementedError(args.package)
 
@@ -81,7 +72,6 @@ def _main():
     output = {
         "name": args.package,
         "version": version,
-        "git_tag": git_tag,
         "image_tag": get_image_tag(repo_name, args.package, version),
         "cache_tag": get_cache_image_tag(
             repo_name,

.github/scripts/github.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""
+This module contains some useful classes for interacting with the Github API.
+The full documentation for the API can be found here: https://docs.github.com/en/rest
+
+Mostly, this focusses on two areas, repo branches and repo packages, as the use case
+is cleaning up container images which are no longer referred to.
+
+"""
+import functools
+import logging
+import re
+import urllib.parse
+from typing import Dict
+from typing import List
+from typing import Optional
+
+import httpx
+
+logger = logging.getLogger("github-api")
+
+
+class _GithubApiBase:
+    """
+    A base class for interacting with the Github API.  It
+    will handle the session and setting authorization headers.
+    """
+
+    def __init__(self, token: str) -> None:
+        self._token = token
+        self._client: Optional[httpx.Client] = None
+
+    def __enter__(self) -> "_GithubApiBase":
+        """
+        Sets up the required headers for auth and response
+        type from the API
+        """
+        self._client = httpx.Client()
+        self._client.headers.update(
+            {
+                "Accept": "application/vnd.github.v3+json",
+                "Authorization": f"token {self._token}",
+            },
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """
+        Ensures the authorization token is cleaned up no matter
+        the reason for the exit
+        """
+        if "Accept" in self._client.headers:
+            del self._client.headers["Accept"]
+        if "Authorization" in self._client.headers:
+            del self._client.headers["Authorization"]
+
+        # Close the session as well
+        self._client.close()
+        self._client = None
+
+    def _read_all_pages(self, endpoint):
+        """
+        Helper function to read all pages of an endpoint, utilizing the
+        next.url until exhausted.  Assumes the endpoint returns a list
+        """
+        internal_data = []
+
+        while True:
+            resp = self._client.get(endpoint)
+            if resp.status_code == 200:
+                internal_data += resp.json()
+                if "next" in resp.links:
+                    endpoint = resp.links["next"]["url"]
+                else:
+                    logger.debug("Exiting pagination loop")
+                    break
+            else:
+                logger.warning(f"Request to {endpoint} return HTTP {resp.status_code}")
+                resp.raise_for_status()
+
+        return internal_data
+
+
+class _EndpointResponse:
+    """
+    For all endpoint JSON responses, store the full
+    response data, for ease of extending later, if need be.
+    """
+
+    def __init__(self, data: Dict) -> None:
+        self._data = data
+
+
+class GithubBranch(_EndpointResponse):
+    """
+    Simple wrapper for a repository branch, only extracts name information
+    for now.
+    """
+
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        self.name = self._data["name"]
+
+
+class GithubBranchApi(_GithubApiBase):
+    """
+    Wrapper around branch API.
+
+    See https://docs.github.com/en/rest/branches/branches
+
+    """
+
+    def __init__(self, token: str) -> None:
+        super().__init__(token)
+
+        self._ENDPOINT = "https://api.github.com/repos/{REPO}/branches"
+
+    def get_branches(self, repo: str) -> List[GithubBranch]:
+        """
+        Returns all current branches of the given repository owned by the given
+        owner or organization.
+        """
+        # The environment GITHUB_REPOSITORY already contains the owner in the correct location
+        endpoint = self._ENDPOINT.format(REPO=repo)
+        internal_data = self._read_all_pages(endpoint)
+        return [GithubBranch(branch) for branch in internal_data]
+
+
+class ContainerPackage(_EndpointResponse):
+    """
+    Data class wrapping the JSON response from the package related
+    endpoints
+    """
+
+    def __init__(self, data: Dict):
+        super().__init__(data)
+        # This is a numerical ID, required for interactions with this
+        # specific package, including deletion of it or restoration
+        self.id: int = self._data["id"]
+
+        # A string name.  This might be an actual name or it could be a
+        # digest string like "sha256:"
+        self.name: str = self._data["name"]
+
+        # URL to the package, including its ID, can be used for deletion
+        # or restoration without needing to build up a URL ourselves
+        self.url: str = self._data["url"]
+
+        # The list of tags applied to this image. Maybe an empty list
+        self.tags: List[str] = self._data["metadata"]["container"]["tags"]
+
+    @functools.cached_property
+    def untagged(self) -> bool:
+        """
+        Returns True if the image has no tags applied to it, False otherwise
+        """
+        return len(self.tags) == 0
+
+    @functools.cache
+    def tag_matches(self, pattern: str) -> bool:
+        """
+        Returns True if the image has at least one tag which matches the given regex,
+        False otherwise
+        """
+        for tag in self.tags:
+            if re.match(pattern, tag) is not None:
+                return True
+        return False
+
+    def __repr__(self):
+        return f"Package {self.name}"
+
+
+class GithubContainerRegistryApi(_GithubApiBase):
+    """
+    Class wrapper to deal with the Github packages API.  This class only deals with
+    container type packages, the only type published by paperless-ngx.
+    """
+
+    def __init__(self, token: str, owner_or_org: str) -> None:
+        super().__init__(token)
+        self._owner_or_org = owner_or_org
+        if self._owner_or_org == "paperless-ngx":
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-an-organization
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-package-version-for-an-organization
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+        else:
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-the-authenticated-user
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+        self._PACKAGE_VERSION_RESTORE_ENDPOINT = (
+            f"{self._PACKAGE_VERSION_DELETE_ENDPOINT}/restore"
+        )
+
+    def get_active_package_versions(
+        self,
+        package_name: str,
+    ) -> List[ContainerPackage]:
+        """
+        Returns all the versions of a given package (container images) from
+        the API
+        """
+
+        package_type: str = "container"
+        # Need to quote this for slashes in the name
+        package_name = urllib.parse.quote(package_name, safe="")
+
+        endpoint = self._PACKAGES_VERSIONS_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_type,
+            PACKAGE_NAME=package_name,
+        )
+
+        pkgs = []
+
+        for data in self._read_all_pages(endpoint):
+            pkgs.append(ContainerPackage(data))
+
+        return pkgs
+
+    def get_deleted_package_versions(
+        self,
+        package_name: str,
+    ) -> List[ContainerPackage]:
+        package_type: str = "container"
+        # Need to quote this for slashes in the name
+        package_name = urllib.parse.quote(package_name, safe="")
+
+        endpoint = (
+            self._PACKAGES_VERSIONS_ENDPOINT.format(
+                ORG=self._owner_or_org,
+                PACKAGE_TYPE=package_type,
+                PACKAGE_NAME=package_name,
+            )
+            + "?state=deleted"
+        )
+
+        pkgs = []
+
+        for data in self._read_all_pages(endpoint):
+            pkgs.append(ContainerPackage(data))
+
+        return pkgs
+
+    def delete_package_version(self, package_data: ContainerPackage):
+        """
+        Deletes the given package version from the GHCR
+        """
+        resp = self._client.delete(package_data.url)
+        if resp.status_code != 204:
+            logger.warning(
+                f"Request to delete {package_data.url} returned HTTP {resp.status_code}",
+            )
+
+    def restore_package_version(
+        self,
+        package_name: str,
+        package_data: ContainerPackage,
+    ):
+        package_type: str = "container"
+        endpoint = self._PACKAGE_VERSION_RESTORE_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_type,
+            PACKAGE_NAME=package_name,
+            PACKAGE_VERSION_ID=package_data.id,
+        )
+
+        resp = self._client.post(endpoint)
+        if resp.status_code != 204:
+            logger.warning(
+                f"Request to delete {endpoint} returned HTTP {resp.status_code}",
+            )

.github/stale.yml

@@ -1,15 +1,23 @@
 # Number of days of inactivity before an issue becomes stale
 daysUntilStale: 30
+
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
-onlyLabels:
-  - unconfirmed
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: [cant-reproduce]
+
 # Label to use when marking an issue as stale
 staleLabel: stale
+
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
   This issue has been automatically marked as stale because it has not had
   recent activity. It will be closed if no further activity occurs. Thank you
   for your contributions.
+
 # Comment to post when closing a stale issue. Set to `false` to disable
 closeComment: false
+
+# See https://github.com/marketplace/stale for more info on the app
+# and https://github.com/probot/stale for the configuration docs

.github/workflows/ci.yml

@@ -14,19 +14,40 @@ on:
       - 'translations**'
 
 jobs:
+  pre-commit:
+    name: Linting Checks
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout repository
+        uses: actions/checkout@v3
+
+      -
+        name: Install tools
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      -
+        name: Check files
+        uses: pre-commit/action@v3.0.0
+
   documentation:
     name: "Build Documentation"
     runs-on: ubuntu-20.04
+    needs:
+      - pre-commit
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
       -
         name: Install pipenv
-        run: pipx install pipenv
+        run: |
+          pipx install pipenv==2022.10.12
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
           cache: "pipenv"
@@ -35,6 +56,10 @@ jobs:
         name: Install dependencies
         run: |
           pipenv sync --dev
+      -
+        name: List installed Python dependencies
+        run: |
+          pipenv run pip list
       -
         name: Make documentation
         run: |
@@ -47,27 +72,135 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  ci-backend:
-    uses: ./.github/workflows/reusable-ci-backend.yml
+  tests-backend:
+    name: "Tests (${{ matrix.python-version }})"
+    runs-on: ubuntu-20.04
+    needs:
+      - pre-commit
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+      fail-fast: false
+    services:
+      tika:
+        image: ghcr.io/paperless-ngx/tika:latest
+        ports:
+          - "9998:9998/tcp"
+      gotenberg:
+        image: docker.io/gotenberg/gotenberg:7.6
+        ports:
+          - "3000:3000/tcp"
+    env:
+      # Enable Tika end to end testing
+      TIKA_LIVE: 1
+      # Enable paperless_mail testing against real server
+      PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
+      PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
+      PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Install pipenv
+        run: |
+          pipx install pipenv==2022.10.12
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: List installed Python dependencies
+        run: |
+          pipenv run pip list
+      -
+        name: Tests
+        run: |
+          cd src/
+          pipenv run pytest -rfEp
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v34
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          pipenv run coveralls --service=github
 
-  ci-frontend:
-    uses: ./.github/workflows/reusable-ci-frontend.yml
+  tests-frontend:
+    name: "Tests Frontend"
+    runs-on: ubuntu-20.04
+    needs:
+      - pre-commit
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      -
+        name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci
 
   prepare-docker-build:
     name: Prepare Docker Pipeline Data
     if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-20.04
+    # If the push triggered the installer library workflow, wait for it to
+    # complete here.  This ensures the required versions for the final
+    # image have been built, while not waiting at all if the versions haven't changed
+    concurrency:
+      group: build-installer-library
+      cancel-in-progress: false
     needs:
       - documentation
-      - ci-backend
-      - ci-frontend
+      - tests-backend
+      - tests-frontend
     steps:
+      -
+        name: Set ghcr repository name
+        id: set-ghcr-repository
+        run: |
+          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
+          echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
       -
         name: Checkout
         uses: actions/checkout@v3
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: "3.9"
       -
@@ -78,7 +211,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=qpdf-json::${build_json}
+          echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup psycopg2 image
         id: psycopg2-setup
@@ -87,7 +220,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=psycopg2-json::${build_json}
+          echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup pikepdf image
         id: pikepdf-setup
@@ -96,7 +229,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=pikepdf-json::${build_json}
+          echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup jbig2enc image
         id: jbig2enc-setup
@@ -105,10 +238,12 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=jbig2enc-json::${build_json}
+          echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
 
     outputs:
 
+      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
+
       qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
 
       pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
@@ -117,55 +252,6 @@ jobs:
 
       jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
 
-  build-qpdf-debs:
-    name: qpdf
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.qpdf
-      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
-      build-args: |
-        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-
-  build-jbig2enc:
-    name: jbig2enc
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.jbig2enc
-      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
-      build-args: |
-        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
-
-  build-psycopg2-wheel:
-    name: psycopg2
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.psycopg2
-      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
-      build-args: |
-        PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
-        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
-
-  build-pikepdf-wheel:
-    name: pikepdf
-    needs:
-      - prepare-docker-build
-      - build-qpdf-debs
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.pikepdf
-      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
-      build-args: |
-        REPO=${{ github.repository }}
-        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-        PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
-        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
-
   # build and push image to docker hub.
   build-docker-image:
     runs-on: ubuntu-20.04
@@ -174,29 +260,31 @@ jobs:
       cancel-in-progress: true
     needs:
       - prepare-docker-build
-      - build-psycopg2-wheel
-      - build-jbig2enc
-      - build-qpdf-debs
-      - build-pikepdf-wheel
     steps:
       -
         name: Check pushing to Docker Hub
         id: docker-hub
-        # Only push to Dockerhub from the main repo
+        # Only push to Dockerhub from the main repo AND the ref is either:
+        #  main
+        #  dev
+        #  beta
+        #  a tag
         # Otherwise forks would require a Docker Hub account and secrets setup
         run: |
-          if [[ ${{ github.repository }} == "paperless-ngx/paperless-ngx" ]] ; then
-            echo ::set-output name=enable::"true"
+          if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
+            echo "Enabling DockerHub image push"
+            echo "enable=true" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=enable::"false"
+            echo "Not pushing to DockerHub"
+            echo "enable=false" >> $GITHUB_OUTPUT
           fi
       -
         name: Gather Docker metadata
         id: docker-meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
           images: |
-            ghcr.io/${{ github.repository }}
+            ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}
             name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
           tags: |
             # Tag branches with branch name
@@ -210,20 +298,20 @@ jobs:
         uses: actions/checkout@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       -
         name: Login to Github Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         # Don't attempt to login is not pushing to Docker Hub
         if: steps.docker-hub.outputs.enable == 'true'
         with:
@@ -231,7 +319,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./Dockerfile
@@ -247,11 +335,11 @@ jobs:
           # Get cache layers from this branch, then dev, then main
           # This allows new branches to get at least some cache benefits, generally from dev
           cache-from: |
-            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
-            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:dev
-            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:main
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main
           cache-to: |
-            type=registry,mode=max,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
       -
         name: Inspect image
         run: |
@@ -276,18 +364,27 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
+      -
+        name: Install pipenv
+        run: |
+          pip3 install --upgrade pip setuptools wheel pipx
+          pipx install pipenv
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Install dependencies
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Install system dependencies
         run: |
           sudo apt-get update -qq
           sudo apt-get install -qq --no-install-recommends gettext liblept5
-          pip3 install --upgrade pip setuptools wheel
-          pip3 install -r requirements.txt
       -
         name: Download frontend artifact
         uses: actions/download-artifact@v3
@@ -300,34 +397,38 @@ jobs:
         with:
           name: documentation
           path: docs/_build/html/
+      -
+        name: Generate requirements file
+        run: |
+          pipenv requirements > requirements.txt
+      -
+        name: Compile messages
+        run: |
+          cd src/
+          pipenv run python3 manage.py compilemessages
+      -
+        name: Collect static files
+        run: |
+          cd src/
+          pipenv run python3 manage.py collectstatic --no-input
       -
         name: Move files
         run: |
           mkdir dist
           mkdir dist/paperless-ngx
           mkdir dist/paperless-ngx/scripts
-          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ngx/
+          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
           cp paperless.conf.example dist/paperless-ngx/paperless.conf
           cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
-          cp docker/ dist/paperless-ngx/docker -r
+          cp -r docker/ dist/paperless-ngx/docker
           cp scripts/*.service scripts/*.sh dist/paperless-ngx/scripts/
-          cp src/ dist/paperless-ngx/src -r
-          cp docs/_build/html/ dist/paperless-ngx/docs -r
-      -
-        name: Compile messages
-        run: |
-          cd dist/paperless-ngx/src
-          python3 manage.py compilemessages
-      -
-        name: Collect static files
-        run: |
-          cd dist/paperless-ngx/src
-          python3 manage.py collectstatic --no-input
+          cp -r src/ dist/paperless-ngx/src
+          cp -r docs/_build/html/ dist/paperless-ngx/docs
+          mv static dist/paperless-ngx
       -
         name: Make release package
         run: |
           cd dist
-          find . -name __pycache__ | xargs rm -r
           tar -cJf paperless-ngx.tar.xz paperless-ngx/
       -
         name: Upload release artifact
@@ -338,6 +439,10 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
+    outputs:
+      prerelease: ${{ steps.get_version.outputs.prerelease }}
+      changelog: ${{ steps.create-release.outputs.body }}
+      version: ${{ steps.get_version.outputs.version }}
     needs:
       - build-release
     if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
@@ -352,16 +457,16 @@ jobs:
         name: Get version
         id: get_version
         run: |
-          echo ::set-output name=version::${{ github.ref_name }}
+          echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
           if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
-            echo ::set-output name=prerelease::true
+            echo "prerelease=true" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=prerelease::false
+            echo "prerelease=false" >> $GITHUB_OUTPUT
           fi
       -
         name: Create Release and Changelog
         id: create-release
-        uses: release-drafter/release-drafter@v5
+        uses: paperless-ngx/release-drafter@master
         with:
           name: Paperless-ngx ${{ steps.get_version.outputs.version }}
           tag: ${{ steps.get_version.outputs.version }}
@@ -381,21 +486,65 @@ jobs:
           asset_path: ./paperless-ngx.tar.xz
           asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
           asset_content_type: application/x-xz
+
+  append-changelog:
+    runs-on: ubuntu-20.04
+    needs:
+      - publish-release
+    if: needs.publish-release.outputs.prerelease == 'false'
+    steps:
       -
         name: Checkout
         uses: actions/checkout@v3
         with:
           ref: main
+      -
+        name: Install pipenv
+        run: |
+          pip3 install --upgrade pip setuptools wheel pipx
+          pipx install pipenv
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
       -
         name: Append Changelog to docs
         id: append-Changelog
         working-directory: docs
         run: |
-          echo -e "# Changelog\n\n${{ steps.create-release.outputs.body }}\n" > changelog-new.md
+          git branch ${{ needs.publish-release.outputs.version }}-changelog
+          git checkout ${{ needs.publish-release.outputs.version }}-changelog
+          echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md
+          echo "Manually linking usernames"
+          sed -i -r 's|@(.+?) \(\[#|[@\1](https://github.com/\1) ([#|ig' changelog-new.md
           CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
           echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
           mv changelog-new.md changelog.md
+          pipenv run pre-commit --files changelog.md
           git config --global user.name "github-actions"
           git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git commit -am "Changelog ${{ steps.get_version.outputs.version }} - GHA"
-          git push origin HEAD:main
+          git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"
+          git push origin ${{ needs.publish-release.outputs.version }}-changelog
+      -
+        name: Create Pull Request
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const { repo, owner } = context.repo;
+            const result = await github.rest.pulls.create({
+              title: '[Documentation] Add ${{ needs.publish-release.outputs.version }} changelog',
+              owner,
+              repo,
+              head: '${{ needs.publish-release.outputs.version }}-changelog',
+              base: 'main',
+              body: 'This PR is auto-generated by CI.'
+            });
+            github.rest.issues.addLabels({
+              owner,
+              repo,
+              issue_number: result.data.number,
+              labels: ['documentation']
+            });

.github/workflows/cleanup-tags.yml

@@ -0,0 +1,95 @@
+# This workflow runs on certain conditions to check for and potentially
+# delete container images from the GHCR which no longer have an associated
+# code branch.
+# Requires a PAT with the correct scope set in the secrets
+
+name: Cleanup Image Tags
+
+on:
+  schedule:
+    - cron: '0 0 * * SAT'
+  delete:
+  pull_request:
+    types:
+      - closed
+  push:
+    paths:
+      - ".github/workflows/cleanup-tags.yml"
+      - ".github/scripts/cleanup-tags.py"
+      - ".github/scripts/github.py"
+      - ".github/scripts/common.py"
+
+concurrency:
+  group: registry-tags-cleanup
+  cancel-in-progress: false
+
+jobs:
+  cleanup-images:
+    name: Cleanup Image Tags for ${{ matrix.primary-name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - primary-name: "paperless-ngx"
+            cache-name: "paperless-ngx/builder/cache/app"
+
+          - primary-name: "paperless-ngx/builder/qpdf"
+            cache-name: "paperless-ngx/builder/cache/qpdf"
+
+          - primary-name: "paperless-ngx/builder/pikepdf"
+            cache-name: "paperless-ngx/builder/cache/pikepdf"
+
+          - primary-name: "paperless-ngx/builder/jbig2enc"
+            cache-name: "paperless-ngx/builder/cache/jbig2enc"
+
+          - primary-name: "paperless-ngx/builder/psycopg2"
+            cache-name: "paperless-ngx/builder/cache/psycopg2"
+    env:
+      # Requires a personal access token with the OAuth scope delete:packages
+      TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      -
+        name: Install httpx
+        run: |
+          python -m pip install httpx
+      #
+      # Clean up primary package
+      #
+      -
+        name: Cleanup for package "${{ matrix.primary-name }}"
+        if: "${{ env.TOKEN != '' }}"
+        run: |
+          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --is-manifest --delete "${{ matrix.primary-name }}"
+      #
+      # Clean up registry cache package
+      #
+      -
+        name: Cleanup for package "${{ matrix.cache-name }}"
+        if: "${{ env.TOKEN != '' }}"
+        run: |
+          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
+      #
+      # Verify tags which are left still pull
+      #
+      -
+        name: Check all tags still pull
+        run: |
+          ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
+          echo "Pulling all tags of ${ghcr_name}"
+          docker pull --quiet --all-tags ${ghcr_name}
+          docker image list

.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/installer-library.yml

@@ -0,0 +1,170 @@
+# This workflow will run to update the installer library of
+# Docker images.  These are the images which provide updated wheels
+# .deb installation packages or maybe just some compiled library
+
+name: Build Image Library
+
+on:
+  push:
+    # Must match one of these branches AND one of the paths
+    # to be triggered
+    branches:
+      - "main"
+      - "dev"
+      - "library-*"
+      - "feature-*"
+    paths:
+      # Trigger the workflow if a Dockerfile changed
+      - "docker-builders/**"
+      # Trigger if a package was updated
+      - ".build-config.json"
+      - "Pipfile.lock"
+      # Also trigger on workflow changes related to the library
+      - ".github/workflows/installer-library.yml"
+      - ".github/workflows/reusable-workflow-builder.yml"
+      - ".github/scripts/**"
+
+# Set a workflow level concurrency group so primary workflow
+# can wait for this to complete if needed
+# DO NOT CHANGE without updating main workflow group
+concurrency:
+  group: build-installer-library
+  cancel-in-progress: false
+
+jobs:
+  prepare-docker-build:
+    name: Prepare Docker Image Version Data
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Set ghcr repository name
+        id: set-ghcr-repository
+        run: |
+          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
+          echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      -
+        name: Install jq
+        run: |
+          sudo apt-get update
+          sudo apt-get install jq
+      -
+        name: Setup qpdf image
+        id: qpdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
+
+          echo ${build_json}
+
+          echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
+      -
+        name: Setup psycopg2 image
+        id: psycopg2-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
+      -
+        name: Setup other versions
+        id: cache-bust-setup
+        run: |
+          pillow_version=$(jq ".default.pillow.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+          lxml_version=$(jq ".default.lxml.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+
+          echo "Pillow is ${pillow_version}"
+          echo "lxml is ${lxml_version}"
+
+          echo "pillow-version=${pillow_version}" >> $GITHUB_OUTPUT
+          echo "lxml-version=${lxml_version}" >> $GITHUB_OUTPUT
+
+    outputs:
+
+      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json }}
+
+      pillow-version: ${{ steps.cache-bust-setup.outputs.pillow-version }}
+
+      lxml-version: ${{ steps.cache-bust-setup.outputs.lxml-version }}
+
+  build-qpdf-debs:
+    name: qpdf
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+      build-args: |
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+  build-jbig2enc:
+    name: jbig2enc
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.jbig2enc
+      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+      build-args: |
+        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+  build-psycopg2-wheel:
+    name: psycopg2
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.psycopg2
+      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+      build-args: |
+        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+  build-pikepdf-wheel:
+    name: pikepdf
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.pikepdf
+      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+      build-args: |
+        REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }}
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+        PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
+        LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}

.github/workflows/project-actions.yml

@@ -13,6 +13,9 @@ on:
       - main
       - dev
 
+permissions:
+  contents: read
+
 env:
   todo: Todo
   done: Done
@@ -24,8 +27,8 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'issues' && (github.event.action == 'opened' || github.event.action == 'reopened')
     steps:
-      - name: Set issue status to ${{ env.todo }}
-        uses: leonsteinhaeuser/project-beta-automations@v1.2.1
+      - name: Add issue to project and set status to ${{ env.todo }}
+        uses: leonsteinhaeuser/project-beta-automations@v2.0.1
         with:
           gh_token: ${{ secrets.GH_TOKEN }}
           organization: paperless-ngx
@@ -35,13 +38,20 @@ jobs:
   pr_opened_or_reopened:
     name: pr_opened_or_reopened
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened')
+    permissions:
+      # write permission is required for autolabeler
+      pull-requests: write
+    if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened') && github.event.pull_request.user.login != 'dependabot'
     steps:
-      - name: Set PR status to ${{ env.in_progress }}
-        uses: leonsteinhaeuser/project-beta-automations@v1.2.1
+      - name: Add PR to project and set status to "Needs Review"
+        uses: leonsteinhaeuser/project-beta-automations@v2.0.1
         with:
           gh_token: ${{ secrets.GH_TOKEN }}
           organization: paperless-ngx
           project_id: 2
           resource_node_id: ${{ github.event.pull_request.node_id }}
-          status_value: ${{ env.in_progress }} # Target status
+          status_value: "Needs Review" # Target status
+      - name: Label PR with release-drafter
+        uses: release-drafter/release-drafter@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/reusable-ci-backend.yml

@@ -1,108 +0,0 @@
-name: Backend CI Jobs
-
-on:
-  workflow_call:
-
-jobs:
-
-  code-checks-backend:
-    name: "Code Style Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Install checkers
-        run: |
-          pipx install reorder-python-imports
-          pipx install yesqa
-          pipx install add-trailing-comma
-          pipx install flake8
-      -
-        name: Run reorder-python-imports
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-      -
-        name: Run yesqa
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-      -
-        name: Run add-trailing-comma
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
-      # black is placed after add-trailing-comma because it may format differently
-      # if a trailing comma is added
-      -
-        name: Run black
-        uses: psf/black@stable
-        with:
-          options: "--check --diff"
-          version: "22.3.0"
-      -
-        name: Run flake8 checks
-        run: |
-          cd src/
-          flake8 --max-line-length=88 --ignore=E203,W503
-
-  tests-backend:
-    name: "Tests (${{ matrix.python-version }})"
-    runs-on: ubuntu-20.04
-    needs:
-      - code-checks-backend
-    strategy:
-      matrix:
-        python-version: ['3.8', '3.9', '3.10']
-      fail-fast: false
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      -
-        name: Install pipenv
-        run: pipx install pipenv
-      -
-        name: Set up Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
-      -
-        name: Install system dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
-      -
-        name: Install Python dependencies
-        run: |
-          pipenv sync --dev
-      -
-        name: Tests
-        run: |
-          cd src/
-          pipenv run pytest
-      -
-        name: Get changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v19
-        with:
-          files: |
-            src/**
-      -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github

.github/workflows/reusable-ci-frontend.yml

@@ -1,42 +0,0 @@
-name: Frontend CI Jobs
-
-on:
-  workflow_call:
-
-jobs:
-
-  code-checks-frontend:
-    name: "Code Style Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-      -
-        name: Install prettier
-        run: |
-          npm install prettier
-      -
-        name: Run prettier
-        run:
-          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
-  tests-frontend:
-    name: "Tests"
-    runs-on: ubuntu-20.04
-    needs:
-      - code-checks-frontend
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-      - run: cd src-ui && npm ci
-      - run: cd src-ui && npm run test
-      - run: cd src-ui && npm run e2e:ci

.github/workflows/reusable-workflow-builder.yml

@@ -28,20 +28,20 @@ jobs:
         uses: actions/checkout@v3
       -
         name: Login to Github Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       -
         name: Build ${{ fromJSON(inputs.build-json).name }}
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ${{ inputs.dockerfile }}

.gitignore

@@ -63,11 +63,14 @@ target/
 
 # VS Code
 .vscode
+/src-ui/.vscode
+/docs/.vscode
 
 # Other stuff that doesn't belong
 .virtualenv
 virtualenv
 /venv
+.venv/
 /docker-compose.env
 /docker-compose.yml
 
@@ -84,8 +87,12 @@ scripts/nuke
 /paperless.conf
 /consume/
 /export/
-/src-ui/.vscode
 
 # this is where the compiled frontend is moved to.
 /src/documents/static/frontend/
-/docs/.vscode/settings.json
+
+# mac os
+.DS_Store
+
+# celery schedule file
+celerybeat-schedule*

.hadolint.yml

@@ -0,0 +1,8 @@
+failure-threshold: warning
+ignored:
+  # https://github.com/hadolint/hadolint/wiki/DL3008
+  - DL3008
+  # https://github.com/hadolint/hadolint/wiki/DL3013
+  - DL3013
+  # https://github.com/hadolint/hadolint/wiki/DL3003
+  - DL3003

.pre-commit-config.yaml

@@ -5,7 +5,7 @@
 repos:
   # General hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v4.3.0
     hooks:
       - id: check-docstring-first
       - id: check-json
@@ -27,7 +27,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.6.2"
+    rev: "v2.7.1"
     hooks:
       - id: prettier
         types_or:
@@ -37,17 +37,17 @@ repos:
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.1.0
+    rev: v3.9.0
     hooks:
       - id: reorder-python-imports
         exclude: "(migrations)"
   - repo: https://github.com/asottile/yesqa
-    rev: "v1.3.0"
+    rev: "v1.4.0"
     hooks:
       - id: yesqa
         exclude: "(migrations)"
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: "v2.2.3"
+    rev: "v2.3.0"
     hooks:
       - id: add-trailing-comma
         exclude: "(migrations)"
@@ -59,11 +59,11 @@ repos:
         args:
           - "--config=./src/setup.cfg"
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.10.0
     hooks:
       - id: black
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.1
+    rev: v3.1.0
     hooks:
       - id: pyupgrade
         exclude: "(migrations)"
@@ -74,13 +74,6 @@ repos:
     rev: v2.10.0
     hooks:
       - id: hadolint
-        args:
-          - --ignore
-          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
-          - --ignore
-          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
-          - --ignore
-          - DL3003      # https://github.com/hadolint/hadolint/wiki/DL3003  (seems excessive to use WORKDIR so much)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1

CODEOWNERS

@@ -7,4 +7,3 @@
 /src/ @paperless-ngx/backend
 Pipfile* @paperless-ngx/backend
 *.py @paperless-ngx/backend
-requirements.txt @paperless-ngx/backend

Dockerfile

@@ -26,10 +26,29 @@ COPY ./src-ui /src/src-ui
 WORKDIR /src/src-ui
 RUN set -eux \
   && npm update npm -g \
-  && npm ci --no-optional
+  && npm ci --omit=optional
 RUN set -eux \
   && ./node_modules/.bin/ng build --configuration production
 
+FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base
+
+# This stage generates the requirements.txt file using pipenv
+# This stage runs once for the native platform, as the outputs are not
+# dependent on target arch
+# This way, pipenv dependencies are not left in the final image
+# nor can pipenv mess up the final image somehow
+# Inputs: None
+
+WORKDIR /usr/src/pipenv
+
+COPY Pipfile* ./
+
+RUN set -eux \
+  && echo "Installing pipenv" \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv \
+  && echo "Generating requirement.txt" \
+    && pipenv requirements > requirements.txt
+
 FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
@@ -77,15 +96,13 @@ ARG RUNTIME_PACKAGES="\
   libraqm0 \
   libgnutls30 \
   libjpeg62-turbo \
-  optipng \
   python3 \
   python3-pip \
   python3-setuptools \
   postgresql-client \
+  mariadb-client \
   # For Numpy
   libatlas3-base \
-  # thumbnail size reduction
-  pngquant \
   # OCRmyPDF dependencies
   tesseract-ocr \
   tesseract-ocr-eng \
@@ -93,6 +110,10 @@ ARG RUNTIME_PACKAGES="\
   tesseract-ocr-fra \
   tesseract-ocr-ita \
   tesseract-ocr-spa \
+  # Suggested for OCRmyPDF
+  pngquant \
+  # Suggested for pikepdf
+  jbig2dec \
   tzdata \
   unpaper \
   # Mime type detection
@@ -122,20 +143,36 @@ COPY gunicorn.conf.py .
 # These change sometimes, but rarely
 WORKDIR /usr/src/paperless/src/docker/
 
-RUN --mount=type=bind,readwrite,source=docker,target=./ \
-  set -eux \
+COPY [ \
+  "docker/imagemagick-policy.xml", \
+  "docker/supervisord.conf", \
+  "docker/docker-entrypoint.sh", \
+  "docker/docker-prepare.sh", \
+  "docker/paperless_cmd.sh", \
+  "docker/wait-for-redis.py", \
+  "docker/management_script.sh", \
+  "docker/flower-conditional.sh", \
+  "docker/install_management_commands.sh", \
+  "/usr/src/paperless/src/docker/" \
+]
+
+RUN set -eux \
   && echo "Configuring ImageMagick" \
-    && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+    && mv imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
   && echo "Configuring supervisord" \
     && mkdir /var/log/supervisord /var/run/supervisord \
-    && cp supervisord.conf /etc/supervisord.conf \
+    && mv supervisord.conf /etc/supervisord.conf \
   && echo "Setting up Docker scripts" \
-    && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
+    && mv docker-entrypoint.sh /sbin/docker-entrypoint.sh \
     && chmod 755 /sbin/docker-entrypoint.sh \
-    && cp docker-prepare.sh /sbin/docker-prepare.sh \
+    && mv docker-prepare.sh /sbin/docker-prepare.sh \
     && chmod 755 /sbin/docker-prepare.sh \
-    && cp wait-for-redis.py /sbin/wait-for-redis.py \
+    && mv wait-for-redis.py /sbin/wait-for-redis.py \
     && chmod 755 /sbin/wait-for-redis.py \
+    && mv paperless_cmd.sh /usr/local/bin/paperless_cmd.sh \
+    && chmod 755 /usr/local/bin/paperless_cmd.sh \
+    && mv flower-conditional.sh /usr/local/bin/flower-conditional.sh \
+    && chmod 755 /usr/local/bin/flower-conditional.sh \
   && echo "Installing managment commands" \
     && chmod +x install_management_commands.sh \
     && ./install_management_commands.sh
@@ -148,27 +185,31 @@ RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
     --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
   set -eux \
   && echo "Installing qpdf" \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf29_*.deb \
     && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
   && echo "Installing pikepdf and dependencies" \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/packaging*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/lxml*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/Pillow*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pyparsing*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pikepdf*.whl \
-    && python -m pip list \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \
+    && python3 -m pip list \
   && echo "Installing psycopg2" \
-    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/psycopg2/wheels/psycopg2*.whl \
-    && python -m pip list
+    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
+    && python3 -m pip list
+
+WORKDIR /usr/src/paperless/src/
 
 # Python dependencies
 # Change pretty frequently
-COPY requirements.txt ../
+COPY --from=pipenv-base /usr/src/pipenv/requirements.txt ./
 
 # Packages needed only for building a few quick Python
 # dependencies
 ARG BUILD_PACKAGES="\
   build-essential \
+  git \
+  default-libmysqlclient-dev \
   python3-dev"
 
 RUN set -eux \
@@ -177,7 +218,7 @@ RUN set -eux \
     && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
     && python3 -m pip install --no-cache-dir --upgrade wheel \
   && echo "Installing Python requirements" \
-    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir --requirement requirements.txt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
@@ -188,8 +229,6 @@ RUN set -eux \
     && rm -rf /var/cache/apt/archives/* \
     && truncate -s 0 /var/log/*log
 
-WORKDIR /usr/src/paperless/src/
-
 # copy backend
 COPY ./src ./
 
@@ -213,4 +252,4 @@ ENTRYPOINT ["/sbin/docker-entrypoint.sh"]
 
 EXPOSE 8000
 
-CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf"]
+CMD ["/usr/local/bin/paperless_cmd.sh"]

Pipfile

@@ -10,41 +10,42 @@ name = "piwheels"
 
 [packages]
 dateparser = "~=1.1"
-django = "~=4.0"
+django = "~=4.1"
 django-cors-headers = "*"
 django-extensions = "*"
-django-filter = "~=21.1"
-django-q = "~=1.3"
-djangorestframework = "~=3.13"
+django-filter = "~=22.1"
+djangorestframework = "~=3.14"
 filelock = "*"
-fuzzywuzzy = {extras = ["speedup"], version = "*"}
 gunicorn = "*"
-imap-tools = "~=0.54.0"
+imap-tools = "*"
 langdetect = "*"
 pathvalidate = "*"
-pillow = "~=9.1"
-# Any version update to pikepdf requires a base image update
-pikepdf = "~=5.1"
+pillow = "~=9.3"
+pikepdf = "*"
 python-gnupg = "*"
 python-dotenv = "*"
 python-dateutil = "*"
 python-magic = "*"
-# Any version update to psycopg2 requires a base image update
 psycopg2 = "*"
-redis = "*"
-# Pinned because aarch64 wheels and updates cause warnings when loading the classifier model.
-scikit-learn="==1.0.2"
-whitenoise = "~=6.0.0"
-watchdog = "~=2.1.0"
-whoosh="~=2.7.4"
+rapidfuzz = "*"
+redis = {extras = ["hiredis"], version = "*"}
+scikit-learn = "~=1.1"
+# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
+scipy = "==1.8.1"
+numpy = "*"
+whitenoise = "~=6.2"
+watchdog = "~=2.1"
+whoosh="~=2.7"
 inotifyrecursive = "~=0.3"
-ocrmypdf = "~=13.4"
+ocrmypdf = "~=14.0"
 tqdm = "*"
 tika = "*"
 # TODO: This will sadly also install daphne+dependencies,
 #  which an ASGI server we don't need. Adds about 15MB image size.
 channels = "~=3.0"
-channels-redis = "*"
+# Locked version until https://github.com/django/channels_redis/issues/332
+# is resolved
+channels-redis = "==3.4.1"
 uvicorn = {extras = ["standard"], version = "*"}
 concurrent-log-handler = "*"
 "pdfminer.six" = "*"
@@ -52,7 +53,13 @@ concurrent-log-handler = "*"
 "importlib-resources" = {version = "*", markers = "python_version < '3.9'"}
 zipp = {version = "*", markers = "python_version < '3.9'"}
 pyzbar = "*"
+mysqlclient = "*"
+celery = {extras = ["redis"], version = "*"}
+django-celery-results = "*"
+setproctitle = "*"
+nltk = "*"
 pdf2image = "*"
+flower = "*"
 
 [dev-packages]
 coveralls = "*"
@@ -64,9 +71,10 @@ pytest-django = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-sphinx = "~=4.5.0"
+sphinx = "~=5.3"
 sphinx_rtd_theme = "*"
 tox = "*"
 black = "*"
 pre-commit = "*"
+sphinx-autobuild = "*"
 myst-parser = "*"

README.md

@@ -2,7 +2,7 @@
 [![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx)
 [![Documentation Status](https://readthedocs.org/projects/paperless-ngx/badge/?version=latest)](https://paperless-ngx.readthedocs.io/en/latest/?badge=latest)
 [![Coverage Status](https://coveralls.io/repos/github/paperless-ngx/paperless-ngx/badge.svg?branch=master)](https://coveralls.io/github/paperless-ngx/paperless-ngx?branch=master)
-[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/#paperless:adnidor.de)
+[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org)
 
 <p align="center">
 <img src="https://github.com/paperless-ngx/paperless-ngx/raw/main/resources/logo/web/png/Black%20logo%20-%20no%20background.png#gh-light-mode-only" width="50%" />
@@ -102,9 +102,10 @@ For bugs please [open an issue](https://github.com/paperless-ngx/paperless-ngx/i
 
 Paperless has been around a while now, and people are starting to build stuff on top of it. If you're one of those people, we can add your project to this list:
 
-- [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ngx.
+- [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ng.
 - [Paperless Share](https://github.com/qcasey/paperless_share). Share any files from your Android application with paperless. Very simple, but works with all of the mobile scanning apps out there that allow you to share scanned documents.
 - [Scan to Paperless](https://github.com/sbrunner/scan-to-paperless): Scan and prepare (crop, deskew, OCR, ...) your documents for Paperless.
+- [Paperless Mobile](https://github.com/astubenbord/paperless-mobile): A modern, feature rich mobile application for Paperless.
 
 These projects also exist, but their status and compatibility with paperless-ngx is unknown.
 

build-docker-image.sh

@@ -23,6 +23,8 @@ fi
 # Parse what we can from Pipfile.lock
 pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
 psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+pillow_version=$(jq ".default.pillow.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+lxml_version=$(jq ".default.lxml.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
 # Read this from the other config file
 qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
 jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
@@ -34,9 +36,12 @@ branch_name=$(git rev-parse --abbrev-ref HEAD)
 export DOCKER_BUILDKIT=1
 
 docker build --file "$1" \
+	--progress=plain \
 	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
 	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
 	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
 	--build-arg QPDF_VERSION="${qpdf_version}" \
 	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PILLOW_VERSION="${pillow_version}" \
+	--build-arg LXML_VERSION="${lxml_version}" \
 	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" .

docker-builders/Dockerfile.frontend

@@ -1,14 +0,0 @@
-# This Dockerfile compiles the frontend
-# Inputs: None
-
-FROM node:16-bullseye-slim AS compile-frontend
-
-COPY ./src /src/src
-COPY ./src-ui /src/src-ui
-
-WORKDIR /src/src-ui
-RUN set -eux \
-  && npm update npm -g \
-  && npm ci --no-optional
-RUN set -eux \
-  && ./node_modules/.bin/ng build --configuration production

docker-builders/Dockerfile.jbig2enc

@@ -7,6 +7,7 @@ FROM debian:bullseye-slim as main
 LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG JBIG2ENC_VERSION
 
 ARG BUILD_PACKAGES="\
   build-essential \
@@ -19,21 +20,16 @@ ARG BUILD_PACKAGES="\
 
 WORKDIR /usr/src/jbig2enc
 
-# As this is an base image for a multi-stage final image
-# the added size of the install is basically irrelevant
-RUN apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
-  && rm -rf /var/lib/apt/lists/*
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-ARG JBIG2ENC_VERSION
-
 RUN set -eux \
-  && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
-RUN set -eux \
-  && ./autogen.sh
-RUN set -eux \
-  && ./configure && make
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && echo "Building jbig2enc" \
+    && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc . \
+    && ./autogen.sh \
+    && ./configure \
+    && make \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker-builders/Dockerfile.pikepdf

@@ -2,8 +2,7 @@
 # Inputs:
 #    - REPO - Docker repository to pull qpdf from
 #    - QPDF_VERSION - The image qpdf version to copy .deb files from
-#    - PIKEPDF_GIT_TAG - The Git tag to clone and build from
-#    - PIKEPDF_VERSION - Used to force the built pikepdf version to match
+#    - PIKEPDF_VERSION - Version of pikepdf to build wheel for
 
 # Default to pulling from the main repo registry when manually building
 ARG REPO="paperless-ngx/paperless-ngx"
@@ -18,12 +17,16 @@ FROM python:3.9-slim-bullseye as main
 LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG PIKEPDF_VERSION
+# These are not used, but will still bust the cache if one changes
+# Otherwise, the main image will try to build thing (and fail)
+ARG PILLOW_VERSION
+ARG LXML_VERSION
 
 ARG BUILD_PACKAGES="\
   build-essential \
   python3-dev \
   python3-pip \
-  git \
   # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers
   libgnutls28-dev \
   # lxml requrements - https://lxml.de/installation.html
@@ -57,36 +60,33 @@ COPY --from=qpdf-builder /usr/src/qpdf/*.deb ./
 # the added size of the install is basically irrelevant
 
 RUN set -eux \
-  && apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
-  && dpkg --install libqpdf28_*.deb \
-  && dpkg --install libqpdf-dev_*.deb \
-  && python3 -m pip install --no-cache-dir --upgrade \
-    pip \
-    wheel \
-    # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
-    pybind11 \
-  && rm -rf /var/lib/apt/lists/*
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-ARG PIKEPDF_GIT_TAG
-ARG PIKEPDF_VERSION
-
-RUN set -eux \
-  && echo "building pikepdf wheel" \
-  # Note the v in the tag name here
-  && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
-  && cd pikepdf \
-  # pikepdf seems to specifciy either a next version when built OR
-  # a post release tag.
-  # In either case, this won't match what we want from requirements.txt
-  # Directly modify the setup.py to set the version we just checked out of Git
-  && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \
-  # https://github.com/pikepdf/pikepdf/issues/323
-  && rm pyproject.toml \
-  && mkdir wheels \
-  && python3 -m pip wheel . --wheel-dir wheels \
-  && ls -ahl wheels
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && echo "Installing qpdf" \
+    && dpkg --install libqpdf29_*.deb \
+    && dpkg --install libqpdf-dev_*.deb \
+  && echo "Installing Python tools" \
+    && python3 -m pip install --no-cache-dir --upgrade \
+      pip \
+      wheel \
+      # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
+      pybind11 \
+  && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \
+    && mkdir wheels \
+    && python3 -m pip wheel \
+      # Build the package at the required version
+      pikepdf==${PIKEPDF_VERSION} \
+      # Output the *.whl into this directory
+      --wheel-dir wheels \
+      # Do not use a binary packge for the package being built
+      --no-binary=pikepdf \
+      # Do use binary packages for dependencies
+      --prefer-binary \
+      # Don't cache build files
+      --no-cache-dir \
+    && ls -ahl wheels \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker-builders/Dockerfile.psycopg2

@@ -1,17 +1,16 @@
 # This Dockerfile builds the psycopg2 wheel
 # Inputs:
-#    - PSYCOPG2_GIT_TAG - The Git tag to clone and build from
-#    - PSYCOPG2_VERSION - Unused, kept for future possible usage
+#    - PSYCOPG2_VERSION - Version to build
 
 FROM python:3.9-slim-bullseye as main
 
 LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
 
+ARG PSYCOPG2_VERSION
 ARG DEBIAN_FRONTEND=noninteractive
 
 ARG BUILD_PACKAGES="\
   build-essential \
-  git \
   python3-dev \
   python3-pip \
   # https://www.psycopg.org/docs/install.html#prerequisites
@@ -23,23 +22,27 @@ WORKDIR /usr/src
 # the added size of the install is basically irrelevant
 
 RUN set -eux \
-  && apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
-  && rm -rf /var/lib/apt/lists/* \
-  && python3 -m pip install --no-cache-dir --upgrade pip wheel
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-ARG PSYCOPG2_GIT_TAG
-ARG PSYCOPG2_VERSION
-
-RUN set -eux \
-  && echo "Building psycopg2 wheel" \
-  && cd /usr/src \
-  && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \
-  && cd psycopg2 \
-  && mkdir wheels \
-  && python3 -m pip wheel . --wheel-dir wheels \
-  && ls -ahl wheels/
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && echo "Installing Python tools" \
+    && python3 -m pip install --no-cache-dir --upgrade pip wheel \
+  && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \
+    && cd /usr/src \
+    && mkdir wheels \
+    && python3 -m pip wheel \
+      # Build the package at the required version
+      psycopg2==${PSYCOPG2_VERSION} \
+      # Output the *.whl into this directory
+      --wheel-dir wheels \
+      # Do not use a binary packge for the package being built
+      --no-binary=psycopg2 \
+      # Do use binary packages for dependencies
+      --prefer-binary \
+      # Don't cache build files
+      --no-cache-dir \
+    && ls -ahl wheels/ \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker-builders/Dockerfile.qpdf

@@ -1,8 +1,15 @@
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+#    - QPDF_VERSION - the version of qpdf to build a .deb.
+#                     Must be present as a deb-src in bookworm
+
 FROM debian:bullseye-slim as main
 
 LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
 
 ARG DEBIAN_FRONTEND=noninteractive
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
 
 ARG BUILD_PACKAGES="\
   build-essential \
@@ -15,39 +22,27 @@ ARG BUILD_PACKAGES="\
   libjpeg62-turbo-dev \
   libgnutls28-dev \
   packaging-dev \
+  cmake \
   zlib1g-dev"
 
 WORKDIR /usr/src
 
-# As this is an base image for a multi-stage final image
-# the added size of the install is basically irrelevant
-
-RUN set -eux \
-  && apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
-  && rm -rf /var/lib/apt/lists/*
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-# This must match to pikepdf's minimum at least
-ARG QPDF_VERSION
-
-# In order to get the required version of qpdf, it is backported from bookwork
-# and then built from source
 RUN set -eux \
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && echo "Getting qpdf src" \
+    && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
+    && apt-get update \
+    && mkdir qpdf \
+    && cd qpdf \
+    && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
   && echo "Building qpdf" \
-  && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
-  && apt-get update \
-  && mkdir qpdf \
-  && cd qpdf \
-  && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
-  && rm -rf /var/lib/apt/lists/* \
-  && cd qpdf-$QPDF_VERSION \
-  # We don't need to build the tests (also don't run them)
-  && rm -rf libtests \
-  && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
-  && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
-  && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
-  && ls -ahl ../*.deb
+    && cd qpdf-$QPDF_VERSION \
+    && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+    && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean \
+    && ls -ahl ../*.deb \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker/compose/docker-compose.env

@@ -36,3 +36,7 @@
 # The default language to use for OCR. Set this to the language most of your
 # documents are written in.
 #PAPERLESS_OCR_LANGUAGE=eng
+
+# Set if accessing paperless via a domain subpath e.g. https://domain.com/PATHPREFIX and using a reverse-proxy like traefik or nginx
+#PAPERLESS_FORCE_SCRIPT_NAME=/PATHPREFIX
+#PAPERLESS_STATIC_URL=/PATHPREFIX/static/ # trailing slash required

docker/compose/docker-compose.mariadb-tika.yml

@@ -0,0 +1,102 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), MariaDB is used as the database server.
+# - Apache Tika and Gotenberg servers are started with paperless and paperless
+#   is configured to use these services. These provide support for consuming
+#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
+#   parts.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: docker.io/library/redis:7
+    restart: unless-stopped
+    volumes:
+      - redisdata:/data
+
+  db:
+    image: docker.io/library/mariadb:10
+    restart: unless-stopped
+    volumes:
+      - dbdata:/var/lib/mysql
+    environment:
+      MARIADB_HOST: paperless
+      MARIADB_DATABASE: paperless
+      MARIADB_USER: paperless
+      MARIADB_PASSWORD: paperless
+      MARIADB_ROOT_PASSWORD: paperless
+    ports:
+      - "3306:3306"
+
+  webserver:
+    image: ghcr.io/paperless-ngx/paperless-ngx:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+      - gotenberg
+      - tika
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBENGINE: mariadb
+      PAPERLESS_DBHOST: db
+      PAPERLESS_DBUSER: paperless # only needed if non-default username
+      PAPERLESS_DBPASS: paperless # only needed if non-default password
+      PAPERLESS_DBPORT: 3306
+      PAPERLESS_TIKA_ENABLED: 1
+      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
+      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
+
+  gotenberg:
+    image: docker.io/gotenberg/gotenberg:7.6
+    restart: unless-stopped
+    command:
+      - "gotenberg"
+      - "--chromium-disable-routes=true"
+
+  tika:
+    image: ghcr.io/paperless-ngx/tika:latest
+    restart: unless-stopped
+
+volumes:
+  data:
+  media:
+  dbdata:
+  redisdata:

docker/compose/docker-compose.mariadb.yml

@@ -0,0 +1,83 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), MariaDB is used as the database server.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: docker.io/library/redis:7
+    restart: unless-stopped
+    volumes:
+      - redisdata:/data
+
+  db:
+    image: docker.io/library/mariadb:10
+    restart: unless-stopped
+    volumes:
+      - dbdata:/var/lib/mysql
+    environment:
+      MARIADB_HOST: paperless
+      MARIADB_DATABASE: paperless
+      MARIADB_USER: paperless
+      MARIADB_PASSWORD: paperless
+      MARIADB_ROOT_PASSWORD: paperless
+    ports:
+      - "3306:3306"
+
+  webserver:
+    image: ghcr.io/paperless-ngx/paperless-ngx:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBENGINE: mariadb
+      PAPERLESS_DBHOST: db
+      PAPERLESS_DBUSER: paperless # only needed if non-default username
+      PAPERLESS_DBPASS: paperless # only needed if non-default password
+      PAPERLESS_DBPORT: 3306
+
+
+volumes:
+  data:
+  media:
+  dbdata:
+  redisdata:

docker/compose/docker-compose.portainer.yml

@@ -31,13 +31,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.postgres-tika.yml

@@ -33,13 +33,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
@@ -77,7 +77,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.6
     restart: unless-stopped
     command:
       - "gotenberg"

docker/compose/docker-compose.postgres.yml

@@ -29,13 +29,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.sqlite-tika.yml

@@ -33,7 +33,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data
@@ -65,7 +65,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.6
     restart: unless-stopped
     command:
       - "gotenberg"

docker/compose/docker-compose.sqlite.yml

@@ -26,7 +26,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data

docker/docker-entrypoint.sh

@@ -2,45 +2,142 @@
 
 set -e
 
+# Adapted from:
+# https://github.com/docker-library/postgres/blob/master/docker-entrypoint.sh
+# usage: file_env VAR
+#    ie: file_env 'XYZ_DB_PASSWORD' will allow for "$XYZ_DB_PASSWORD_FILE" to
+# fill in the value of "$XYZ_DB_PASSWORD" from a file, especially for Docker's
+# secrets feature
+file_env() {
+	local -r var="$1"
+	local -r fileVar="${var}_FILE"
+
+	# Basic validation
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+		exit 1
+	fi
+
+	# Only export var if the _FILE exists
+	if [ "${!fileVar:-}" ]; then
+		# And the file exists
+		if [[ -f ${!fileVar} ]]; then
+			echo "Setting ${var} from file"
+			val="$(< "${!fileVar}")"
+			export "$var"="$val"
+		else
+			echo "File ${!fileVar} doesn't exist"
+			exit 1
+		fi
+	fi
+
+}
+
 # Source: https://github.com/sameersbn/docker-gitlab/
 map_uidgid() {
-	USERMAP_ORIG_UID=$(id -u paperless)
-	USERMAP_ORIG_GID=$(id -g paperless)
-	USERMAP_NEW_UID=${USERMAP_UID:-$USERMAP_ORIG_UID}
-	USERMAP_NEW_GID=${USERMAP_GID:-${USERMAP_ORIG_GID:-$USERMAP_NEW_UID}}
-	if [[ ${USERMAP_NEW_UID} != "${USERMAP_ORIG_UID}" || ${USERMAP_NEW_GID} != "${USERMAP_ORIG_GID}" ]]; then
-		echo "Mapping UID and GID for paperless:paperless to $USERMAP_NEW_UID:$USERMAP_NEW_GID"
-		usermod -o -u "${USERMAP_NEW_UID}" paperless
-		groupmod -o -g "${USERMAP_NEW_GID}" paperless
+	local -r usermap_original_uid=$(id -u paperless)
+	local -r usermap_original_gid=$(id -g paperless)
+	local -r usermap_new_uid=${USERMAP_UID:-$usermap_original_uid}
+	local -r usermap_new_gid=${USERMAP_GID:-${usermap_original_gid:-$usermap_new_uid}}
+	if [[ ${usermap_new_uid} != "${usermap_original_uid}" || ${usermap_new_gid} != "${usermap_original_gid}" ]]; then
+		echo "Mapping UID and GID for paperless:paperless to $usermap_new_uid:$usermap_new_gid"
+		usermod -o -u "${usermap_new_uid}" paperless
+		groupmod -o -g "${usermap_new_gid}" paperless
 	fi
 }
 
+map_folders() {
+	# Export these so they can be used in docker-prepare.sh
+	export DATA_DIR="${PAPERLESS_DATA_DIR:-/usr/src/paperless/data}"
+	export MEDIA_ROOT_DIR="${PAPERLESS_MEDIA_ROOT:-/usr/src/paperless/media}"
+	export CONSUME_DIR="${PAPERLESS_CONSUMPTION_DIR:-/usr/src/paperless/consume}"
+}
+
+nltk_data () {
+	# Store the NLTK data outside the Docker container
+	local -r nltk_data_dir="${DATA_DIR}/nltk"
+	local -r truthy_things=("yes y 1 t true")
+
+	# If not set, or it looks truthy
+	if [[ -z "${PAPERLESS_ENABLE_NLTK}" ]] || [[ "${truthy_things[*]}" =~ ${PAPERLESS_ENABLE_NLTK,} ]]; then
+
+		# Download or update the snowball stemmer data
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" snowball_data
+
+		# Download or update the stopwords corpus
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" stopwords
+
+		# Download or update the punkt tokenizer data
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" punkt
+
+	else
+		echo "Skipping NLTK data download"
+
+	fi
+
+}
+
 initialize() {
+
+	# Setup environment from secrets before anything else
+	for env_var in \
+		PAPERLESS_DBUSER \
+		PAPERLESS_DBPASS \
+		PAPERLESS_SECRET_KEY \
+		PAPERLESS_AUTO_LOGIN_USERNAME \
+		PAPERLESS_ADMIN_USER \
+		PAPERLESS_ADMIN_MAIL \
+		PAPERLESS_ADMIN_PASSWORD \
+		PAPERLESS_REDIS; do
+		# Check for a version of this var with _FILE appended
+		# and convert the contents to the env var value
+		file_env ${env_var}
+	done
+
+	# Change the user and group IDs if needed
 	map_uidgid
 
-	for dir in export data data/index media media/documents media/documents/originals media/documents/thumbnails; do
-		if [[ ! -d "../$dir" ]]; then
-			echo "Creating directory ../$dir"
-			mkdir ../$dir
+	# Check for overrides of certain folders
+	map_folders
+
+	local -r export_dir="/usr/src/paperless/export"
+
+	for dir in \
+		"${export_dir}" \
+		"${DATA_DIR}" "${DATA_DIR}/index" \
+		"${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails" \
+		"${CONSUME_DIR}"; do
+		if [[ ! -d "${dir}" ]]; then
+			echo "Creating directory ${dir}"
+			mkdir "${dir}"
 		fi
 	done
 
-	echo "Creating directory /tmp/paperless"
-	mkdir -p /tmp/paperless
+	local -r tmp_dir="/tmp/paperless"
+	echo "Creating directory ${tmp_dir}"
+	mkdir -p "${tmp_dir}"
+
+	nltk_data
 
 	set +e
 	echo "Adjusting permissions of paperless files. This may take a while."
-	chown -R paperless:paperless /tmp/paperless
-	find .. -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	chown -R paperless:paperless ${tmp_dir}
+	for dir in \
+		"${export_dir}" \
+		"${DATA_DIR}" \
+		"${MEDIA_ROOT_DIR}" \
+		"${CONSUME_DIR}"; do
+		find "${dir}" -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	done
 	set -e
 
-	gosu paperless /sbin/docker-prepare.sh
+	"${gosu_cmd[@]}" /sbin/docker-prepare.sh
 }
 
 install_languages() {
 	echo "Installing languages..."
 
-	local langs="$1"
+	local -r langs="$1"
 	read -ra langs <<<"$langs"
 
 	# Check that it is not empty
@@ -76,6 +173,11 @@ install_languages() {
 
 echo "Paperless-ngx docker container starting..."
 
+gosu_cmd=(gosu paperless)
+if [ "$(id -u)" == "$(id -u paperless)" ]; then
+	gosu_cmd=()
+fi
+
 # Install additional languages if specified
 if [[ -n "$PAPERLESS_OCR_LANGUAGES" ]]; then
 	install_languages "$PAPERLESS_OCR_LANGUAGES"
@@ -85,7 +187,7 @@ initialize
 
 if [[ "$1" != "/"* ]]; then
 	echo Executing management command "$@"
-	exec gosu paperless python3 manage.py "$@"
+	exec "${gosu_cmd[@]}" python3 manage.py "$@"
 else
 	echo Executing "$@"
 	exec "$@"

docker/docker-prepare.sh

@@ -3,16 +3,41 @@
 set -e
 
 wait_for_postgres() {
-	attempt_num=1
-	max_attempts=5
+	local attempt_num=1
+	local -r max_attempts=5
 
 	echo "Waiting for PostgreSQL to start..."
 
-	host="${PAPERLESS_DBHOST:=localhost}"
-	port="${PAPERLESS_DBPORT:=5432}"
+	local -r host="${PAPERLESS_DBHOST:-localhost}"
+	local -r port="${PAPERLESS_DBPORT:-5432}"
 
+	# Disable warning, host and port can't have spaces
+	# shellcheck disable=SC2086
+	while [ ! "$(pg_isready -h ${host} -p ${port})" ]; do
 
-	while [ ! "$(pg_isready -h $host -p $port)" ]; do
+		if [ $attempt_num -eq $max_attempts ]; then
+			echo "Unable to connect to database."
+			exit 1
+		else
+			echo "Attempt $attempt_num failed! Trying again in 5 seconds..."
+
+		fi
+
+		attempt_num=$(("$attempt_num" + 1))
+		sleep 5
+	done
+}
+
+wait_for_mariadb() {
+	echo "Waiting for MariaDB to start..."
+
+	local -r host="${PAPERLESS_DBHOST:=localhost}"
+	local -r port="${PAPERLESS_DBPORT:=3306}"
+
+	local attempt_num=1
+	local -r max_attempts=5
+
+	while ! true > /dev/tcp/$host/$port; do
 
 		if [ $attempt_num -eq $max_attempts ]; then
 			echo "Unable to connect to database."
@@ -43,17 +68,18 @@ migrations() {
 		flock 200
 		echo "Apply database migrations..."
 		python3 manage.py migrate
-	) 200>/usr/src/paperless/data/migration_lock
+	) 200>"${DATA_DIR}/migration_lock"
 }
 
 search_index() {
-	index_version=1
-	index_version_file=/usr/src/paperless/data/.index_version
 
-	if [[ (! -f "$index_version_file") || $(<$index_version_file) != "$index_version" ]]; then
+	local -r index_version=1
+	local -r index_version_file=${DATA_DIR}/.index_version
+
+	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
 		echo "Search index out of date. Updating..."
-		python3 manage.py document_index reindex
-		echo $index_version | tee $index_version_file >/dev/null
+		python3 manage.py document_index reindex --no-progress-bar
+		echo ${index_version} | tee "${index_version_file}" >/dev/null
 	fi
 }
 
@@ -63,8 +89,50 @@ superuser() {
 	fi
 }
 
+custom_container_init() {
+	# Mostly borrowed from the LinuxServer.io base image
+	# https://github.com/linuxserver/docker-baseimage-ubuntu/tree/bionic/root/etc/cont-init.d
+	local -r custom_script_dir="/custom-cont-init.d"
+	# Tamper checking.
+	# Don't run files which are owned by anyone except root
+	# Don't run files which are writeable by others
+	if [ -d "${custom_script_dir}" ]; then
+		if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 ! -user root)" ]; then
+			echo "**** Potential tampering with custom scripts detected ****"
+			echo "**** The folder '${custom_script_dir}' must be owned by root ****"
+			return 0
+		fi
+		if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 -perm -o+w)" ]; then
+			echo "**** The folder '${custom_script_dir}' or some of contents have write permissions for others, which is a security risk. ****"
+			echo "**** Please review the permissions and their contents to make sure they are owned by root, and can only be modified by root. ****"
+			return 0
+		fi
+
+		# Make sure custom init directory has files in it
+		if [ -n "$(/bin/ls -A "${custom_script_dir}" 2>/dev/null)" ]; then
+			echo "[custom-init] files found in ${custom_script_dir} executing"
+			# Loop over files in the directory
+			for SCRIPT in "${custom_script_dir}"/*; do
+				NAME="$(basename "${SCRIPT}")"
+				if [ -f "${SCRIPT}" ]; then
+					echo "[custom-init] ${NAME}: executing..."
+					/bin/bash "${SCRIPT}"
+					echo "[custom-init] ${NAME}: exited $?"
+				elif [ ! -f "${SCRIPT}" ]; then
+					echo "[custom-init] ${NAME}: is not a file"
+				fi
+			done
+		else
+			echo "[custom-init] no custom files found exiting..."
+		fi
+
+	fi
+}
+
 do_work() {
-	if [[ -n "${PAPERLESS_DBHOST}" ]]; then
+	if [[ "${PAPERLESS_DBENGINE}" == "mariadb" ]]; then
+		wait_for_mariadb
+	elif [[ -n "${PAPERLESS_DBHOST}" ]]; then
 		wait_for_postgres
 	fi
 
@@ -76,6 +144,9 @@ do_work() {
 
 	superuser
 
+	# Leave this last thing
+	custom_container_init
+
 }
 
 do_work

docker/flower-conditional.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "Checking if we should start flower..."
+
+if [[ -n  "${PAPERLESS_ENABLE_FLOWER}" ]]; then
+	celery --app paperless flower
+fi

docker/install_management_commands.sh

@@ -2,7 +2,18 @@
 
 set -eu
 
-for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
+for command in decrypt_documents \
+	document_archiver \
+	document_exporter \
+	document_importer \
+	mail_fetcher \
+	document_create_classifier \
+	document_index \
+	document_renamer \
+	document_retagger \
+	document_thumbnails \
+	document_sanity_checker \
+	manage_superuser;
 do
 	echo "installing $command..."
 	sed "s/management_command/$command/g" management_script.sh > /usr/local/bin/$command

docker/paperless_cmd.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+rootless_args=()
+if [ "$(id -u)" == "$(id -u paperless)" ]; then
+	rootless_args=(
+		--user
+		paperless
+		--logfile
+		supervisord.log
+		--pidfile
+		supervisord.pid
+	)
+fi
+
+exec /usr/local/bin/supervisord -c /etc/supervisord.conf "${rootless_args[@]}"

docker/supervisord.conf

@@ -10,7 +10,7 @@ user=root
 [program:gunicorn]
 command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.asgi:application
 user=paperless
-
+priority = 1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
@@ -19,17 +19,41 @@ stderr_logfile_maxbytes=0
 [program:consumer]
 command=python3 manage.py document_consumer
 user=paperless
-
+stopsignal=INT
+priority = 20
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:scheduler]
-command=python3 manage.py qcluster
+[program:celery]
+
+command = celery --app paperless worker --loglevel INFO
 user=paperless
 stopasgroup = true
-
+stopwaitsecs = 60
+priority = 5
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:celery-beat]
+
+command = celery --app paperless beat --loglevel INFO
+user=paperless
+stopasgroup = true
+priority = 10
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:celery-flower]
+command = /usr/local/bin/flower-conditional.sh
+user = paperless
+startsecs = 0
+priority = 40
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr

docker/wait-for-redis.py

@@ -18,7 +18,7 @@ if __name__ == "__main__":
 
     REDIS_URL: Final[str] = os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")
 
-    print(f"Waiting for Redis: {REDIS_URL}", flush=True)
+    print(f"Waiting for Redis...", flush=True)
 
     attempt = 0
     with Redis.from_url(url=REDIS_URL) as client:
@@ -26,17 +26,19 @@ if __name__ == "__main__":
             try:
                 client.ping()
                 break
-            except Exception:
+            except Exception as e:
                 print(
-                    f"Redis ping #{attempt} failed, waiting {RETRY_SLEEP_SECONDS}s",
+                    f"Redis ping #{attempt} failed.\n"
+                    f"Error: {str(e)}.\n"
+                    f"Waiting {RETRY_SLEEP_SECONDS}s",
                     flush=True,
                 )
                 time.sleep(RETRY_SLEEP_SECONDS)
                 attempt += 1
 
     if attempt >= MAX_RETRY_COUNT:
-        print(f"Failed to connect to: {REDIS_URL}")
+        print(f"Failed to connect to redis using environment variable PAPERLESS_REDIS.")
         sys.exit(os.EX_UNAVAILABLE)
     else:
-        print(f"Connected to Redis broker: {REDIS_URL}")
+        print(f"Connected to Redis broker.")
         sys.exit(os.EX_OK)

docs/Dockerfile

@@ -1,17 +0,0 @@
-FROM python:3.5.1
-
-# Install Sphinx and Pygments
-RUN pip install --no-cache-dir Sphinx Pygments \
-  # Setup directories, copy data
-  && mkdir /build
-
-COPY . /build
-WORKDIR /build/docs
-
-# Build documentation
-RUN make html
-
-# Start webserver
-WORKDIR /build/docs/_build/html
-EXPOSE 8000/tcp
-CMD ["python3", "-m", "http.server"]

docs/Makefile

@@ -24,6 +24,7 @@ I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html       to make standalone HTML files"
+	@echo "  livehtml   to preview changes with live reload in your browser"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
 	@echo "  singlehtml to make a single large HTML file"
 	@echo "  pickle     to make pickle files"
@@ -54,6 +55,9 @@ html:
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
+livehtml:
+	sphinx-autobuild "./" "$(BUILDDIR)" $(O)
+
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo

docs/_static/css/custom.css

@@ -64,6 +64,10 @@ body {
   color: var(--color-text-body);
 }
 
+.rst-content p {
+  word-break: break-word;
+}
+
 h1, h2, h3, h4, h5, h6 {
   font-family: inherit;
 }
@@ -435,7 +439,8 @@ a.image-reference img {
 }
 
 .rst-content code.literal,
-.rst-content tt.literal {
+.rst-content tt.literal,
+html.writer-html5 .rst-content dl.footnote code {
   border-color: var(--color-border);
   background-color: var(--color-border);
   color: var(--color-text-code-inline)

docs/_static/js/darkmode.js

@@ -1,47 +1,47 @@
-let toggleButton;
-let icon;
+let toggleButton
+let icon
 
 function load() {
-	"use strict";
+	'use strict'
 
-	toggleButton = document.createElement("button");
-	toggleButton.setAttribute("title", "Toggle dark mode");
-	toggleButton.classList.add("dark-mode-toggle");
-	icon = document.createElement("i");
-	icon.classList.add("fa", darkModeState ? "fa-sun-o" : "fa-moon-o");
-	toggleButton.appendChild(icon);
-	document.body.prepend(toggleButton);
+	toggleButton = document.createElement('button')
+	toggleButton.setAttribute('title', 'Toggle dark mode')
+	toggleButton.classList.add('dark-mode-toggle')
+	icon = document.createElement('i')
+	icon.classList.add('fa', darkModeState ? 'fa-sun-o' : 'fa-moon-o')
+	toggleButton.appendChild(icon)
+	document.body.prepend(toggleButton)
 
 	// Listen for changes in the OS settings
 	// addListener is used because older versions of Safari don't support addEventListener
 	// prefersDarkQuery set in <head>
 	if (prefersDarkQuery) {
 		prefersDarkQuery.addListener(function (evt) {
-			toggleDarkMode(evt.matches);
-		});
+			toggleDarkMode(evt.matches)
+		})
 	}
 
 	// Initial setting depending on the prefers-color-mode or localstorage
 	// darkModeState should be set in the document <head> to prevent flash
-	if (darkModeState == undefined) darkModeState = false;
-	toggleDarkMode(darkModeState);
+	if (darkModeState == undefined) darkModeState = false
+	toggleDarkMode(darkModeState)
 
 	// Toggles the "dark-mode" class on click and sets localStorage state
-	toggleButton.addEventListener("click", () => {
-		darkModeState = !darkModeState;
+	toggleButton.addEventListener('click', () => {
+		darkModeState = !darkModeState
 
-		toggleDarkMode(darkModeState);
-		localStorage.setItem("dark-mode", darkModeState);
-	});
+		toggleDarkMode(darkModeState)
+		localStorage.setItem('dark-mode', darkModeState)
+	})
 }
 
 function toggleDarkMode(state) {
-	document.documentElement.classList.toggle("dark-mode", state);
-	document.documentElement.classList.toggle("light-mode", !state);
-	icon.classList.remove("fa-sun-o");
-	icon.classList.remove("fa-moon-o");
-	icon.classList.add(state ? "fa-sun-o" : "fa-moon-o");
-	darkModeState = state;
+	document.documentElement.classList.toggle('dark-mode', state)
+	document.documentElement.classList.toggle('light-mode', !state)
+	icon.classList.remove('fa-sun-o')
+	icon.classList.remove('fa-moon-o')
+	icon.classList.add(state ? 'fa-sun-o' : 'fa-moon-o')
+	darkModeState = state
 }
 
-document.addEventListener("DOMContentLoaded", load);
+document.addEventListener('DOMContentLoaded', load)

docs/administration.rst

@@ -35,13 +35,15 @@ Options available to docker installations:
     ``/var/lib/docker/volumes`` on the host and you need to be root in order
     to access them.
 
-    Paperless uses 3 volumes:
+    Paperless uses 4 volumes:
 
     *   ``paperless_media``: This is where your documents are stored.
     *   ``paperless_data``: This is where auxillary data is stored. This
         folder also contains the SQLite database, if you use it.
     *   ``paperless_pgdata``: Exists only if you use PostgreSQL and contains
         the database.
+    *   ``paperless_dbdata``: Exists only if you use MariaDB and contains
+        the database.
 
 Options available to bare-metal and non-docker installations:
 
@@ -49,7 +51,7 @@ Options available to bare-metal and non-docker installations:
     crashes at some point or your disk fails, you can simply copy the folder back
     into place and it works.
 
-    When using PostgreSQL, you'll also have to backup the database.
+    When using PostgreSQL or MariaDB, you'll also have to backup the database.
 
 .. _migrating-restoring:
 
@@ -118,10 +120,10 @@ Then you can start paperless-ngx with ``-d`` to have it run in the background.
                 image: ghcr.io/paperless-ngx/paperless-ngx:latest
 
     .. note::
-        In version 1.7.1 and onwards, the Docker image can now pinned to a release series.
+        In version 1.7.1 and onwards, the Docker image can now be pinned to a release series.
         This is often combined with automatic updaters such as Watchtower to allow safer
         unattended upgrading to new bugfix releases only.  It is still recommended to always
-        review release notes before upgrading.  To ping your install to a release series, edit
+        review release notes before upgrading.  To pin your install to a release series, edit
         the ``docker-compose.yml`` find the line that says
 
             .. code::
@@ -287,6 +289,10 @@ When you use the provided docker compose script, put the export inside the
 ``export`` folder in your paperless source directory. Specify ``../export``
 as the ``source``.
 
+.. note::
+
+    Importing from a previous version of Paperless may work, but for best results
+    it is suggested to match the versions.
 
 .. _utilities-retagger:
 
@@ -306,6 +312,7 @@ there are tools for it.
     -c, --correspondent
     -T, --tags
     -t, --document_type
+    -s, --storage_path
     -i, --inbox-only
     --use-first
     -f, --overwrite
@@ -314,7 +321,7 @@ Run this after changing or adding matching rules. It'll loop over all
 of the documents in your database and attempt to match documents
 according to the new rules.
 
-Specify any combination of ``-c``, ``-T`` and ``-t`` to have the
+Specify any combination of ``-c``, ``-T``, ``-t`` and ``-s`` to have the
 retagger perform matching of the specified metadata type. If you don't
 specify any of these options, the document retagger won't do anything.
 
@@ -386,8 +393,8 @@ the naming scheme.
 
 .. warning::
 
-    Since this command moves you documents around a lot, it is advised to to
-    a backup before. The renaming logic is robust and will never overwrite
+    Since this command moves your documents, it is advised to do
+    a backup beforehand. The renaming logic is robust and will never overwrite
     or delete a file, but you can't ever be careful enough.
 
 .. code::
@@ -440,6 +447,14 @@ command:
 
 The command takes no arguments and processes all your mail accounts and rules.
 
+ .. note::
+
+    As of October 2022 Microsoft no longer supports IMAP authentication for Exchange
+    servers, thus Exchange is no longer supported until a solution is implemented in
+    the Python IMAP library used by Paperless. See  `learn.microsoft.com`_
+
+.. _learn.microsoft.com: https://learn.microsoft.com/en-us/exchange/clients-and-mobile-in-exchange-online/deprecation-of-basic-authentication-exchange-online
+
 .. _utilities-archiver:
 
 Creating archived documents

docs/advanced_usage.rst

@@ -7,12 +7,12 @@ easier.
 
 .. _advanced-matching:
 
-Matching tags, correspondents and document types
-################################################
+Matching tags, correspondents, document types, and storage paths
+################################################################
 
-Paperless will compare the matching algorithms defined by every tag and
-correspondent already set in your database to see if they apply to the text in
-a document.  In other words, if you defined a tag called ``Home Utility``
+Paperless will compare the matching algorithms defined by every tag, correspondent,
+document type, and storage path in your database to see if they apply to the text
+in a document. In other words, if you define a tag called ``Home Utility``
 that had a ``match`` property of ``bc hydro`` and a ``matching_algorithm`` of
 ``literal``, Paperless will automatically tag your newly-consumed document with
 your ``Home Utility`` tag so long as the text ``bc hydro`` appears in the body
@@ -22,10 +22,10 @@ The matching logic is quite powerful. It supports searching the text of your
 document with different algorithms, and as such, some experimentation may be
 necessary to get things right.
 
-In order to have a tag, correspondent, or type assigned automatically to newly
-consumed documents, assign a match and matching algorithm using the web
-interface. These settings define when to assign correspondents, tags, and types
-to documents.
+In order to have a tag, correspondent, document type, or storage path assigned
+automatically to newly consumed documents, assign a match and matching algorithm
+using the web interface. These settings define when to assign tags, correspondents,
+document types, and storage paths to documents.
 
 The following algorithms are available:
 
@@ -37,7 +37,7 @@ The following algorithms are available:
 * **Literal:** Matches only if the match appears exactly as provided (i.e. preserve ordering) in the PDF.
 * **Regular expression:** Parses the match as a regular expression and tries to
   find a match within the document.
-* **Fuzzy match:** I dont know. Look at the source.
+* **Fuzzy match:** I don't know. Look at the source.
 * **Auto:** Tries to automatically match new documents. This does not require you
   to set a match. See the notes below.
 
@@ -47,9 +47,9 @@ defining a match text of ``"Bank of America" BofA`` using the *any* algorithm,
 will match documents that contain either "Bank of America" or "BofA", but will
 not match documents containing "Bank of South America".
 
-Then just save your tag/correspondent and run another document through the
-consumer.  Once complete, you should see the newly-created document,
-automatically tagged with the appropriate data.
+Then just save your tag, correspondent, document type, or storage path and run
+another document through the consumer.  Once complete, you should see the
+newly-created document, automatically tagged with the appropriate data.
 
 
 .. _advanced-automatic_matching:
@@ -58,9 +58,9 @@ Automatic matching
 ==================
 
 Paperless-ngx comes with a new matching algorithm called *Auto*. This matching
-algorithm tries to assign tags, correspondents, and document types to your
-documents based on how you have already assigned these on existing documents. It
-uses a neural network under the hood.
+algorithm tries to assign tags, correspondents, document types, and storage paths
+to your documents based on how you have already assigned these on existing documents.
+It uses a neural network under the hood.
 
 If, for example, all your bank statements of your account 123 at the Bank of
 America are tagged with the tag "bofa_123" and the matching algorithm of this
@@ -80,20 +80,21 @@ feature:
   that the neural network only learns from documents which you have correctly
   tagged before.
 * The matching algorithm can only work if there is a correlation between the
-  tag, correspondent, or document type and the document itself. Your bank
-  statements usually contain your bank account number and the name of the bank,
-  so this works reasonably well, However, tags such as "TODO" cannot be
-  automatically assigned.
+  tag, correspondent, document type, or storage path and the document itself.
+  Your bank statements usually contain your bank account number and the name
+  of the bank, so this works reasonably well, However, tags such as "TODO"
+  cannot be automatically assigned.
 * The matching algorithm needs a reasonable number of documents to identify when
-  to assign tags, correspondents, and types. If one out of a thousand documents
-  has the correspondent "Very obscure web shop I bought something five years
-  ago", it will probably not assign this correspondent automatically if you buy
-  something from them again. The more documents, the better.
+  to assign tags, correspondents, storage paths, and types. If one out of a
+  thousand documents has the correspondent "Very obscure web shop I bought
+  something five years ago", it will probably not assign this correspondent
+  automatically if you buy something from them again. The more documents, the better.
 * Paperless also needs a reasonable amount of negative examples to decide when
-  not to assign a certain tag, correspondent or type. This will usually be the
-  case as you start filling up paperless with documents. Example: If all your
-  documents are either from "Webshop" and "Bank", paperless will assign one of
-  these correspondents to ANY new document, if both are set to automatic matching.
+  not to assign a certain tag, correspondent, document type, or storage path. This will
+  usually be the case as you start filling up paperless with documents.
+  Example: If all your documents are either from "Webshop" and "Bank", paperless
+  will assign one of these correspondents to ANY new document, if both are set
+  to automatic matching.
 
 Hooking into the consumption process
 ####################################
@@ -120,10 +121,10 @@ Pre-consumption script
 ======================
 
 Executed after the consumer sees a new document in the consumption folder, but
-before any processing of the document is performed. This script receives exactly
-one argument:
+before any processing of the document is performed. This script can access the
+following relevant environment variables set:
 
-* Document file name
+* ``DOCUMENT_SOURCE_PATH``
 
 A simple but common example for this would be creating a simple script like
 this:
@@ -133,7 +134,7 @@ this:
 .. code:: bash
 
     #!/usr/bin/env bash
-    pdf2pdfocr.py -i ${1}
+    pdf2pdfocr.py -i ${DOCUMENT_SOURCE_PATH}
 
 ``/etc/paperless.conf``
 
@@ -156,16 +157,21 @@ Post-consumption script
 =======================
 
 Executed after the consumer has successfully processed a document and has moved it
-into paperless. It receives the following arguments:
+into paperless. It receives the following environment variables:
 
-* Document id
-* Generated file name
-* Source path
-* Thumbnail path
-* Download URL
-* Thumbnail URL
-* Correspondent
-* Tags
+* ``DOCUMENT_ID``
+* ``DOCUMENT_FILE_NAME``
+* ``DOCUMENT_CREATED``
+* ``DOCUMENT_MODIFIED``
+* ``DOCUMENT_ADDED``
+* ``DOCUMENT_SOURCE_PATH``
+* ``DOCUMENT_ARCHIVE_PATH``
+* ``DOCUMENT_THUMBNAIL_PATH``
+* ``DOCUMENT_DOWNLOAD_URL``
+* ``DOCUMENT_THUMBNAIL_URL``
+* ``DOCUMENT_CORRESPONDENT``
+* ``DOCUMENT_TAGS``
+* ``DOCUMENT_ORIGINAL_FILENAME``
 
 The script can be in any language, but for a simple shell script
 example, you can take a look at `post-consumption-example.sh`_ in this project.
@@ -212,7 +218,8 @@ using the identifier which it has assigned to each document. You will end up get
 files like ``0000123.pdf`` in your media directory. This isn't necessarily a bad
 thing, because you normally don't have to access these files manually. However, if
 you wish to name your files differently, you can do that by adjusting the
-``PAPERLESS_FILENAME_FORMAT`` configuration option.
+``PAPERLESS_FILENAME_FORMAT`` configuration option. Paperless adds the correct
+file extension e.g. ``.pdf``, ``.jpg`` automatically.
 
 This variable allows you to configure the filename (folders are allowed) using
 placeholders. For example, configuring this to
@@ -251,12 +258,18 @@ Paperless provides the following placeholders within filenames:
 * ``{tag_list}``: A comma separated list of all tags assigned to the document.
 * ``{title}``: The title of the document.
 * ``{created}``: The full date (ISO format) the document was created.
-* ``{created_year}``: Year created only.
+* ``{created_year}``: Year created only, formatted as the year with century.
+* ``{created_year_short}``: Year created only, formatted as the year without century, zero padded.
 * ``{created_month}``: Month created only (number 01-12).
+* ``{created_month_name}``: Month created name, as per locale
+* ``{created_month_name_short}``: Month created abbreviated name, as per locale
 * ``{created_day}``: Day created only (number 01-31).
 * ``{added}``: The full date (ISO format) the document was added to paperless.
 * ``{added_year}``: Year added only.
+* ``{added_year_short}``: Year added only, formatted as the year without century, zero padded.
 * ``{added_month}``: Month added only (number 01-12).
+* ``{added_month_name}``: Month added name, as per locale
+* ``{added_month_name_short}``: Month added abbreviated name, as per locale
 * ``{added_day}``: Day added only (number 01-31).
 
 
@@ -268,6 +281,17 @@ If paperless detects that two documents share the same filename, paperless will
 append ``_01``, ``_02``, etc to the filename. This happens if all the placeholders in a filename
 evaluate to the same value.
 
+.. hint::
+    You can affect how empty placeholders are treated by changing the following setting to
+    `true`.
+
+    .. code::
+
+        PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=True
+
+    Doing this results in all empty placeholders resolving to "" instead of "none" as stated above.
+    Spaces before empty placeholders are removed as well, empty directories are omitted.
+
 .. hint::
 
     Paperless checks the filename of a document whenever it is saved. Therefore,
@@ -290,3 +314,106 @@ evaluate to the same value.
 
     However, keep in mind that inside docker, if files get stored outside of the
     predefined volumes, they will be lost after a restart of paperless.
+
+
+Storage paths
+#############
+
+One of the best things in Paperless is that you can not only access the documents via the
+web interface, but also via the file system.
+
+When as single storage layout is not sufficient for your use case, storage paths come to
+the rescue. Storage paths allow you to configure more precisely where each document is stored
+in the file system.
+
+- Each storage path is a `PAPERLESS_FILENAME_FORMAT` and follows the rules described above
+- Each document is assigned a storage path using the matching algorithms described above, but
+  can be overwritten at any time
+
+For example, you could define the following two storage paths:
+
+1. Normal communications are put into a folder structure sorted by `year/correspondent`
+2. Communications with insurance companies are stored in a flat structure with longer file names,
+   but containing the full date of the correspondence.
+
+.. code::
+
+    By Year = {created_year}/{correspondent}/{title}
+    Insurances = Insurances/{correspondent}/{created_year}-{created_month}-{created_day} {title}
+
+
+If you then map these storage paths to the documents, you might get the following result.
+For simplicity, `By Year` defines the same structure as in the previous example above.
+
+.. code:: text
+
+   2019/                                   # By Year
+      My bank/
+        Statement January.pdf
+        Statement February.pdf
+
+    Insurances/                           # Insurances
+      Healthcare 123/
+        2022-01-01 Statement January.pdf
+        2022-02-02 Letter.pdf
+        2022-02-03 Letter.pdf
+      Dental 456/
+        2021-12-01 New Conditions.pdf
+
+
+.. hint::
+
+    Defining a storage path is optional. If no storage path is defined for a document, the global
+    `PAPERLESS_FILENAME_FORMAT` is applied.
+
+.. caution::
+
+    If you adjust the format of an existing storage path, old documents don't get relocated automatically.
+    You need to run the :ref:`document renamer <utilities-renamer>` to adjust their pathes.
+
+.. _advanced-celery-monitoring:
+
+Celery Monitoring
+#################
+
+The monitoring tool `Flower <https://flower.readthedocs.io/en/latest/index.html>`_ can be used to view more
+detailed information about the health of the celery workers used for asynchronous tasks.  This includes details
+on currently running, queued and completed tasks, timing and more.  Flower can also be used with Prometheus, as it
+exports metrics.  For details on its capabilities, refer to the Flower documentation.
+
+To configure Flower further, create a `flowerconfig.py` and place it into the `src/paperless` directory.  For
+a Docker installation, you can use volumes to accomplish this:
+
+.. code:: yaml
+
+    services:
+      # ...
+      webserver:
+        # ...
+        volumes:
+          - /path/to/my/flowerconfig.py:/usr/src/paperless/src/paperless/flowerconfig.py:ro
+
+Custom Container Initialization
+###############################
+
+The Docker image includes the ability to run custom user scripts during startup.  This could be
+utilized for installing additional tools or Python packages, for example.
+
+To utilize this, mount a folder containing your scripts to the custom initialization directory, `/custom-cont-init.d`
+and place scripts you wish to run inside.  For security, the folder and its contents must be owned by `root`.
+Additionally, scripts must only be writable by `root`.
+
+Your scripts will be run directly before the webserver completes startup.  Scripts will be run by the `root` user.
+This is an advanced functionality with which you could break functionality or lose data.
+
+For example, using Docker Compose:
+
+
+.. code:: yaml
+
+    services:
+      # ...
+      webserver:
+        # ...
+        volumes:
+          - /path/to/my/scripts:/custom-cont-init.d:ro

docs/api.rst

@@ -31,7 +31,8 @@ The objects served by the document endpoint contain the following fields:
 *   ``tags``: List of IDs of tags assigned to this document, or empty list.
 *   ``document_type``: Document type of this document, or null.
 *   ``correspondent``:  Correspondent of this document or null.
-*   ``created``: The date at which this document was created.
+*   ``created``: The date time at which this document was created.
+*   ``created_date``: The date (YYYY-MM-DD) at which this document was created. Optional. If also passed with created, this is ignored.
 *   ``modified``: The date at which this document was last edited in paperless. Read-only.
 *   ``added``: The date at which this document was added to paperless. Read-only.
 *   ``archive_serial_number``: The identifier of this document in a physical document archive.
@@ -240,11 +241,13 @@ be instructed to consume the document from there.
 The endpoint supports the following optional form fields:
 
 *   ``title``: Specify a title that the consumer should use for the document.
+*   ``created``: Specify a DateTime where the document was created (e.g. "2016-04-19" or "2016-04-19 06:15:00+02:00").
 *   ``correspondent``: Specify the ID of a correspondent that the consumer should use for the document.
 *   ``document_type``: Similar to correspondent.
 *   ``tags``: Similar to correspondent. Specify this multiple times to have multiple tags added
     to the document.
 
+
 The endpoint will immediately return "OK" if the document consumption process
 was started successfully. No additional status information about the consumption
 process itself is available, since that happens in a different process.

docs/changelog.md

@@ -1,206 +1,536 @@
 # Changelog
 
+## paperless-ngx 1.9.2
+
+### Bug Fixes
+
+- Bugfix: Allow PAPERLESS_OCR_CLEAN=none [@shamoon](https://github.com/shamoon) ([#1670](https://github.com/paperless-ngx/paperless-ngx/pull/1670))
+
+### All App Changes
+
+- Chore: Bumps version numbers to 1.9.2 [@stumpylog](https://github.com/stumpylog) ([#1666](https://github.com/paperless-ngx/paperless-ngx/pull/1666))
+
+## paperless-ngx 1.9.1
+
+### Notes
+
+- Version 1.9.1 incorrectly displays the version string as 1.9.0
+
+### Bug Fixes
+
+- Bugfix: Fixes missing OCR mode skip_noarchive [@stumpylog](https://github.com/stumpylog) ([#1645](https://github.com/paperless-ngx/paperless-ngx/pull/1645))
+- Fix reset button padding on small screens [@shamoon](https://github.com/shamoon) ([#1646](https://github.com/paperless-ngx/paperless-ngx/pull/1646))
+
+### Documentation
+
+- Improve docs re [@janis-ax](https://github.com/janis-ax) ([#1625](https://github.com/paperless-ngx/paperless-ngx/pull/1625))
+- [Documentation] Add v1.9.0 changelog [@github-actions](https://github.com/github-actions) ([#1639](https://github.com/paperless-ngx/paperless-ngx/pull/1639))
+
+### All App Changes
+
+- Bugfix: Fixes missing OCR mode skip_noarchive [@stumpylog](https://github.com/stumpylog) ([#1645](https://github.com/paperless-ngx/paperless-ngx/pull/1645))
+- Fix reset button padding on small screens [@shamoon](https://github.com/shamoon) ([#1646](https://github.com/paperless-ngx/paperless-ngx/pull/1646))
+
+## paperless-ngx 1.9.0
+
+### Features
+
+- Feature: Faster, less memory barcode handling [@stumpylog](https://github.com/stumpylog) ([#1594](https://github.com/paperless-ngx/paperless-ngx/pull/1594))
+- Feature: Display django-q process names [@stumpylog](https://github.com/stumpylog) ([#1567](https://github.com/paperless-ngx/paperless-ngx/pull/1567))
+- Feature: Add MariaDB support [@bckelly1](https://github.com/bckelly1) ([#543](https://github.com/paperless-ngx/paperless-ngx/pull/543))
+- Feature: Simplify IMAP login for UTF-8 [@stumpylog](https://github.com/stumpylog) ([#1492](https://github.com/paperless-ngx/paperless-ngx/pull/1492))
+- Feature: Even better re-do of OCR [@stumpylog](https://github.com/stumpylog) ([#1451](https://github.com/paperless-ngx/paperless-ngx/pull/1451))
+- Feature: document comments [@tim-vogel](https://github.com/tim-vogel) ([#1375](https://github.com/paperless-ngx/paperless-ngx/pull/1375))
+- Adding date suggestions to the documents details view [@Eckii24](https://github.com/Eckii24) ([#1367](https://github.com/paperless-ngx/paperless-ngx/pull/1367))
+- Feature: Event driven consumer [@stumpylog](https://github.com/stumpylog) ([#1421](https://github.com/paperless-ngx/paperless-ngx/pull/1421))
+- Feature: Adds storage paths to re-tagger command [@stumpylog](https://github.com/stumpylog) ([#1446](https://github.com/paperless-ngx/paperless-ngx/pull/1446))
+- Feature: Preserve original filename in metadata [@GwynHannay](https://github.com/GwynHannay) ([#1440](https://github.com/paperless-ngx/paperless-ngx/pull/1440))
+- Handle tags for gmail email accounts [@sisao](https://github.com/sisao) ([#1433](https://github.com/paperless-ngx/paperless-ngx/pull/1433))
+- Update redis image [@tribut](https://github.com/tribut) ([#1436](https://github.com/paperless-ngx/paperless-ngx/pull/1436))
+- PAPERLESS_REDIS may be set via docker secrets [@DennisGaida](https://github.com/DennisGaida) ([#1405](https://github.com/paperless-ngx/paperless-ngx/pull/1405))
+
+### Bug Fixes
+
+- paperless_cmd.sh: use exec to run supervisord [@lemmi](https://github.com/lemmi) ([#1617](https://github.com/paperless-ngx/paperless-ngx/pull/1617))
+- Fix: Double barcode separation creates empty file [@stumpylog](https://github.com/stumpylog) ([#1596](https://github.com/paperless-ngx/paperless-ngx/pull/1596))
+- Fix: Resolve issue with slow classifier [@stumpylog](https://github.com/stumpylog) ([#1576](https://github.com/paperless-ngx/paperless-ngx/pull/1576))
+- Fix document comments not updating on document navigation [@shamoon](https://github.com/shamoon) ([#1566](https://github.com/paperless-ngx/paperless-ngx/pull/1566))
+- Fix: Include storage paths in document exporter [@shamoon](https://github.com/shamoon) ([#1557](https://github.com/paperless-ngx/paperless-ngx/pull/1557))
+- Chore: Cleanup and validate settings [@stumpylog](https://github.com/stumpylog) ([#1551](https://github.com/paperless-ngx/paperless-ngx/pull/1551))
+- Bugfix: Better gunicorn settings for workers [@stumpylog](https://github.com/stumpylog) ([#1500](https://github.com/paperless-ngx/paperless-ngx/pull/1500))
+- Fix actions button in tasks table [@shamoon](https://github.com/shamoon) ([#1488](https://github.com/paperless-ngx/paperless-ngx/pull/1488))
+- Fix: Add missing filter rule types to SavedViewFilterRule model \& fix migrations [@shamoon](https://github.com/shamoon) ([#1463](https://github.com/paperless-ngx/paperless-ngx/pull/1463))
+- Fix paperless.conf.example typo [@qcasey](https://github.com/qcasey) ([#1460](https://github.com/paperless-ngx/paperless-ngx/pull/1460))
+- Bugfix: Fixes the creation of an archive file, even if noarchive was specified [@stumpylog](https://github.com/stumpylog) ([#1442](https://github.com/paperless-ngx/paperless-ngx/pull/1442))
+- Fix: created_date should not be required [@shamoon](https://github.com/shamoon) ([#1412](https://github.com/paperless-ngx/paperless-ngx/pull/1412))
+- Fix: dev backend testing [@stumpylog](https://github.com/stumpylog) ([#1420](https://github.com/paperless-ngx/paperless-ngx/pull/1420))
+- Bugfix: Catch all exceptions during the task signals [@stumpylog](https://github.com/stumpylog) ([#1387](https://github.com/paperless-ngx/paperless-ngx/pull/1387))
+- Fix: saved view page parameter [@shamoon](https://github.com/shamoon) ([#1376](https://github.com/paperless-ngx/paperless-ngx/pull/1376))
+- Fix: Correct browser unsaved changes warning [@shamoon](https://github.com/shamoon) ([#1369](https://github.com/paperless-ngx/paperless-ngx/pull/1369))
+- Fix: correct date pasting with other formats [@shamoon](https://github.com/shamoon) ([#1370](https://github.com/paperless-ngx/paperless-ngx/pull/1370))
+- Bugfix: Allow webserver bind address to be configured [@stumpylog](https://github.com/stumpylog) ([#1358](https://github.com/paperless-ngx/paperless-ngx/pull/1358))
+- Bugfix: Chain exceptions during exception handling [@stumpylog](https://github.com/stumpylog) ([#1354](https://github.com/paperless-ngx/paperless-ngx/pull/1354))
+- Fix: missing tooltip translation \& filter editor wrapping [@shamoon](https://github.com/shamoon) ([#1305](https://github.com/paperless-ngx/paperless-ngx/pull/1305))
+- Bugfix: Interaction between barcode and directories as tags [@stumpylog](https://github.com/stumpylog) ([#1303](https://github.com/paperless-ngx/paperless-ngx/pull/1303))
+
+### Documentation
+
+- [Beta] Paperless-ngx v1.9.0 Release Candidate [@stumpylog](https://github.com/stumpylog) ([#1560](https://github.com/paperless-ngx/paperless-ngx/pull/1560))
+- docs/configuration: Fix binary variable defaults [@erikarvstedt](https://github.com/erikarvstedt) ([#1528](https://github.com/paperless-ngx/paperless-ngx/pull/1528))
+- Info about installing on subpath [@viktor-c](https://github.com/viktor-c) ([#1350](https://github.com/paperless-ngx/paperless-ngx/pull/1350))
+- Docs: move scanner \& software recs to GH wiki [@shamoon](https://github.com/shamoon) ([#1482](https://github.com/paperless-ngx/paperless-ngx/pull/1482))
+- Docs: Update mobile scanner section [@tooomm](https://github.com/tooomm) ([#1467](https://github.com/paperless-ngx/paperless-ngx/pull/1467))
+- Adding date suggestions to the documents details view [@Eckii24](https://github.com/Eckii24) ([#1367](https://github.com/paperless-ngx/paperless-ngx/pull/1367))
+- docs: scanners: add Brother ads4700w [@ocelotsloth](https://github.com/ocelotsloth) ([#1450](https://github.com/paperless-ngx/paperless-ngx/pull/1450))
+- Feature: Adds storage paths to re-tagger command [@stumpylog](https://github.com/stumpylog) ([#1446](https://github.com/paperless-ngx/paperless-ngx/pull/1446))
+- Changes to Redis documentation [@Zerteax](https://github.com/Zerteax) ([#1441](https://github.com/paperless-ngx/paperless-ngx/pull/1441))
+- Update scanners.rst [@glassbox-sco](https://github.com/glassbox-sco) ([#1430](https://github.com/paperless-ngx/paperless-ngx/pull/1430))
+- Update scanners.rst [@derlucas](https://github.com/derlucas) ([#1415](https://github.com/paperless-ngx/paperless-ngx/pull/1415))
+- Bugfix: Allow webserver bind address to be configured [@stumpylog](https://github.com/stumpylog) ([#1358](https://github.com/paperless-ngx/paperless-ngx/pull/1358))
+- docs: fix small typo [@tooomm](https://github.com/tooomm) ([#1352](https://github.com/paperless-ngx/paperless-ngx/pull/1352))
+- [Documentation] Add v1.8.0 changelog [@github-actions](https://github.com/github-actions) ([#1298](https://github.com/paperless-ngx/paperless-ngx/pull/1298))
+
+### Maintenance
+
+- [Beta] Paperless-ngx v1.9.0 Release Candidate [@stumpylog](https://github.com/stumpylog) ([#1560](https://github.com/paperless-ngx/paperless-ngx/pull/1560))
+- paperless_cmd.sh: use exec to run supervisord [@lemmi](https://github.com/lemmi) ([#1617](https://github.com/paperless-ngx/paperless-ngx/pull/1617))
+- Chore: Extended container image cleanup [@stumpylog](https://github.com/stumpylog) ([#1556](https://github.com/paperless-ngx/paperless-ngx/pull/1556))
+- Chore: Smaller library images [@stumpylog](https://github.com/stumpylog) ([#1546](https://github.com/paperless-ngx/paperless-ngx/pull/1546))
+- Bump tj-actions/changed-files from 24 to 29.0.2 [@dependabot](https://github.com/dependabot) ([#1493](https://github.com/paperless-ngx/paperless-ngx/pull/1493))
+- Bugfix: Better gunicorn settings for workers [@stumpylog](https://github.com/stumpylog) ([#1500](https://github.com/paperless-ngx/paperless-ngx/pull/1500))
+- [CI] Fix release drafter issues [@qcasey](https://github.com/qcasey) ([#1301](https://github.com/paperless-ngx/paperless-ngx/pull/1301))
+- Fix: dev backend testing [@stumpylog](https://github.com/stumpylog) ([#1420](https://github.com/paperless-ngx/paperless-ngx/pull/1420))
+- Chore: Exclude dependabot PRs from Project, set status to Needs Review [@qcasey](https://github.com/qcasey) ([#1397](https://github.com/paperless-ngx/paperless-ngx/pull/1397))
+- Chore: Add to label PRs based on and title [@qcasey](https://github.com/qcasey) ([#1396](https://github.com/paperless-ngx/paperless-ngx/pull/1396))
+- Chore: use pre-commit in the Ci workflow [@stumpylog](https://github.com/stumpylog) ([#1362](https://github.com/paperless-ngx/paperless-ngx/pull/1362))
+- Chore: Fixes permissions for image tag cleanup [@stumpylog](https://github.com/stumpylog) ([#1315](https://github.com/paperless-ngx/paperless-ngx/pull/1315))
+- Bump leonsteinhaeuser/project-beta-automations from 1.2.1 to 1.3.0 [@dependabot](https://github.com/dependabot) ([#1328](https://github.com/paperless-ngx/paperless-ngx/pull/1328))
+- Bump tj-actions/changed-files from 23.1 to 24 [@dependabot](https://github.com/dependabot) ([#1329](https://github.com/paperless-ngx/paperless-ngx/pull/1329))
+- Feature: Remove requirements.txt and use pipenv everywhere [@stumpylog](https://github.com/stumpylog) ([#1316](https://github.com/paperless-ngx/paperless-ngx/pull/1316))
+
+### Dependencies
+
+<details>
+<summary>34 changes</summary>
+
+- Bump pikepdf from 5.5.0 to 5.6.1 [@dependabot](https://github.com/dependabot) ([#1537](https://github.com/paperless-ngx/paperless-ngx/pull/1537))
+- Bump black from 22.6.0 to 22.8.0 [@dependabot](https://github.com/dependabot) ([#1539](https://github.com/paperless-ngx/paperless-ngx/pull/1539))
+- Bump tqdm from 4.64.0 to 4.64.1 [@dependabot](https://github.com/dependabot) ([#1540](https://github.com/paperless-ngx/paperless-ngx/pull/1540))
+- Bump pytest from 7.1.2 to 7.1.3 [@dependabot](https://github.com/dependabot) ([#1538](https://github.com/paperless-ngx/paperless-ngx/pull/1538))
+- Bump tj-actions/changed-files from 24 to 29.0.2 [@dependabot](https://github.com/dependabot) ([#1493](https://github.com/paperless-ngx/paperless-ngx/pull/1493))
+- Bump angular packages, jest-preset-angular in src-ui [@dependabot](https://github.com/dependabot) ([#1502](https://github.com/paperless-ngx/paperless-ngx/pull/1502))
+- Bump jest-environment-jsdom from 28.1.3 to 29.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1507](https://github.com/paperless-ngx/paperless-ngx/pull/1507))
+- Bump [@<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot) ([#1506](https://github.com/paperless-ngx/paperless-ngx/pull/1506))
+- Bump [@<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot](https://github.com/<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot) ([#1505](https://github.com/paperless-ngx/paperless-ngx/pull/1505))
+- Bump zone.js from 0.11.7 to 0.11.8 in /src-ui [@dependabot](https://github.com/dependabot) ([#1504](https://github.com/paperless-ngx/paperless-ngx/pull/1504))
+- Bump ngx-color from 8.0.1 to 8.0.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1494](https://github.com/paperless-ngx/paperless-ngx/pull/1494))
+- Bump cypress from 10.3.1 to 10.7.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1496](https://github.com/paperless-ngx/paperless-ngx/pull/1496))
+- Bump [@<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot](https://github.com/<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot) ([#1495](https://github.com/paperless-ngx/paperless-ngx/pull/1495))
+- Bump [@<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot](https://github.com/<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot) ([#1498](https://github.com/paperless-ngx/paperless-ngx/pull/1498))
+- Bump sphinx from 5.0.2 to 5.1.1 [@dependabot](https://github.com/dependabot) ([#1297](https://github.com/paperless-ngx/paperless-ngx/pull/1297))
+- Chore: Bump Python dependencies [@stumpylog](https://github.com/stumpylog) ([#1445](https://github.com/paperless-ngx/paperless-ngx/pull/1445))
+- Chore: Update Python deps [@stumpylog](https://github.com/stumpylog) ([#1391](https://github.com/paperless-ngx/paperless-ngx/pull/1391))
+- Bump watchfiles from 0.15.0 to 0.16.1 [@dependabot](https://github.com/dependabot) ([#1285](https://github.com/paperless-ngx/paperless-ngx/pull/1285))
+- Bump leonsteinhaeuser/project-beta-automations from 1.2.1 to 1.3.0 [@dependabot](https://github.com/dependabot) ([#1328](https://github.com/paperless-ngx/paperless-ngx/pull/1328))
+- Bump tj-actions/changed-files from 23.1 to 24 [@dependabot](https://github.com/dependabot) ([#1329](https://github.com/paperless-ngx/paperless-ngx/pull/1329))
+- Bump cypress from 10.3.0 to 10.3.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1342](https://github.com/paperless-ngx/paperless-ngx/pull/1342))
+- Bump ngx-color from 7.3.3 to 8.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1343](https://github.com/paperless-ngx/paperless-ngx/pull/1343))
+- Bump [@<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot](https://github.com/<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot) ([#1330](https://github.com/paperless-ngx/paperless-ngx/pull/1330))
+- Bump [@<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot) ([#1341](https://github.com/paperless-ngx/paperless-ngx/pull/1341))
+- Bump jest-preset-angular from 12.1.0 to 12.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1340](https://github.com/paperless-ngx/paperless-ngx/pull/1340))
+- Bump concurrently from 7.2.2 to 7.3.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1326](https://github.com/paperless-ngx/paperless-ngx/pull/1326))
+- Bump ng2-pdf-viewer from 9.0.0 to 9.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1337](https://github.com/paperless-ngx/paperless-ngx/pull/1337))
+- Bump jest-environment-jsdom from 28.1.2 to 28.1.3 in /src-ui [@dependabot](https://github.com/dependabot) ([#1336](https://github.com/paperless-ngx/paperless-ngx/pull/1336))
+- Bump ngx-file-drop from 13.0.0 to 14.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1331](https://github.com/paperless-ngx/paperless-ngx/pull/1331))
+- Bump jest and [@<!---->types/jest in /src-ui @dependabot](https://github.com/<!---->types/jest in /src-ui @dependabot) ([#1333](https://github.com/paperless-ngx/paperless-ngx/pull/1333))
+- Bump bootstrap from 5.1.3 to 5.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1327](https://github.com/paperless-ngx/paperless-ngx/pull/1327))
+- Bump typescript from 4.6.4 to 4.7.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1324](https://github.com/paperless-ngx/paperless-ngx/pull/1324))
+- Bump ts-node from 10.8.1 to 10.9.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1325](https://github.com/paperless-ngx/paperless-ngx/pull/1325))
+- Bump rxjs from 7.5.5 to 7.5.6 in /src-ui [@dependabot](https://github.com/dependabot) ([#1323](https://github.com/paperless-ngx/paperless-ngx/pull/1323))
+</details>
+
+### All App Changes
+
+- [Beta] Paperless-ngx v1.9.0 Release Candidate [@stumpylog](https://github.com/stumpylog) ([#1560](https://github.com/paperless-ngx/paperless-ngx/pull/1560))
+- Feature: Faster, less memory barcode handling [@stumpylog](https://github.com/stumpylog) ([#1594](https://github.com/paperless-ngx/paperless-ngx/pull/1594))
+- Fix: Consume directory permissions were not updated [@stumpylog](https://github.com/stumpylog) ([#1605](https://github.com/paperless-ngx/paperless-ngx/pull/1605))
+- Fix: Double barcode separation creates empty file [@stumpylog](https://github.com/stumpylog) ([#1596](https://github.com/paperless-ngx/paperless-ngx/pull/1596))
+- Fix: Parsing Tika documents fails with AttributeError [@stumpylog](https://github.com/stumpylog) ([#1591](https://github.com/paperless-ngx/paperless-ngx/pull/1591))
+- Fix: Resolve issue with slow classifier [@stumpylog](https://github.com/stumpylog) ([#1576](https://github.com/paperless-ngx/paperless-ngx/pull/1576))
+- Feature: Display django-q process names [@stumpylog](https://github.com/stumpylog) ([#1567](https://github.com/paperless-ngx/paperless-ngx/pull/1567))
+- Fix document comments not updating on document navigation [@shamoon](https://github.com/shamoon) ([#1566](https://github.com/paperless-ngx/paperless-ngx/pull/1566))
+- Feature: Add MariaDB support [@bckelly1](https://github.com/bckelly1) ([#543](https://github.com/paperless-ngx/paperless-ngx/pull/543))
+- Fix: Include storage paths in document exporter [@shamoon](https://github.com/shamoon) ([#1557](https://github.com/paperless-ngx/paperless-ngx/pull/1557))
+- Chore: Cleanup and validate settings [@stumpylog](https://github.com/stumpylog) ([#1551](https://github.com/paperless-ngx/paperless-ngx/pull/1551))
+- Bump pikepdf from 5.5.0 to 5.6.1 [@dependabot](https://github.com/dependabot) ([#1537](https://github.com/paperless-ngx/paperless-ngx/pull/1537))
+- Bump black from 22.6.0 to 22.8.0 [@dependabot](https://github.com/dependabot) ([#1539](https://github.com/paperless-ngx/paperless-ngx/pull/1539))
+- Bump tqdm from 4.64.0 to 4.64.1 [@dependabot](https://github.com/dependabot) ([#1540](https://github.com/paperless-ngx/paperless-ngx/pull/1540))
+- Bump pytest from 7.1.2 to 7.1.3 [@dependabot](https://github.com/dependabot) ([#1538](https://github.com/paperless-ngx/paperless-ngx/pull/1538))
+- Bump angular packages, jest-preset-angular in src-ui [@dependabot](https://github.com/dependabot) ([#1502](https://github.com/paperless-ngx/paperless-ngx/pull/1502))
+- Bump jest-environment-jsdom from 28.1.3 to 29.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1507](https://github.com/paperless-ngx/paperless-ngx/pull/1507))
+- Bump [@<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot) ([#1506](https://github.com/paperless-ngx/paperless-ngx/pull/1506))
+- Bump [@<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot](https://github.com/<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot) ([#1505](https://github.com/paperless-ngx/paperless-ngx/pull/1505))
+- Bump zone.js from 0.11.7 to 0.11.8 in /src-ui [@dependabot](https://github.com/dependabot) ([#1504](https://github.com/paperless-ngx/paperless-ngx/pull/1504))
+- Bump ngx-color from 8.0.1 to 8.0.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1494](https://github.com/paperless-ngx/paperless-ngx/pull/1494))
+- Bump cypress from 10.3.1 to 10.7.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1496](https://github.com/paperless-ngx/paperless-ngx/pull/1496))
+- Bump [@<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot](https://github.com/<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot) ([#1495](https://github.com/paperless-ngx/paperless-ngx/pull/1495))
+- Bump [@<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot](https://github.com/<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot) ([#1498](https://github.com/paperless-ngx/paperless-ngx/pull/1498))
+- Feature: Simplify IMAP login for UTF-8 [@stumpylog](https://github.com/stumpylog) ([#1492](https://github.com/paperless-ngx/paperless-ngx/pull/1492))
+- Fix actions button in tasks table [@shamoon](https://github.com/shamoon) ([#1488](https://github.com/paperless-ngx/paperless-ngx/pull/1488))
+- Fix: Add missing filter rule types to SavedViewFilterRule model \& fix migrations [@shamoon](https://github.com/shamoon) ([#1463](https://github.com/paperless-ngx/paperless-ngx/pull/1463))
+- Feature: Even better re-do of OCR [@stumpylog](https://github.com/stumpylog) ([#1451](https://github.com/paperless-ngx/paperless-ngx/pull/1451))
+- Feature: document comments [@tim-vogel](https://github.com/tim-vogel) ([#1375](https://github.com/paperless-ngx/paperless-ngx/pull/1375))
+- Adding date suggestions to the documents details view [@Eckii24](https://github.com/Eckii24) ([#1367](https://github.com/paperless-ngx/paperless-ngx/pull/1367))
+- Bump sphinx from 5.0.2 to 5.1.1 [@dependabot](https://github.com/dependabot) ([#1297](https://github.com/paperless-ngx/paperless-ngx/pull/1297))
+- Feature: Event driven consumer [@stumpylog](https://github.com/stumpylog) ([#1421](https://github.com/paperless-ngx/paperless-ngx/pull/1421))
+- Bugfix: Fixes the creation of an archive file, even if noarchive was specified [@stumpylog](https://github.com/stumpylog) ([#1442](https://github.com/paperless-ngx/paperless-ngx/pull/1442))
+- Feature: Adds storage paths to re-tagger command [@stumpylog](https://github.com/stumpylog) ([#1446](https://github.com/paperless-ngx/paperless-ngx/pull/1446))
+- Feature: Preserve original filename in metadata [@GwynHannay](https://github.com/GwynHannay) ([#1440](https://github.com/paperless-ngx/paperless-ngx/pull/1440))
+- Handle tags for gmail email accounts [@sisao](https://github.com/sisao) ([#1433](https://github.com/paperless-ngx/paperless-ngx/pull/1433))
+- Fix: should not be required [@shamoon](https://github.com/shamoon) ([#1412](https://github.com/paperless-ngx/paperless-ngx/pull/1412))
+- Bugfix: Catch all exceptions during the task signals [@stumpylog](https://github.com/stumpylog) ([#1387](https://github.com/paperless-ngx/paperless-ngx/pull/1387))
+- Fix: saved view page parameter [@shamoon](https://github.com/shamoon) ([#1376](https://github.com/paperless-ngx/paperless-ngx/pull/1376))
+- Fix: Correct browser unsaved changes warning [@shamoon](https://github.com/shamoon) ([#1369](https://github.com/paperless-ngx/paperless-ngx/pull/1369))
+- Fix: correct date pasting with other formats [@shamoon](https://github.com/shamoon) ([#1370](https://github.com/paperless-ngx/paperless-ngx/pull/1370))
+- Chore: use pre-commit in the Ci workflow [@stumpylog](https://github.com/stumpylog) ([#1362](https://github.com/paperless-ngx/paperless-ngx/pull/1362))
+- Bugfix: Chain exceptions during exception handling [@stumpylog](https://github.com/stumpylog) ([#1354](https://github.com/paperless-ngx/paperless-ngx/pull/1354))
+- Bump watchfiles from 0.15.0 to 0.16.1 [@dependabot](https://github.com/dependabot) ([#1285](https://github.com/paperless-ngx/paperless-ngx/pull/1285))
+- Bump cypress from 10.3.0 to 10.3.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1342](https://github.com/paperless-ngx/paperless-ngx/pull/1342))
+- Bump ngx-color from 7.3.3 to 8.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1343](https://github.com/paperless-ngx/paperless-ngx/pull/1343))
+- Bump [@<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot](https://github.com/<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot) ([#1330](https://github.com/paperless-ngx/paperless-ngx/pull/1330))
+- Bump [@<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot) ([#1341](https://github.com/paperless-ngx/paperless-ngx/pull/1341))
+- Bump jest-preset-angular from 12.1.0 to 12.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1340](https://github.com/paperless-ngx/paperless-ngx/pull/1340))
+- Bump concurrently from 7.2.2 to 7.3.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1326](https://github.com/paperless-ngx/paperless-ngx/pull/1326))
+- Bump ng2-pdf-viewer from 9.0.0 to 9.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1337](https://github.com/paperless-ngx/paperless-ngx/pull/1337))
+- Bump jest-environment-jsdom from 28.1.2 to 28.1.3 in /src-ui [@dependabot](https://github.com/dependabot) ([#1336](https://github.com/paperless-ngx/paperless-ngx/pull/1336))
+- Bump ngx-file-drop from 13.0.0 to 14.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1331](https://github.com/paperless-ngx/paperless-ngx/pull/1331))
+- Bump jest and [@<!---->types/jest in /src-ui @dependabot](https://github.com/<!---->types/jest in /src-ui @dependabot) ([#1333](https://github.com/paperless-ngx/paperless-ngx/pull/1333))
+- Bump bootstrap from 5.1.3 to 5.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1327](https://github.com/paperless-ngx/paperless-ngx/pull/1327))
+- Bump typescript from 4.6.4 to 4.7.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1324](https://github.com/paperless-ngx/paperless-ngx/pull/1324))
+- Bump ts-node from 10.8.1 to 10.9.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1325](https://github.com/paperless-ngx/paperless-ngx/pull/1325))
+- Bump rxjs from 7.5.5 to 7.5.6 in /src-ui [@dependabot](https://github.com/dependabot) ([#1323](https://github.com/paperless-ngx/paperless-ngx/pull/1323))
+- Fix: missing tooltip translation \& filter editor wrapping [@shamoon](https://github.com/shamoon) ([#1305](https://github.com/paperless-ngx/paperless-ngx/pull/1305))
+- Feature: Remove requirements.txt and use pipenv everywhere [@stumpylog](https://github.com/stumpylog) ([#1316](https://github.com/paperless-ngx/paperless-ngx/pull/1316))
+- Bugfix: Interaction between barcode and directories as tags [@stumpylog](https://github.com/stumpylog) ([#1303](https://github.com/paperless-ngx/paperless-ngx/pull/1303))
+
+## paperless-ngx 1.8.0
+
+### Features
+
+- Feature use env vars in pre post scripts [@ziprandom](https://github.com/ziprandom) ([#1154](https://github.com/paperless-ngx/paperless-ngx/pull/1154))
+- frontend task queue [@shamoon](https://github.com/shamoon) ([#1020](https://github.com/paperless-ngx/paperless-ngx/pull/1020))
+- Fearless scikit-learn updates [@stumpylog](https://github.com/stumpylog) ([#1082](https://github.com/paperless-ngx/paperless-ngx/pull/1082))
+- Adds support for Docker secrets [@stumpylog](https://github.com/stumpylog) ([#1034](https://github.com/paperless-ngx/paperless-ngx/pull/1034))
+- make frontend timezone un-aware [@shamoon](https://github.com/shamoon) ([#957](https://github.com/paperless-ngx/paperless-ngx/pull/957))
+- Change document thumbnails to WebP [@stumpylog](https://github.com/stumpylog) ([#1127](https://github.com/paperless-ngx/paperless-ngx/pull/1127))
+- Fork django-q to update dependencies [@stumpylog](https://github.com/stumpylog) ([#1014](https://github.com/paperless-ngx/paperless-ngx/pull/1014))
+- Fix: Rework query params logic [@shamoon](https://github.com/shamoon) ([#1000](https://github.com/paperless-ngx/paperless-ngx/pull/1000))
+- Enhancement: show note on language change and offer reload [@shamoon](https://github.com/shamoon) ([#1030](https://github.com/paperless-ngx/paperless-ngx/pull/1030))
+- Include error information when Redis connection fails [@stumpylog](https://github.com/stumpylog) ([#1016](https://github.com/paperless-ngx/paperless-ngx/pull/1016))
+- frontend settings saved to database [@shamoon](https://github.com/shamoon) ([#919](https://github.com/paperless-ngx/paperless-ngx/pull/919))
+- Add "Created" as additional (optional) parameter for post_documents [@eingemaischt](https://github.com/eingemaischt) ([#965](https://github.com/paperless-ngx/paperless-ngx/pull/965))
+- Convert Changelog to markdown, auto-commit future changelogs [@qcasey](https://github.com/qcasey) ([#935](https://github.com/paperless-ngx/paperless-ngx/pull/935))
+- allow all ASN filtering functions [@shamoon](https://github.com/shamoon) ([#920](https://github.com/paperless-ngx/paperless-ngx/pull/920))
+- gunicorn: Allow IPv6 sockets [@vlcty](https://github.com/vlcty) ([#924](https://github.com/paperless-ngx/paperless-ngx/pull/924))
+- initial app loading indicators [@shamoon](https://github.com/shamoon) ([#899](https://github.com/paperless-ngx/paperless-ngx/pull/899))
+
+### Bug Fixes
+
+- Fix: dropdown selected items not visible again [@shamoon](https://github.com/shamoon) ([#1261](https://github.com/paperless-ngx/paperless-ngx/pull/1261))
+- [CI] Fix automatic changelog generation on release [@qcasey](https://github.com/qcasey) ([#1249](https://github.com/paperless-ngx/paperless-ngx/pull/1249))
+- Fix: Prevent duplicate api calls on text filtering [@shamoon](https://github.com/shamoon) ([#1133](https://github.com/paperless-ngx/paperless-ngx/pull/1133))
+- make frontend timezone un-aware [@shamoon](https://github.com/shamoon) ([#957](https://github.com/paperless-ngx/paperless-ngx/pull/957))
+- Feature / fix quick toggleable filters [@shamoon](https://github.com/shamoon) ([#1122](https://github.com/paperless-ngx/paperless-ngx/pull/1122))
+- Chore: Manually downgrade reportlab (and update everything else) [@stumpylog](https://github.com/stumpylog) ([#1116](https://github.com/paperless-ngx/paperless-ngx/pull/1116))
+- Bugfix: Don't assume default Docker folders [@stumpylog](https://github.com/stumpylog) ([#1088](https://github.com/paperless-ngx/paperless-ngx/pull/1088))
+- Bugfix: Better sanity check messages [@stumpylog](https://github.com/stumpylog) ([#1049](https://github.com/paperless-ngx/paperless-ngx/pull/1049))
+- Fix vertical margins between pages of pdf viewer [@shamoon](https://github.com/shamoon) ([#1081](https://github.com/paperless-ngx/paperless-ngx/pull/1081))
+- Bugfix: Pass debug setting on to django-q [@stumpylog](https://github.com/stumpylog) ([#1058](https://github.com/paperless-ngx/paperless-ngx/pull/1058))
+- Bugfix: Don't assume the document has a title set [@stumpylog](https://github.com/stumpylog) ([#1057](https://github.com/paperless-ngx/paperless-ngx/pull/1057))
+- Bugfix: Corrects the setting of max pixel size for OCR [@stumpylog](https://github.com/stumpylog) ([#1008](https://github.com/paperless-ngx/paperless-ngx/pull/1008))
+- better date pasting [@shamoon](https://github.com/shamoon) ([#1007](https://github.com/paperless-ngx/paperless-ngx/pull/1007))
+- Enhancement: Alphabetize tags by default [@shamoon](https://github.com/shamoon) ([#1017](https://github.com/paperless-ngx/paperless-ngx/pull/1017))
+- Fix: Rework query params logic [@shamoon](https://github.com/shamoon) ([#1000](https://github.com/paperless-ngx/paperless-ngx/pull/1000))
+- Fix: add translation for some un-translated tooltips [@shamoon](https://github.com/shamoon) ([#995](https://github.com/paperless-ngx/paperless-ngx/pull/995))
+- Change npm --no-optional to --omit=optional [@shamoon](https://github.com/shamoon) ([#986](https://github.com/paperless-ngx/paperless-ngx/pull/986))
+- Add `myst-parser` to fix readthedocs [@qcasey](https://github.com/qcasey) ([#982](https://github.com/paperless-ngx/paperless-ngx/pull/982))
+- Fix: Title is changed after switching doc quickly [@shamoon](https://github.com/shamoon) ([#979](https://github.com/paperless-ngx/paperless-ngx/pull/979))
+- Fix: warn when closing a document with unsaved changes due to max open docs [@shamoon](https://github.com/shamoon) ([#956](https://github.com/paperless-ngx/paperless-ngx/pull/956))
+- Bugfix: Adds configurable intoify debounce time [@stumpylog](https://github.com/stumpylog) ([#953](https://github.com/paperless-ngx/paperless-ngx/pull/953))
+- Bugfix: Fixes document filename date off by 1 issue [@stumpylog](https://github.com/stumpylog) ([#942](https://github.com/paperless-ngx/paperless-ngx/pull/942))
+- fixes #<!---->949: change to MIME detection for files [@gador](https://github.com/gador) ([#962](https://github.com/paperless-ngx/paperless-ngx/pull/962))
+- docs: fix some typos [@Berjou](https://github.com/Berjou) ([#948](https://github.com/paperless-ngx/paperless-ngx/pull/948))
+- [Docs] Fix 2 small typos [@tooomm](https://github.com/tooomm) ([#946](https://github.com/paperless-ngx/paperless-ngx/pull/946))
+- [Readme] Fix typo [@tooomm](https://github.com/tooomm) ([#941](https://github.com/paperless-ngx/paperless-ngx/pull/941))
+- Fix: management pages plurals incorrect in other languages [@shamoon](https://github.com/shamoon) ([#939](https://github.com/paperless-ngx/paperless-ngx/pull/939))
+- Fix: v1.7.1 frontend visual fixes [@shamoon](https://github.com/shamoon) ([#933](https://github.com/paperless-ngx/paperless-ngx/pull/933))
+- Fix: unassigned query params ignored [@shamoon](https://github.com/shamoon) ([#930](https://github.com/paperless-ngx/paperless-ngx/pull/930))
+- Fix: allow commas in non-multi rules query params [@shamoon](https://github.com/shamoon) ([#923](https://github.com/paperless-ngx/paperless-ngx/pull/923))
+- Fix: Include version in export for better error messages [@stumpylog](https://github.com/stumpylog) ([#883](https://github.com/paperless-ngx/paperless-ngx/pull/883))
+- Bugfix: Superuser Management Won't Reset Password [@stumpylog](https://github.com/stumpylog) ([#903](https://github.com/paperless-ngx/paperless-ngx/pull/903))
+- Fix Ignore Date Parsing [@stumpylog](https://github.com/stumpylog) ([#721](https://github.com/paperless-ngx/paperless-ngx/pull/721))
+
+### Documentation
+
+- Feature use env vars in pre post scripts [@ziprandom](https://github.com/ziprandom) ([#1154](https://github.com/paperless-ngx/paperless-ngx/pull/1154))
+- Add `myst-parser` to fix readthedocs [@qcasey](https://github.com/qcasey) ([#982](https://github.com/paperless-ngx/paperless-ngx/pull/982))
+- Add "Created" as additional (optional) parameter for post_documents [@eingemaischt](https://github.com/eingemaischt) ([#965](https://github.com/paperless-ngx/paperless-ngx/pull/965))
+- Bugfix: Adds configurable intoify debounce time [@stumpylog](https://github.com/stumpylog) ([#953](https://github.com/paperless-ngx/paperless-ngx/pull/953))
+- docs: fix some typos [@Berjou](https://github.com/Berjou) ([#948](https://github.com/paperless-ngx/paperless-ngx/pull/948))
+- [Docs] Fix 2 small typos [@tooomm](https://github.com/tooomm) ([#946](https://github.com/paperless-ngx/paperless-ngx/pull/946))
+- Convert Changelog to markdown, auto-commit future changelogs [@qcasey](https://github.com/qcasey) ([#935](https://github.com/paperless-ngx/paperless-ngx/pull/935))
+- [Readme] Fix typo [@tooomm](https://github.com/tooomm) ([#941](https://github.com/paperless-ngx/paperless-ngx/pull/941))
+
+### Maintenance
+
+- Adds support for Docker secrets [@stumpylog](https://github.com/stumpylog) ([#1034](https://github.com/paperless-ngx/paperless-ngx/pull/1034))
+- Bugfix: Don't assume default Docker folders [@stumpylog](https://github.com/stumpylog) ([#1088](https://github.com/paperless-ngx/paperless-ngx/pull/1088))
+- Include error information when Redis connection fails [@stumpylog](https://github.com/stumpylog) ([#1016](https://github.com/paperless-ngx/paperless-ngx/pull/1016))
+- Fix: add translation for some un-translated tooltips [@shamoon](https://github.com/shamoon) ([#995](https://github.com/paperless-ngx/paperless-ngx/pull/995))
+- gunicorn: Allow IPv6 sockets [@vlcty](https://github.com/vlcty) ([#924](https://github.com/paperless-ngx/paperless-ngx/pull/924))
+
+### Dependencies
+
+<details>
+<summary>34 changes</summary>
+
+- Fearless scikit-learn updates [@stumpylog](https://github.com/stumpylog) ([#1082](https://github.com/paperless-ngx/paperless-ngx/pull/1082))
+- Bump pillow from 9.1.1 to 9.2.0 [@dependabot](https://github.com/dependabot) ([#1193](https://github.com/paperless-ngx/paperless-ngx/pull/1193))
+- Bump watchdog from 2.1.8 to 2.1.9 [@dependabot](https://github.com/dependabot) ([#1132](https://github.com/paperless-ngx/paperless-ngx/pull/1132))
+- Bump scikit-learn from 1.0.2 to 1.1.1 [@dependabot](https://github.com/dependabot) ([#992](https://github.com/paperless-ngx/paperless-ngx/pull/992))
+- Bump setuptools from 62.3.3 to 62.6.0 [@dependabot](https://github.com/dependabot) ([#1150](https://github.com/paperless-ngx/paperless-ngx/pull/1150))
+- Bump django-filter from 21.1 to 22.1 [@dependabot](https://github.com/dependabot) ([#1191](https://github.com/paperless-ngx/paperless-ngx/pull/1191))
+- Bump actions/setup-python from 3 to 4 [@dependabot](https://github.com/dependabot) ([#1176](https://github.com/paperless-ngx/paperless-ngx/pull/1176))
+- Bump sphinx from 4.5.0 to 5.0.2 [@dependabot](https://github.com/dependabot) ([#1151](https://github.com/paperless-ngx/paperless-ngx/pull/1151))
+- Bump docker/metadata-action from 3 to 4 [@dependabot](https://github.com/dependabot) ([#1178](https://github.com/paperless-ngx/paperless-ngx/pull/1178))
+- Bump tj-actions/changed-files from 22.1 to 23.1 [@dependabot](https://github.com/dependabot) ([#1179](https://github.com/paperless-ngx/paperless-ngx/pull/1179))
+- Bump @<!---->angular/cli from 13.3.7 to 14.0.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1177](https://github.com/paperless-ngx/paperless-ngx/pull/1177))
+- Bump cypress from 10.0.1 to 10.3.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1187](https://github.com/paperless-ngx/paperless-ngx/pull/1187))
+- Bump zone.js from 0.11.5 to 0.11.6 in /src-ui [@dependabot](https://github.com/dependabot) ([#1185](https://github.com/paperless-ngx/paperless-ngx/pull/1185))
+- Bump ts-node from 10.8.0 to 10.8.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1184](https://github.com/paperless-ngx/paperless-ngx/pull/1184))
+- Bump jest-environment-jsdom from 28.1.0 to 28.1.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1175](https://github.com/paperless-ngx/paperless-ngx/pull/1175))
+- Bump @<!---->types/node from 17.0.38 to 18.0.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1183](https://github.com/paperless-ngx/paperless-ngx/pull/1183))
+- Bump concurrently from 7.2.1 to 7.2.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1181](https://github.com/paperless-ngx/paperless-ngx/pull/1181))
+- Bump jest-preset-angular from 12.0.1 to 12.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1182](https://github.com/paperless-ngx/paperless-ngx/pull/1182))
+- Bump jest and @<!---->types/jest in /src-ui [@dependabot](https://github.com/dependabot) ([#1180](https://github.com/paperless-ngx/paperless-ngx/pull/1180))
+- Bump whitenoise from 6.1.0 to 6.2.0 [@dependabot](https://github.com/dependabot) ([#1103](https://github.com/paperless-ngx/paperless-ngx/pull/1103))
+- Bump cypress from 9.6.1 to 10.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1083](https://github.com/paperless-ngx/paperless-ngx/pull/1083))
+- Bump docker/setup-qemu-action from 1 to 2 [@dependabot](https://github.com/dependabot) ([#1065](https://github.com/paperless-ngx/paperless-ngx/pull/1065))
+- Bump docker/setup-buildx-action from 1 to 2 [@dependabot](https://github.com/dependabot) ([#1064](https://github.com/paperless-ngx/paperless-ngx/pull/1064))
+- Bump docker/build-push-action from 2 to 3 [@dependabot](https://github.com/dependabot) ([#1063](https://github.com/paperless-ngx/paperless-ngx/pull/1063))
+- Bump @<!---->cypress/schematic from 1.7.0 to 2.0.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1075](https://github.com/paperless-ngx/paperless-ngx/pull/1075))
+- Bump tj-actions/changed-files from 19 to 22.1 [@dependabot](https://github.com/dependabot) ([#1062](https://github.com/paperless-ngx/paperless-ngx/pull/1062))
+- Bump concurrently from 7.1.0 to 7.2.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1073](https://github.com/paperless-ngx/paperless-ngx/pull/1073))
+- Bump @<!---->types/jest from 27.4.1 to 27.5.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1074](https://github.com/paperless-ngx/paperless-ngx/pull/1074))
+- Bump ts-node from 10.7.0 to 10.8.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1070](https://github.com/paperless-ngx/paperless-ngx/pull/1070))
+- Bump jest from 28.0.3 to 28.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1071](https://github.com/paperless-ngx/paperless-ngx/pull/1071))
+- Chore: npm package updates 22-06-01 [@shamoon](https://github.com/shamoon) ([#1069](https://github.com/paperless-ngx/paperless-ngx/pull/1069))
+- Bump docker/login-action from 1 to 2 [@dependabot](https://github.com/dependabot) ([#1061](https://github.com/paperless-ngx/paperless-ngx/pull/1061))
+- Chore: Manually update dependencies [@stumpylog](https://github.com/stumpylog) ([#1013](https://github.com/paperless-ngx/paperless-ngx/pull/1013))
+- Chore: Manually update all Python dependencies [@stumpylog](https://github.com/stumpylog) ([#973](https://github.com/paperless-ngx/paperless-ngx/pull/973))
+</details>
+
 ## paperless-ngx 1.7.1
 
 ### Features
 
-- (chore) Runs pyupgrade to Python 3.8+ @stumpylog (#890)
-- Dockerfile Organization \& Enhancements @stumpylog (#888)
-- mobile friendlier manage pages @shamoon (#873)
-- Use semver for release process @stumpylog (#851)
-- Enable Docker Hub push @stumpylog (#828)
-- Feature barcode tiff support @gador (#766)
-- Updates GHA workflow to rebuild intermediate images on changes @stumpylog (#820)
-- Adds simple Python to wait for Redis broker to be ready @stumpylog (#788)
-- Update GHA workflow to build all Docker images @stumpylog (#761)
+- (chore) Runs pyupgrade to Python 3.8+ [\@stumpylog](https://github.com/stumpylog) ([\#890](https://github.com/paperless-ngx/paperless-ngx/pull/890))
+- Dockerfile Organization \& Enhancements [\@stumpylog](https://github.com/stumpylog) ([\#888](https://github.com/paperless-ngx/paperless-ngx/pull/888))
+- mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873))
+- Use semver for release process [\@stumpylog](https://github.com/stumpylog) ([\#851](https://github.com/paperless-ngx/paperless-ngx/pull/851))
+- Enable Docker Hub push [\@stumpylog](https://github.com/stumpylog) ([\#828](https://github.com/paperless-ngx/paperless-ngx/pull/828))
+- Feature barcode tiff support [\@gador](https://github.com/gador) ([\#766](https://github.com/paperless-ngx/paperless-ngx/pull/766))
+- Updates GHA workflow to rebuild intermediate images on changes [\@stumpylog](https://github.com/stumpylog) ([\#820](https://github.com/paperless-ngx/paperless-ngx/pull/820))
+- Adds simple Python to wait for Redis broker to be ready [\@stumpylog](https://github.com/stumpylog) ([\#788](https://github.com/paperless-ngx/paperless-ngx/pull/788))
+- Update GHA workflow to build all Docker images [\@stumpylog](https://github.com/stumpylog) ([\#761](https://github.com/paperless-ngx/paperless-ngx/pull/761))
 
 ### Bug Fixes
 
-- Feature / fix saved view \& sort field query params @shamoon (#881)
-- mobile friendlier manage pages @shamoon (#873)
-- Add timeout to healthcheck @shamoon (#880)
-- Always accept yyyy-mm-dd date inputs @shamoon (#864)
-- Fix local Docker image building @stumpylog (#849)
-- Fix: show errors on invalid date input @shamoon (#862)
-- Fix: Older dates do not display on frontend @shamoon (#852)
-- Fixes IMAP UTF8 Authenication @stumpylog (#725)
-- Fix password field remains visible @shamoon (#840)
-- Fixes Pillow build for armv7 @stumpylog (#815)
-- Update frontend localization source file @shamoon (#814)
-- Fix install script extra OCR languages format @stumpylog (#777)
+- Feature / fix saved view \& sort field query params [\@shamoon](https://github.com/shamoon) ([\#881](https://github.com/paperless-ngx/paperless-ngx/pull/881))
+- Mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873))
+- Add timeout to healthcheck [\@shamoon](https://github.com/shamoon) ([\#880](https://github.com/paperless-ngx/paperless-ngx/pull/880))
+- Always accept yyyy-mm-dd date inputs [\@shamoon](https://github.com/shamoon) ([\#864](https://github.com/paperless-ngx/paperless-ngx/pull/864))
+- Fix local Docker image building [\@stumpylog](https://github.com/stumpylog) ([\#849](https://github.com/paperless-ngx/paperless-ngx/pull/849))
+- Fix: show errors on invalid date input [\@shamoon](https://github.com/shamoon) ([\#862](https://github.com/paperless-ngx/paperless-ngx/pull/862))
+- Fix: Older dates do not display on frontend [\@shamoon](https://github.com/shamoon) ([\#852](https://github.com/paperless-ngx/paperless-ngx/pull/852))
+- Fixes IMAP UTF8 Authenication [\@stumpylog](https://github.com/stumpylog) ([\#725](https://github.com/paperless-ngx/paperless-ngx/pull/725))
+- Fix password field remains visible [\@shamoon](https://github.com/shamoon) ([\#840](https://github.com/paperless-ngx/paperless-ngx/pull/840))
+- Fixes Pillow build for armv7 [\@stumpylog](https://github.com/stumpylog) ([\#815](https://github.com/paperless-ngx/paperless-ngx/pull/815))
+- Update frontend localization source file [\@shamoon](https://github.com/shamoon) ([\#814](https://github.com/paperless-ngx/paperless-ngx/pull/814))
+- Fix install script extra OCR languages format [\@stumpylog](https://github.com/stumpylog) ([\#777](https://github.com/paperless-ngx/paperless-ngx/pull/777))
 
 ### Documentation
 
-- Use semver for release process @stumpylog (#851)
-- Deployment: Consolidate tika compose files @qcasey (#866)
-- Fix local Docker image building @stumpylog (#849)
+- Use semver for release process [\@stumpylog](https://github.com/stumpylog) ([\#851](https://github.com/paperless-ngx/paperless-ngx/pull/851))
+- Deployment: Consolidate tika compose files [\@qcasey](https://github.com/qcasey) ([\#866](https://github.com/paperless-ngx/paperless-ngx/pull/866))
+- Fix local Docker image building [\@stumpylog](https://github.com/stumpylog) ([\#849](https://github.com/paperless-ngx/paperless-ngx/pull/849))
 
 ### Maintenance
 
-- Dockerfile Organization \& Enhancements @stumpylog (#888)
-- Add timeout to healthcheck @shamoon (#880)
-- Use semver for release process @stumpylog (#851)
-- Deployment: Consolidate tika compose files @qcasey (#866)
-- Fixes Pillow build for armv7 @stumpylog (#815)
-- Update frontend localization source file @shamoon (#814)
-- Fix install script extra OCR languages format @stumpylog (#777)
-- Adds simple Python to wait for Redis broker to be ready @stumpylog (#788)
+- Dockerfile Organization \& Enhancements [\@stumpylog](https://github.com/stumpylog) ([\#888](https://github.com/paperless-ngx/paperless-ngx/pull/888))
+- Add timeout to healthcheck [\@shamoon](https://github.com/shamoon) ([\#880](https://github.com/paperless-ngx/paperless-ngx/pull/880))
+- Use semver for release process [\@stumpylog](https://github.com/stumpylog) ([\#851](https://github.com/paperless-ngx/paperless-ngx/pull/851))
+- Deployment: Consolidate tika compose files [\@qcasey](https://github.com/qcasey) ([\#866](https://github.com/paperless-ngx/paperless-ngx/pull/866))
+- Fixes Pillow build for armv7 [\@stumpylog](https://github.com/stumpylog) ([\#815](https://github.com/paperless-ngx/paperless-ngx/pull/815))
+- Update frontend localization source file [\@shamoon](https://github.com/shamoon) ([\#814](https://github.com/paperless-ngx/paperless-ngx/pull/814))
+- Fix install script extra OCR languages format [\@stumpylog](https://github.com/stumpylog) ([\#777](https://github.com/paperless-ngx/paperless-ngx/pull/777))
+- Adds simple Python to wait for Redis broker to be ready [\@stumpylog](https://github.com/stumpylog) ([\#788](https://github.com/paperless-ngx/paperless-ngx/pull/788))
 
 ### Dependencies
 
 <details>
 <summary>15 changes</summary>
 
-- Bump tj-actions/changed-files from 18.7 to 19 @dependabot (#830)
-- Bump asgiref from 3.5.0 to 3.5.1 @dependabot (#867)
-- Bump jest from 27.5.1 to 28.0.3 in /src-ui @dependabot (#860)
-- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.1.0 to 12.1.1 in /src-ui @dependabot (#861)
-- Bump @<!---->types/node from 17.0.27 to 17.0.29 in /src-ui @dependabot (#833)
-- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.0.2 to 12.1.0 in /src-ui @dependabot (#834)
-- Bump pytest from 7.1.1 to 7.1.2 @dependabot (#806)
-- Bump github/codeql-action from 1 to 2 @dependabot (#792)
-- Bump imap-tools from 0.53.0 to 0.54.0 @dependabot (#758)
-- Bump ocrmypdf from 13.4.2 to 13.4.3 @dependabot (#757)
-- Bump importlib-resources from 5.6.0 to 5.7.1 @dependabot (#756)
-- Bump tox from 3.24.5 to 3.25.0 @dependabot (#692)
-- Bump cypress from 9.5.3 to 9.6.0 in /src-ui @dependabot (#800)
-- Bump angular \& tools to 13.3.4 or 13.3.3 @shamoon (#799)
-- Bump concurrently from 7.0.0 to 7.1.0 in /src-ui @dependabot (#797)
+- Bump tj-actions/changed-files from 18.7 to 19 @dependabot ([\#830](https://github.com/paperless-ngx/paperless-ngx/pull/830))
+- Bump asgiref from 3.5.0 to 3.5.1 @dependabot ([\#867](https://github.com/paperless-ngx/paperless-ngx/pull/867))
+- Bump jest from 27.5.1 to 28.0.3 in /src-ui @dependabot ([\#860](https://github.com/paperless-ngx/paperless-ngx/pull/860))
+- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.1.0 to 12.1.1 in /src-ui @dependabot ([\#861](https://github.com/paperless-ngx/paperless-ngx/pull/861))
+- Bump @<!---->types/node from 17.0.27 to 17.0.29 in /src-ui @dependabot ([\#833](https://github.com/paperless-ngx/paperless-ngx/pull/833))
+- Bump @<!---->ng-bootstrap/ng-bootstrap from 12.0.2 to 12.1.0 in /src-ui @dependabot ([\#834](https://github.com/paperless-ngx/paperless-ngx/pull/834))
+- Bump pytest from 7.1.1 to 7.1.2 @dependabot ([\#806](https://github.com/paperless-ngx/paperless-ngx/pull/806))
+- Bump github/codeql-action from 1 to 2 @dependabot ([\#792](https://github.com/paperless-ngx/paperless-ngx/pull/792))
+- Bump imap-tools from 0.53.0 to 0.54.0 @dependabot ([\#758](https://github.com/paperless-ngx/paperless-ngx/pull/758))
+- Bump ocrmypdf from 13.4.2 to 13.4.3 @dependabot ([\#757](https://github.com/paperless-ngx/paperless-ngx/pull/757))
+- Bump importlib-resources from 5.6.0 to 5.7.1 @dependabot ([\#756](https://github.com/paperless-ngx/paperless-ngx/pull/756))
+- Bump tox from 3.24.5 to 3.25.0 @dependabot ([\#692](https://github.com/paperless-ngx/paperless-ngx/pull/692))
+- Bump cypress from 9.5.3 to 9.6.0 in /src-ui @dependabot ([\#800](https://github.com/paperless-ngx/paperless-ngx/pull/800))
+- Bump angular \& tools to 13.3.4 or 13.3.3 [\@shamoon](https://github.com/shamoon) ([\#799](https://github.com/paperless-ngx/paperless-ngx/pull/799))
+- Bump concurrently from 7.0.0 to 7.1.0 in /src-ui @dependabot ([\#797](https://github.com/paperless-ngx/paperless-ngx/pull/797))
 </details>
 
 ## paperless-ngx 1.7.0
 
-Breaking Changes
+### Breaking Changes
 
 - `PAPERLESS_URL` is now required when using a reverse proxy. See
   [\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674).
 
-Features
+### Features
 
 - Allow setting more than one tag in mail rules
-  [\@jonasc](https://github.com/jonasc) (\#270)
-- global drag\'n\'drop [\@shamoon](https://github.com/shamoon)
-  (\#283).
+  [\@jonasc](https://github.com/jonasc) ([\#270](https://github.com/paperless-ngx/paperless-ngx/pull/270))
+- Global drag\'n\'drop [\@shamoon](https://github.com/shamoon)
+  ([\#283](https://github.com/paperless-ngx/paperless-ngx/pull/283))
 - Fix: download buttons should disable while waiting
-  [\@shamoon](https://github.com/shamoon) (\#630).
-- Update checker [\@shamoon](https://github.com/shamoon) (\#591).
+  [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630))
+- Update checker [\@shamoon](https://github.com/shamoon) ([\#591](https://github.com/paperless-ngx/paperless-ngx/pull/591))
 - Show prompt on password-protected pdfs
-  [\@shamoon](https://github.com/shamoon) (\#564).
+  [\@shamoon](https://github.com/shamoon) ([\#564](https://github.com/paperless-ngx/paperless-ngx/pull/564))
 - Filtering query params aka browser navigation for filtering
-  [\@shamoon](https://github.com/shamoon) (\#540).
+  [\@shamoon](https://github.com/shamoon) ([\#540](https://github.com/paperless-ngx/paperless-ngx/pull/540))
 - Clickable tags in dashboard widgets
-  [\@shamoon](https://github.com/shamoon) (\#515).
+  [\@shamoon](https://github.com/shamoon) ([\#515](https://github.com/paperless-ngx/paperless-ngx/pull/515))
 - Add bottom pagination [\@shamoon](https://github.com/shamoon)
-  (\#372).
+  ([\#372](https://github.com/paperless-ngx/paperless-ngx/pull/372))
 - Feature barcode splitter [\@gador](https://github.com/gador)
-  (\#532).
-- App loading screen [\@shamoon](https://github.com/shamoon) (\#298).
+  ([\#532](https://github.com/paperless-ngx/paperless-ngx/pull/532))
+- App loading screen [\@shamoon](https://github.com/shamoon) ([\#298](https://github.com/paperless-ngx/paperless-ngx/pull/298))
 - Use progress bar for delayed buttons
-  [\@shamoon](https://github.com/shamoon) (\#415).
+  [\@shamoon](https://github.com/shamoon) ([\#415](https://github.com/paperless-ngx/paperless-ngx/pull/415))
 - Add minimum length for documents text filter
-  [\@shamoon](https://github.com/shamoon) (\#401).
+  [\@shamoon](https://github.com/shamoon) ([\#401](https://github.com/paperless-ngx/paperless-ngx/pull/401))
 - Added nav buttons in the document detail view
-  [\@GruberViktor](https://github.com/gruberviktor) (\#273).
+  [\@GruberViktor](https://github.com/gruberviktor) ([\#273](https://github.com/paperless-ngx/paperless-ngx/pull/273))
 - Improve date keyboard input [\@shamoon](https://github.com/shamoon)
-  (\#253).
-- Color theming [\@shamoon](https://github.com/shamoon) (\#243).
+  ([\#253](https://github.com/paperless-ngx/paperless-ngx/pull/253))
+- Color theming [\@shamoon](https://github.com/shamoon) ([\#243](https://github.com/paperless-ngx/paperless-ngx/pull/243))
 - Parse dates when entered without separators
-  [\@GruberViktor](https://github.com/gruberviktor) (\#250).
+  [\@GruberViktor](https://github.com/gruberviktor) ([\#250](https://github.com/paperless-ngx/paperless-ngx/pull/250))
 
-Bug Fixes
+### Bug Fixes
 
-- add \"localhost\" to ALLOWED_HOSTS
-  [\@gador](https://github.com/gador) (\#700).
-- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690).
+- Add \"localhost\" to ALLOWED_HOSTS
+  [\@gador](https://github.com/gador) ([\#700](https://github.com/paperless-ngx/paperless-ngx/pull/700))
+- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690))
 - Adds wait for file before consuming
-  [\@stumpylog](https://github.com/stumpylog) (\#483).
+  [\@stumpylog](https://github.com/stumpylog) ([\#483](https://github.com/paperless-ngx/paperless-ngx/pull/483))
 - Fix: frontend document editing erases time data
-  [\@shamoon](https://github.com/shamoon) (\#654).
+  [\@shamoon](https://github.com/shamoon) ([\#654](https://github.com/paperless-ngx/paperless-ngx/pull/654))
 - Increase length of SavedViewFilterRule
-  [\@stumpylog](https://github.com/stumpylog) (\#612).
+  [\@stumpylog](https://github.com/stumpylog) ([\#612](https://github.com/paperless-ngx/paperless-ngx/pull/612))
 - Fixes attachment filename matching during mail fetching
-  [\@stumpylog](https://github.com/stumpylog) (\#680).
+  [\@stumpylog](https://github.com/stumpylog) ([\#680](https://github.com/paperless-ngx/paperless-ngx/pull/680))
 - Add `PAPERLESS_URL` env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/discussions/674))
 - Fix: download buttons should disable while waiting
-  [\@shamoon](https://github.com/shamoon) (\#630).
+  [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630))
 - Fixes downloaded filename, add more consumer ignore settings
-  [\@stumpylog](https://github.com/stumpylog) (\#599).
+  [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599))
 - FIX BUG: case-sensitive matching was not possible
-  [\@danielBreitlauch](https://github.com/danielbreitlauch) (\#594).
-- uses shutil.move instead of rename
-  [\@gador](https://github.com/gador) (\#617).
+  [\@danielBreitlauch](https://github.com/danielbreitlauch) ([\#594](https://github.com/paperless-ngx/paperless-ngx/pull/594))
+- Uses shutil.move instead of rename
+  [\@gador](https://github.com/gador) ([\#617](https://github.com/paperless-ngx/paperless-ngx/pull/617))
 - Fix npm deps 01.02.22 2 [\@shamoon](https://github.com/shamoon)
-  (\#610).
+  ([\#610](https://github.com/paperless-ngx/paperless-ngx/discussions/610))
 - Fix npm dependencies 01.02.22
-  [\@shamoon](https://github.com/shamoon) (\#600).
-- fix issue 416: implement PAPERLESS_OCR_MAX_IMAGE_PIXELS
-  [\@hacker-h](https://github.com/hacker-h) (\#441).
-- fix: exclude cypress from build in Dockerfile
-  [\@FrankStrieter](https://github.com/FrankStrieter) (\#526).
+  [\@shamoon](https://github.com/shamoon) ([\#600](https://github.com/paperless-ngx/paperless-ngx/pull/600))
+- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
+  [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441))
+- Fix: exclude cypress from build in Dockerfile
+  [\@FrankStrieter](https://github.com/FrankStrieter) ([\#526](https://github.com/paperless-ngx/paperless-ngx/pull/526))
 - Corrections to pass pre-commit hooks
-  [\@schnuffle](https://github.com/schnuffle) (\#454).
+  [\@schnuffle](https://github.com/schnuffle) ([\#454](https://github.com/paperless-ngx/paperless-ngx/pull/454))
 - Fix 311 unable to click checkboxes in document list
-  [\@shamoon](https://github.com/shamoon) (\#313).
+  [\@shamoon](https://github.com/shamoon) ([\#313](https://github.com/paperless-ngx/paperless-ngx/pull/313))
 - Fix imap tools bug [\@stumpylog](https://github.com/stumpylog)
-  (\#393).
+  ([\#393](https://github.com/paperless-ngx/paperless-ngx/pull/393))
 - Fix filterable dropdown buttons arent translated
-  [\@shamoon](https://github.com/shamoon) (\#366).
+  [\@shamoon](https://github.com/shamoon) ([\#366](https://github.com/paperless-ngx/paperless-ngx/pull/366))
 - Fix 224: \"Auto-detected date is day before receipt date\"
-  [\@a17t](https://github.com/a17t) (\#246).
+  [\@a17t](https://github.com/a17t) ([\#246](https://github.com/paperless-ngx/paperless-ngx/pull/246))
 - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon)
-  (\#322).
+  ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322))
 - Fix page links hidden [\@shamoon](https://github.com/shamoon)
-  (\#314).
+  ([\#314](https://github.com/paperless-ngx/paperless-ngx/pull/314))
 - Fix: Include excluded items in dropdown count
-  [\@shamoon](https://github.com/shamoon) (\#263).
+  [\@shamoon](https://github.com/shamoon) ([\#263](https://github.com/paperless-ngx/paperless-ngx/pull/263))
 
-Translation
+### Translation
 
 - [\@miku323](https://github.com/miku323) contributed to Slovenian
-  translation.
+  translation
 - [\@FaintGhost](https://github.com/FaintGhost) contributed to Chinese
-  Simplified translation.
+  Simplified translation
 - [\@DarkoBG79](https://github.com/DarkoBG79) contributed to Serbian
-  translation.
+  translation
 - [Kemal Secer](https://crowdin.com/profile/kemal.secer) contributed
-  to Turkish translation.
+  to Turkish translation
 - [\@Prominence](https://github.com/Prominence) contributed to
-  Belarusian translation.
+  Belarusian translation
 
-Documentation
+### Documentation
 
-- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690).
-- Add [PAPERLESS\_URL]{.title-ref} env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690))
+- Add `PAPERLESS_URL` env variable & CSRF var
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674))
 - Fixes downloaded filename, add more consumer ignore settings
-  [\@stumpylog](https://github.com/stumpylog) (\#599).
-- fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
-  [\@hacker-h](https://github.com/hacker-h) (\#441).
+  [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599))
+- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
+  [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441))
 - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon)
-  (\#322).
+  ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322))
 
-Maintenance
+### Maintenance
 
 - Add `PAPERLESS_URL` env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674))
 - Chore: Implement release-drafter action for Changelogs
-  [\@qcasey](https://github.com/qcasey) (\#669).
-- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) (\#667).
+  [\@qcasey](https://github.com/qcasey) ([\#669](https://github.com/paperless-ngx/paperless-ngx/pull/669))
+- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) ([\#667](https://github.com/paperless-ngx/paperless-ngx/pull/667))
 - Support docker-compose v2 in install
-  [\@stumpylog](https://github.com/stumpylog) (\#611).
+  [\@stumpylog](https://github.com/stumpylog) ([\#611](https://github.com/paperless-ngx/paperless-ngx/pull/611))
 - Add Belarusian localization [\@shamoon](https://github.com/shamoon)
-  (\#588).
+  ([\#588](https://github.com/paperless-ngx/paperless-ngx/pull/588))
 - Add Turkish localization [\@shamoon](https://github.com/shamoon)
-  (\#536).
+  ([\#536](https://github.com/paperless-ngx/paperless-ngx/pull/536))
 - Add Serbian localization [\@shamoon](https://github.com/shamoon)
-  (\#504).
+  ([\#504](https://github.com/paperless-ngx/paperless-ngx/pull/504))
 - Create PULL_REQUEST_TEMPLATE.md
-  [\@shamoon](https://github.com/shamoon) (\#304).
+  [\@shamoon](https://github.com/shamoon) ([\#304](https://github.com/paperless-ngx/paperless-ngx/pull/304))
 - Add Chinese localization [\@shamoon](https://github.com/shamoon)
-  (\#247).
+  ([\#247](https://github.com/paperless-ngx/paperless-ngx/pull/247))
 - Add Slovenian language for frontend
-  [\@shamoon](https://github.com/shamoon) (\#315).
+  [\@shamoon](https://github.com/shamoon) ([\#315](https://github.com/paperless-ngx/paperless-ngx/pull/315))
 
 ## paperless-ngx 1.6.0
 
@@ -216,46 +546,46 @@ include:
 - Updated Python and Angular dependencies.
 - Dropped support for Python 3.7.
 - Dropped support for Ansible playbooks (thanks
-  [\@slankes](https://github.com/slankes) \#109). If someone would
-  like to continue supporting them, please see the [ansible
+  [\@slankes](https://github.com/slankes) [\#109](https://github.com/paperless-ngx/paperless-ngx/pull/109)). If someone would
+  like to continue supporting them, please see our [ansible
   repo](https://github.com/paperless-ngx/paperless-ngx-ansible).
 - Python code is now required to use Black formatting (thanks
-  [\@kpj](https://github.com/kpj) \#168).
+  [\@kpj](https://github.com/kpj) [\#168](https://github.com/paperless-ngx/paperless-ngx/pull/168)).
 - [\@tribut](https://github.com/tribut) added support for a custom SSO
-  logout redirect (jonaswinkler\#1258). See
+  logout redirect ([jonaswinkler\#1258](https://github.com/jonaswinkler/paperless-ng/pull/1258)). See
   `PAPERLESS_LOGOUT_REDIRECT_URL`.
 - [\@shamoon](https://github.com/shamoon) added a loading indicator
-  when document list is reloading (jonaswinkler\#1297).
+  when document list is reloading ([jonaswinkler\#1297](https://github.com/jonaswinkler/paperless-ng/pull/1297)).
 - [\@shamoon](https://github.com/shamoon) improved the PDF viewer on
-  mobile (\#2).
+  mobile ([\#2](https://github.com/paperless-ngx/paperless-ngx/pull/2)).
 - [\@shamoon](https://github.com/shamoon) added \'any\' / \'all\' and
-  \'not\' filtering with tags (\#10).
+  \'not\' filtering with tags ([\#10](https://github.com/paperless-ngx/paperless-ngx/pull/10)).
 - [\@shamoon](https://github.com/shamoon) added warnings for unsaved
-  changes, with smart edit buttons (\#13).
+  changes, with smart edit buttons ([\#13](https://github.com/paperless-ngx/paperless-ngx/pull/13)).
 - [\@benjaminfrank](https://github.com/benjaminfrank) enabled a
-  non-root access to port 80 via systemd (\#18).
+  non-root access to port 80 via systemd ([\#18](https://github.com/paperless-ngx/paperless-ngx/pull/18)).
 - [\@tribut](https://github.com/tribut) added simple \"delete to
-  trash\" functionality (\#24). See `PAPERLESS_TRASH_DIR`.
+  trash\" functionality ([\#24](https://github.com/paperless-ngx/paperless-ngx/pull/24)). See `PAPERLESS_TRASH_DIR`.
 - [\@amenk](https://github.com/amenk) fixed the search box overlay
-  menu on mobile (\#32).
+  menu on mobile ([\#32](https://github.com/paperless-ngx/paperless-ngx/pull/32)).
 - [\@dblitt](https://github.com/dblitt) updated the login form to not
-  auto-capitalize usernames (\#36).
+  auto-capitalize usernames ([\#36](https://github.com/paperless-ngx/paperless-ngx/pull/36)).
 - [\@evilsidekick293](https://github.com/evilsidekick293) made the
-  worker timeout configurable (\#37). See `PAPERLESS_WORKER_TIMEOUT`.
+  worker timeout configurable ([\#37](https://github.com/paperless-ngx/paperless-ngx/pull/37)). See `PAPERLESS_WORKER_TIMEOUT`.
 - [\@Nicarim](https://github.com/Nicarim) fixed downloads of UTF-8
-  formatted documents in Firefox (\#56).
+  formatted documents in Firefox ([\#56](https://github.com/paperless-ngx/paperless-ngx/pull/56)).
 - [\@mweimerskirch](https://github.com/mweimerskirch) sorted the
-  language dropdown by locale (\#78).
+  language dropdown by locale ([\#78](https://github.com/paperless-ngx/paperless-ngx/issues/78)).
 - [\@mweimerskirch](https://github.com/mweimerskirch) enabled the
-  Czech (\#83) and Danish (\#84) translations.
+  Czech ([\#83](https://github.com/paperless-ngx/paperless-ngx/pull/83)) and Danish ([\#84](https://github.com/paperless-ngx/paperless-ngx/pull/84)) translations.
 - [\@cschmatzler](https://github.com/cschmatzler) enabled specifying
-  the webserver port (\#124). See `PAPERLESS_PORT`.
+  the webserver port ([\#124](https://github.com/paperless-ngx/paperless-ngx/pull/124)). See `PAPERLESS_PORT`.
 - [\@muellermartin](https://github.com/muellermartin) fixed an error
-  when uploading transparent PNGs (\#133).
+  when uploading transparent PNGs ([\#133](https://github.com/paperless-ngx/paperless-ngx/pull/133)).
 - [\@shamoon](https://github.com/shamoon) created a slick new logo
-  (\#165).
+  ([\#165](https://github.com/paperless-ngx/paperless-ngx/pull/165)).
 - [\@tim-vogel](https://github.com/tim-vogel) fixed exports missing
-  groups (\#193).
+  groups ([\#193](https://github.com/paperless-ngx/paperless-ngx/pull/193)).
 
 Known issues:
 

docs/configuration.rst

@@ -27,11 +27,23 @@ PAPERLESS_REDIS=<url>
     This is required for processing scheduled tasks such as email fetching, index
     optimization and for training the automatic document matcher.
 
+    * If your Redis server needs login credentials PAPERLESS_REDIS = ``redis://<username>:<password>@<host>:<port>``
+
+    * With the requirepass option PAPERLESS_REDIS = ``redis://:<password>@<host>:<port>``
+
+    `More information on securing your Redis Instance <https://redis.io/docs/getting-started/#securing-redis>`_.
+
     Defaults to redis://localhost:6379.
 
+PAPERLESS_DBENGINE=<engine_name>
+    Optional, gives the ability to choose Postgres or MariaDB for database engine.
+    Available options are `postgresql` and `mariadb`.
+    Default is `postgresql`.
+
 PAPERLESS_DBHOST=<hostname>
     By default, sqlite is used as the database backend. This can be changed here.
-    Set PAPERLESS_DBHOST and PostgreSQL will be used instead of mysql.
+
+    Set PAPERLESS_DBHOST and another database will be used instead of sqlite.
 
 PAPERLESS_DBPORT=<port>
     Adjust port if necessary.
@@ -39,17 +51,17 @@ PAPERLESS_DBPORT=<port>
     Default is 5432.
 
 PAPERLESS_DBNAME=<name>
-    Database name in PostgreSQL.
+    Database name in PostgreSQL or MariaDB.
 
     Defaults to "paperless".
 
 PAPERLESS_DBUSER=<name>
-    Database user in PostgreSQL.
+    Database user in PostgreSQL or MariaDB.
 
     Defaults to "paperless".
 
 PAPERLESS_DBPASS=<password>
-    Database password for PostgreSQL.
+    Database password for PostgreSQL or MariaDB.
 
     Defaults to "paperless".
 
@@ -60,6 +72,13 @@ PAPERLESS_DBSSLMODE=<mode>
 
     Default is ``prefer``.
 
+PAPERLESS_DB_TIMEOUT=<float>
+    Amount of time for a database connection to wait for the database to unlock.
+    Mostly applicable for an sqlite based installation, consider changing to postgresql
+    if you need to increase this.
+
+    Defaults to unset, keeping the Django defaults.
+
 Paths and folders
 #################
 
@@ -111,6 +130,14 @@ PAPERLESS_FILENAME_FORMAT=<format>
 
     Default is none, which disables this feature.
 
+PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=<bool>
+    Tells paperless to replace placeholders in `PAPERLESS_FILENAME_FORMAT` that would resolve
+    to 'none' to be omitted from the resulting filename. This also holds true for directory
+    names.
+    See :ref:`advanced-file_name_handling` for details.
+
+    Defaults to `false` which disables this feature.
+
 PAPERLESS_LOGGING_DIR=<path>
     This is where paperless will store log files.
 
@@ -194,9 +221,16 @@ PAPERLESS_FORCE_SCRIPT_NAME=<path>
 PAPERLESS_STATIC_URL=<path>
     Override the STATIC_URL here.  Unless you're hosting Paperless off a
     subdomain like /paperless/, you probably don't need to change this.
+    If you do change it, be sure to include the trailing slash.
 
     Defaults to "/static/".
 
+    .. note::
+
+        When hosting paperless behind a reverse proxy like Traefik or Nginx at a subpath e.g.
+        example.com/paperlessngx you will also need to set ``PAPERLESS_FORCE_SCRIPT_NAME``
+        (see above).
+
 PAPERLESS_AUTO_LOGIN_USERNAME=<username>
     Specify a username here so that paperless will automatically perform login
     with the selected user.
@@ -416,14 +450,23 @@ PAPERLESS_OCR_IMAGE_DPI=<num>
     the produced PDF documents are A4 sized.
 
 PAPERLESS_OCR_MAX_IMAGE_PIXELS=<num>
-    Paperless will not OCR images that have more pixels than this limit.
-    This is intended to prevent decompression bombs from overloading paperless.
-    Increasing this limit is desired if you face a DecompressionBombError despite
-    the concerning file not being malicious; this could e.g. be caused by invalidly
-    recognized metadata.
-    If you have enough resources or if you are certain that your uploaded files
-    are not malicious you can increase this value to your needs.
-    The default value is 256000000, an image with more pixels than that would not be parsed.
+    Paperless will raise a warning when OCRing images which are over this limit and
+    will not OCR images which are more than twice this limit.  Note this does not
+    prevent the document from being consumed, but could result in missing text content.
+
+    If unset, will default to the value determined by
+    `Pillow <https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.MAX_IMAGE_PIXELS>`_.
+
+    .. note::
+
+        Increasing this limit could cause Paperless to consume additional resources
+        when consuming a file.  Be sure you have sufficient system resources.
+
+    .. caution::
+
+        The limit is intended to prevent malicious files from consuming system resources
+        and causing crashes and other errors.  Only increase this value if you are certain
+        your documents are not malicious and you need the text which was not OCRed
 
 PAPERLESS_OCR_USER_ARGS=<json>
     OCRmyPDF offers many more options. Use this parameter to specify any
@@ -495,7 +538,7 @@ requires are as follows:
         # ...
 
         gotenberg:
-            image: gotenberg/gotenberg:7.4
+            image: gotenberg/gotenberg:7.6
             restart: unless-stopped
             command:
                 - "gotenberg"
@@ -519,6 +562,8 @@ PAPERLESS_TASK_WORKERS=<num>
     maintain the automatic matching algorithm, check emails, consume documents,
     etc. This variable specifies how many things it will do in parallel.
 
+    Defaults to 1
+
 
 PAPERLESS_THREADS_PER_WORKER=<num>
     Furthermore, paperless uses multiple threads when consuming documents to
@@ -590,6 +635,28 @@ PAPERLESS_CONSUMER_POLLING=<num>
 
     Defaults to 0, which disables polling and uses filesystem notifications.
 
+PAPERLESS_CONSUMER_POLLING_RETRY_COUNT=<num>
+    If consumer polling is enabled, sets the number of times paperless will check for a
+    file to remain unmodified.
+
+    Defaults to 5.
+
+PAPERLESS_CONSUMER_POLLING_DELAY=<num>
+    If consumer polling is enabled, sets the delay in seconds between each check (above) paperless
+    will do while waiting for a file to remain unmodified.
+
+    Defaults to 5.
+
+.. _configuration-inotify:
+
+PAPERLESS_CONSUMER_INOTIFY_DELAY=<num>
+    Sets the time in seconds the consumer will wait for additional events
+    from inotify before the consumer will consider a file ready and begin consumption.
+    Certain scanners or network setups may generate multiple events for a single file,
+    leading to multiple consumers working on the same file.  Configure this to
+    prevent that.
+
+    Defaults to 0.5 seconds.
 
 PAPERLESS_CONSUMER_DELETE_DUPLICATES=<bool>
     When the consumer detects a duplicate document, it will not touch the
@@ -634,6 +701,7 @@ PAPERLESS_CONSUMER_ENABLE_BARCODES=<bool>
 
     Defaults to false.
 
+
 PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT=<bool>
     Whether TIFF image files should be scanned for barcodes.
     This will automatically convert any TIFF image(s) to pdfs for later
@@ -650,7 +718,6 @@ PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
 
   Defaults to "PATCHT"
 
-
 PAPERLESS_CONVERT_MEMORY_LIMIT=<num>
     On smaller systems, or even in the case of Very Large Documents, the consumer
     may explode, complaining about how it's "unable to extend pixel cache".  In
@@ -674,13 +741,6 @@ PAPERLESS_CONVERT_TMPDIR=<path>
 
     Default is none, which disables the temporary directory.
 
-PAPERLESS_OPTIMIZE_THUMBNAILS=<bool>
-    Use optipng to optimize thumbnails. This usually reduces the size of
-    thumbnails by about 20%, but uses considerable compute time during
-    consumption.
-
-    Defaults to true.
-
 PAPERLESS_POST_CONSUME_SCRIPT=<filename>
     After a document is consumed, Paperless can trigger an arbitrary script if
     you like.  This script will be passed a number of arguments for you to work
@@ -696,8 +756,24 @@ PAPERLESS_FILENAME_DATE_ORDER=<format>
     The filename will be checked first, and if nothing is found, the document
     text will be checked as normal.
 
+    A date in a filename must have some separators (`.`, `-`, `/`, etc)
+    for it to be parsed.
+
     Defaults to none, which disables this feature.
 
+PAPERLESS_NUMBER_OF_SUGGESTED_DATES=<num>
+    Paperless searches an entire document for dates. The first date found will
+    be used as the initial value for the created date. When this variable is
+    greater than 0 (or left to it's default value), paperless will also suggest
+    other dates found in the document, up to a maximum of this setting. Note that
+    duplicates will be removed, which can result in fewer dates displayed in the
+    frontend than this setting value.
+
+    The task to find all dates can be time-consuming and increases with a higher
+    (maximum) number of suggested dates and slower hardware.
+
+    Defaults to 3. Set to 0 to disable this feature.
+
 PAPERLESS_THUMBNAIL_FONT_NAME=<filename>
     Paperless creates thumbnails for plain text files by rendering the content
     of the file on an image and uses a predefined font for that. This
@@ -713,10 +789,7 @@ PAPERLESS_IGNORE_DATES=<string>
     this process. This is useful for special dates (like date of birth) that appear
     in documents regularly but are very unlikely to be the documents creation date.
 
-    You may specify dates in a multitude of formats supported by dateparser (see
-    https://dateparser.readthedocs.io/en/latest/#popular-formats) but as the dates
-    need to be comma separated, the options are limited.
-    Example: "2020-12-02,22.04.1999"
+    The date is parsed using the order specified in PAPERLESS_DATE_ORDER
 
     Defaults to an empty string to not ignore any dates.
 
@@ -746,13 +819,10 @@ the program doesn't automatically execute it (ie. the program isn't in your
 $PATH), then you'll need to specify the literal path for that program.
 
 PAPERLESS_CONVERT_BINARY=<path>
-    Defaults to "/usr/bin/convert".
+    Defaults to "convert".
 
 PAPERLESS_GS_BINARY=<path>
-    Defaults to "/usr/bin/gs".
-
-PAPERLESS_OPTIPNG_BINARY=<path>
-    Defaults to "/usr/bin/optipng".
+    Defaults to "gs".
 
 
 .. _configuration-docker:
@@ -769,9 +839,14 @@ PAPERLESS_WEBSERVER_WORKERS=<num>
     also loads the entire application into memory separately, so increasing this value
     will increase RAM usage.
 
-    Consider configuring this to 1 on low power devices with limited amount of RAM.
+    Defaults to 1.
 
-    Defaults to 2.
+PAPERLESS_BIND_ADDR=<ip address>
+    The IP address the webserver will listen on inside the container. There are
+    special setups where you may need to configure this value to restrict the
+    Ip address or interface the webserver listens on.
+
+    Defaults to [::], meaning all interfaces, including IPv6.
 
 PAPERLESS_PORT=<port>
     The port number the webserver will listen on inside the container. There are
@@ -827,6 +902,14 @@ PAPERLESS_OCR_LANGUAGES=<list>
 
     Defaults to none, which does not install any additional languages.
 
+PAPERLESS_ENABLE_FLOWER=<defined>
+    If this environment variable is defined, the Celery monitoring tool
+    `Flower <https://flower.readthedocs.io/en/latest/index.html>`_ will
+    be started by the container.
+
+    You can read more about this in the :ref:`advanced setup <advanced-celery-monitoring>`
+    documentation.
+
 
 .. _configuration-update-checking:
 
@@ -834,18 +917,9 @@ Update Checking
 ###############
 
 PAPERLESS_ENABLE_UPDATE_CHECK=<bool>
-    Enable (or disable) the automatic check for available updates. This feature is disabled
-    by default but if it is not explicitly set Paperless-ngx will show a message about this.
 
-    If enabled, the feature works by pinging the the Github API for the latest release e.g.
-    https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest
-    to determine whether a new version is available.
+    .. note::
 
-    Actual updating of the app must still be performed manually.
-
-    Note that for users of thirdy-party containers e.g. linuxserver.io this notification
-    may be 'ahead' of a new release from the third-party maintainers.
-
-    In either case, no tracking data is collected by the app in any way.
-
-    Defaults to none, which disables the feature.
+            This setting was deprecated in favor of a frontend setting after v1.9.2. A one-time
+            migration is performed for users who have this setting set. This setting is always
+            ignored if the corresponding frontend setting has been set.

docs/extending.rst

@@ -79,7 +79,7 @@ To do the setup you need to perform the steps from the following chapters in a c
 6.  You can now either ...
 
     *  install redis or
-    *  use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a postgresql server) or
+    *  use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a database server) or
     *  spin up a bare redis container
 
         .. code:: shell-session
@@ -112,7 +112,7 @@ To do the setup you need to perform the steps from the following chapters in a c
 
     .. code:: shell-session
 
-        python3 manage.py runserver & python3 manage.py document_consumer & python3 manage.py qcluster
+        python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
 
 11. Login with the superuser credentials provided in step 8 at ``http://localhost:8000`` to create a session that enables you to use the backend.
 
@@ -128,14 +128,14 @@ Configure the IDE to use the src/ folder as the base source folder. Configure th
 launch configurations in your IDE:
 
 *   python3 manage.py runserver
-*   python3 manage.py qcluster
+*   celery --app paperless worker
 *   python3 manage.py document_consumer
 
 To start them all:
 
 .. code:: shell-session
 
-    python3 manage.py runserver & python3 manage.py document_consumer & python3 manage.py qcluster
+    python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
 
 Testing and code style:
 

docs/index.rst

@@ -44,7 +44,7 @@ resources in the documentation:
     learn about how paperless automates all tagging using machine learning.
 *   Paperless now comes with a :ref:`proper email consumer <usage-email>`
     that's fully tested and production ready.
-*   Paperless creates searchable PDF/A documents from whatever you you put into
+*   Paperless creates searchable PDF/A documents from whatever you put into
     the consumption directory. This means that you can select text in
     image-only documents coming from your scanner.
 *   See :ref:`this note <utilities-encyption>` about GnuPG encryption in

docs/requirements.txt

@@ -0,0 +1 @@
+myst-parser==0.18.1

docs/scanners.rst

@@ -1,133 +1,8 @@
 
 .. _scanners:
 
-***********************
-Scanner recommendations
-***********************
+*******************
+Scanners & Software
+*******************
 
-As Paperless operates by watching a folder for new files, doesn't care what
-scanner you use, but sometimes finding a scanner that will write to an FTP,
-NFS, or SMB server can be difficult.  This page is here to help you find one
-that works right for you based on recommendations from other Paperless users.
-
-Physical scanners
-=================
-
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brand   | Model          | Supports                                 | Recommended By |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-|         |                | FTP | SFTP | NFS | SMB | SMTP | API [1]_ |                |
-+=========+================+=====+======+=====+=====+======+==========+================+
-| Brother | `ADS-1700W`_   | yes |      |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-1600W`_   | yes |      |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-1500W`_   | yes |      |     | yes | yes  |          |`danielquinn`_  |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-1100W`_   | yes |      |     |     |      |          |`ytzelf`_       |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `ADS-2800W`_   | yes | yes  |     | yes | yes  |          |`philpagel`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-J6930DW`_ | yes |      |     |     |      |          |`ayounggun`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-L5850DW`_ | yes |      |     |     | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-L2750DW`_ | yes |      |     | yes | yes  |          |`muued`_        |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-J5910DW`_ | yes |      |     |     |      |          |`bmsleight`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-8950DW`_  | yes |      |     | yes | yes  |          |`philpagel`_    |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Brother | `MFC-9142CDN`_ | yes |      |     | yes |      |          |`REOLDEV`_      |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Fujitsu | `ix500`_       | yes |      |     | yes |      |          |`eonist`_       |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Epson   | `ES-580W`_     | yes |      |     | yes | yes  |          |`fignew`_       |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Epson   | `WF-7710DWF`_  | yes |      |     | yes |      |          |`Skylinar`_     |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Fujitsu | `S1300i`_      | yes |      |     | yes |      |          |`jonaswinkler`_ |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-| Doxie   | `Q2`_          |     |      |     |     |      | yes      |`Unkn0wnCat`_   |
-+---------+----------------+-----+------+-----+-----+------+----------+----------------+
-
-.. _MFC-L5850DW: https://www.brother-usa.com/products/mfcl5850dw
-.. _MFC-L2750DW: https://www.brother.de/drucker/laserdrucker/mfc-l2750dw
-.. _ADS-1700W: https://www.brother-usa.com/products/ads1700w
-.. _ADS-1600W: https://www.brother-usa.com/products/ads1600w
-.. _ADS-1500W: https://www.brother.ca/en/p/ads1500w
-.. _ADS-1100W: https://support.brother.com/g/b/downloadtop.aspx?c=fr&lang=fr&prod=ads1100w_eu_as_cn
-.. _ADS-2800W: https://www.brother-usa.com/products/ads2800w
-.. _MFC-J6930DW: https://www.brother.ca/en/p/MFCJ6930DW
-.. _MFC-J5910DW: https://www.brother.co.uk/printers/inkjet-printers/mfcj5910dw
-.. _MFC-8950DW: https://www.brother-usa.com/products/mfc8950dw
-.. _MFC-9142CDN: https://www.brother.co.uk/printers/laser-printers/mfc9140cdn
-.. _ES-580W: https://epson.com/Support/Scanners/ES-Series/Epson-WorkForce-ES-580W/s/SPT_B11B258201
-.. _WF-7710DWF: https://www.epson.de/en/products/printers/inkjet-printers/for-home/workforce-wf-7710dwf
-.. _ix500: http://www.fujitsu.com/us/products/computing/peripheral/scanners/scansnap/ix500/
-.. _S1300i: https://www.fujitsu.com/global/products/computing/peripheral/scanners/soho/s1300i/
-.. _Q2: https://www.getdoxie.com/product/doxie-q/
-
-.. _ayounggun: https://github.com/ayounggun
-.. _bmsleight: https://github.com/bmsleight
-.. _danielquinn: https://github.com/danielquinn
-.. _eonist: https://github.com/eonist
-.. _fignew: https://github.com/fignew
-.. _holzhannes: https://github.com/holzhannes
-.. _jonaswinkler: https://github.com/jonaswinkler
-.. _REOLDEV: https://github.com/REOLDEV
-.. _Skylinar: https://github.com/Skylinar
-.. _ytzelf: https://github.com/ytzelf
-.. _Unkn0wnCat: https://github.com/Unkn0wnCat
-.. _muued: https://github.com/muued
-.. _philpagel: https://github.com/philpagel
-
-.. [1] Scanners with API Integration allow to push scanned documents directly to :ref:`Paperless API <api-file_uploads>`, sometimes referred to as Webhook or Document POST.
-
-Mobile phone software
-=====================
-
-You can use your phone to "scan" documents. The regular camera app will work, but may have too low contrast for OCR to work well. Apps specifically for scanning are recommended.
-
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| Name                        | OS             | Supports                         | Recommended By   |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-|                             |                | FTP | NFS | SMB | Email | WebDAV |                  |
-+=============================+================+=====+=====+=====+=======+========+==================+
-| `Office Lens`_              | Android        | ?   | ?   | ?   | ?     | ?      | `jonaswinkler`_  |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| `Genius Scan`_              | Android        | yes | no  | yes | yes   | yes    | `hannahswain`_   |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| `OpenScan`_                 | Android        | no  | no  | no  | no    | no     | `benjaminfrank`_ |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| `OCR Scanner - QuickScan`_  | iOS            | no  | no  | no  | no    | yes    | `holzhannes`_    |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-
-On Android, you can use these applications in combination with one of the :ref:`Paperless-ngx compatible apps <usage-mobile_upload>` to "Share" the documents produced by these scanner apps with paperless. On iOS, you can share the scanned documents via iOS-Sharing to other mail, WebDav or FTP apps.
-
-.. _Office Lens: https://play.google.com/store/apps/details?id=com.microsoft.office.officelens
-.. _Genius Scan: https://play.google.com/store/apps/details?id=com.thegrizzlylabs.geniusscan.free
-.. _OCR Scanner - QuickScan: https://apps.apple.com/us/app/quickscan-scanner-text-ocr/id1513790291
-.. _OpenScan: https://github.com/Ethereal-Developers-Inc/OpenScan
-
-.. _hannahswain: https://github.com/hannahswain
-.. _benjaminfrank: https://github.com/benjaminfrank
-
-API Scanning Setup
-==================
-
-This sections contains information on how to set up scanners to post directly to :ref:`Paperless API <api-file_uploads>`.
-
-Doxie Q2
---------
-
-This part assumes your Doxie is connected to WiFi and you know its IP.
-
-1. Open your Doxie web UI by navigating to its IP address
-2. Navigate to Options -> Webhook
-3. Set the *URL* to ``https://[your-paperless-ngx-instance]/api/documents/post_document/``
-4. Set the *File Parameter Name* to ``document``
-5. Add the username and password to the respective fields (Consider creating a user just for your Doxie)
-6. Click *Submit* at the bottom of the page
-
-Congrats, you can now scan directly from your Doxie to your Paperless-ngx instance!
+Paperless-ngx is compatible with many different scanners and scanning tools. A user-maintained list of scanners and other software is available on `the wiki <https://github.com/paperless-ngx/paperless-ngx/wiki/Scanner-&-Software-Recommendations>`_.

docs/setup.rst

@@ -39,7 +39,7 @@ Paperless consists of the following components:
 
     .. _setup-task_processor:
 
-*   **The task processor:** Paperless relies on `Django Q <https://django-q.readthedocs.io/en/latest/>`_
+*   **The task processor:** Paperless relies on `Celery - Distributed Task Queue <https://docs.celeryq.dev/en/stable/index.html>`_
     for doing most of the heavy lifting. This is a task queue that accepts tasks from
     multiple sources and processes these in parallel. It also comes with a scheduler that executes
     certain commands periodically.
@@ -62,18 +62,11 @@ Paperless consists of the following components:
     tasks fail and inspect the errors (i.e., wrong email credentials, errors during consuming a specific
     file, etc).
 
-    You may start the task processor by executing:
-
-    .. code:: shell-session
-
-        $ cd /path/to/paperless/src/
-        $ python3 manage.py qcluster
-
 *   A `redis <https://redis.io/>`_ message broker: This is a really lightweight service that is responsible
     for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different
     process (maybe even on different machines!), and therefore, this is necessary.
 
-*   Optional: A database server. Paperless supports both PostgreSQL and SQLite for storing its data.
+*   Optional: A database server. Paperless supports PostgreSQL, MariaDB and SQLite for storing its data.
 
 
 Installation
@@ -184,6 +177,25 @@ Install Paperless from Docker Hub
     port 8000. Modifying the part before the colon will map requests on another
     port to the webserver running on the default port.
 
+    **Rootless**
+
+    If you want to run Paperless as a rootless container, you will need to do the
+    following in your ``docker-compose.yml``:
+
+    - set the ``user`` running the container to map to the ``paperless`` user in the
+      container.
+      This value (``user_id`` below), should be the same id that ``USERMAP_UID`` and
+      ``USERMAP_GID`` are set to in the next step.
+      See ``USERMAP_UID`` and ``USERMAP_GID`` :ref:`here <configuration-docker>`.
+
+    Your entry for Paperless should contain something like:
+
+     .. code::
+
+        webserver:
+          image: ghcr.io/paperless-ngx/paperless-ngx:latest
+          user: <user_id>
+
 5.  Modify ``docker-compose.env``, following the comments in the file. The
     most important change is to set ``USERMAP_UID`` and ``USERMAP_GID``
     to the uid and gid of your user on the host system. Use ``id -u`` and
@@ -200,6 +212,20 @@ Install Paperless from Docker Hub
         You can copy any setting from the file ``paperless.conf.example`` and paste it here.
         Have a look at :ref:`configuration` to see what's available.
 
+    .. note::
+
+        You can utilize Docker secrets for some configuration settings by
+        appending `_FILE` to some configuration values.  This is supported currently
+        only by:
+
+          * PAPERLESS_DBUSER
+          * PAPERLESS_DBPASS
+          * PAPERLESS_SECRET_KEY
+          * PAPERLESS_AUTO_LOGIN_USERNAME
+          * PAPERLESS_ADMIN_USER
+          * PAPERLESS_ADMIN_MAIL
+          * PAPERLESS_ADMIN_PASSWORD
+
     .. caution::
 
         Some file systems such as NFS network shares don't support file system
@@ -258,7 +284,20 @@ Build the Docker image yourself
     .. code:: yaml
 
         webserver:
-            build: .
+            build:
+              context: .
+              args:
+                QPDF_VERSION: x.y.x
+                PIKEPDF_VERSION: x.y.z
+                PSYCOPG2_VERSION: x.y.z
+                JBIG2ENC_VERSION: 0.29
+
+    .. note::
+
+        You should match the build argument versions to the version for the release you have
+        checked out.  These are pre-built images with certain, more updated software.
+        If you want to build these images your self, that is possible, but beyond
+        the scope of these steps.
 
 4.  Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run
     ``docker-compose pull`` to pull the image, do
@@ -284,12 +323,13 @@ writing. Windows is not and will never be supported.
     *   ``python3-pip``
     *   ``python3-dev``
 
+    *   ``default-libmysqlclient-dev`` for MariaDB
     *   ``fonts-liberation`` for generating thumbnails for plain text files
     *   ``imagemagick`` >= 6 for PDF conversion
-    *   ``optipng`` for optimizing thumbnails
     *   ``gnupg`` for handling encrypted documents
     *   ``libpq-dev`` for PostgreSQL
     *   ``libmagic-dev`` for mime type detection
+    *   ``mariadb-client`` for MariaDB compile time
     *   ``mime-support`` for mime type detection
     *   ``libzbar0`` for barcode detection
     *   ``poppler-utils`` for barcode detection
@@ -298,7 +338,7 @@ writing. Windows is not and will never be supported.
 
     .. code::
 
-        python3 python3-pip python3-dev imagemagick fonts-liberation optipng gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils
+        python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev libmagic-dev mime-support libzbar0 poppler-utils
 
     These dependencies are required for OCRmyPDF, which is used for text recognition.
 
@@ -308,7 +348,7 @@ writing. Windows is not and will never be supported.
     *   ``qpdf``
     *   ``liblept5``
     *   ``libxml2``
-    *   ``pngquant``
+    *   ``pngquant`` (suggested for certain PDF image optimizations)
     *   ``zlib1g``
     *   ``tesseract-ocr`` >= 4.0.0 for OCR
     *   ``tesseract-ocr`` language packs (``tesseract-ocr-eng``, ``tesseract-ocr-deu``, etc)
@@ -327,10 +367,16 @@ writing. Windows is not and will never be supported.
     You will also need ``build-essential``, ``python3-setuptools`` and ``python3-wheel``
     for installing some of the python dependencies.
 
-2.  Install ``redis`` >= 5.0 and configure it to start automatically.
+2.  Install ``redis`` >= 6.0 and configure it to start automatically.
 
 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
-    to use PostgreSQL, SQLite is available as well.
+    to use PostgreSQL, MariaDB and SQLite are available as well.
+
+    .. note::
+
+        On bare-metal installations using SQLite, ensure the
+        `JSON1 extension <https://code.djangoproject.com/wiki/JSON1Extension>`_ is enabled. This is
+        usually the case, but not always.
 
 4.  Get the release archive from `<https://github.com/paperless-ngx/paperless-ngx/releases>`_.
     If you clone the git repo as it is, you also have to compile the front end by yourself.
@@ -340,6 +386,7 @@ writing. Windows is not and will never be supported.
     settings to your needs. Required settings for getting paperless running are:
 
     *   ``PAPERLESS_REDIS`` should point to your redis server, such as redis://localhost:6379.
+    *   ``PAPERLESS_DBENGINE`` optional, and should be one of `postgres, mariadb, or sqlite`
     *   ``PAPERLESS_DBHOST`` should be the hostname on which your PostgreSQL server is running. Do not configure this
         to use SQLite instead. Also configure port, database name, user and password as necessary.
     *   ``PAPERLESS_CONSUMPTION_DIR`` should point to a folder which paperless should watch for documents. You might
@@ -420,8 +467,9 @@ writing. Windows is not and will never be supported.
     as a starting point.
 
     Paperless needs the ``webserver`` script to run the webserver, the
-    ``consumer`` script to watch the input folder, and the ``scheduler``
-    script to run tasks such as email checking and document consumption.
+    ``consumer`` script to watch the input folder, ``taskqueue`` for the background workers
+    used to handle things like document consumption and the ``scheduler`` script to run tasks such as
+    email checking at certain times .
 
 		The ``socket`` script enables ``gunicorn`` to run on port 80 without
 		root privileges. For this you need to uncomment the ``Require=paperless-webserver.socket``
@@ -472,6 +520,13 @@ writing. Windows is not and will never be supported.
     to compile this by yourself, because this software has been patented until around 2017 and
     binary packages are not available for most distributions.
 
+15. Optional: If using the NLTK machine learning processing (see ``PAPERLESS_ENABLE_NLTK`` in
+    :ref:`configuration` for details), download the NLTK data for the Snowball Stemmer, Stopwords
+    and Punkt tokenizer to your ``PAPERLESS_DATA_DIR/nltk``.  Refer to
+    the `NLTK instructions <https://www.nltk.org/data.html>`_ for details on how to
+    download the data.
+
+
 Migrating to Paperless-ngx
 ##########################
 
@@ -593,6 +648,48 @@ Migration to paperless-ngx is then performed in a few simple steps:
 10.  Optionally, follow the instructions below to migrate your existing data to PostgreSQL.
 
 
+Migrating from LinuxServer.io Docker Image
+========================
+
+As with any upgrades and large changes, it is highly recommended to create a backup before
+starting.  This assumes the image was running using Docker Compose, but the instructions
+are translatable to Docker commands as well.
+
+1.  Stop and remove the paperless container
+2.  If using an external database, stop the container
+3.  Update Redis configuration
+
+    a)  If ``REDIS_URL`` is already set, change it to ``PAPERLESS_REDIS`` and continue
+        to step 4.
+    b)  Otherwise, in the ``docker-compose.yml`` add a new service for Redis,
+        following `the example compose files <https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose>`_
+    b)  Set the environment variable ``PAPERLESS_REDIS`` so it points to the new Redis container
+
+4.  Update user mapping
+
+    a)  If set, change the environment variable ``PUID`` to ``USERMAP_UID``
+    b)  If set, change the environment variable ``PGID`` to ``USERMAP_GID``
+
+5.  Update configuration paths
+
+    a) Set the environment variable ``PAPERLESS_DATA_DIR``
+       to ``/config``
+
+6.  Update media paths
+
+    a) Set the environment variable ``PAPERLESS_MEDIA_ROOT``
+       to ``/data/media``
+
+7.  Update timezone
+
+    a) Set the environment variable ``PAPERLESS_TIME_ZONE``
+       to the same value as ``TZ``
+
+8.  Modify the ``image:`` to point to ``ghcr.io/paperless-ngx/paperless-ngx:latest`` or
+    a specific version if preferred.
+
+9.  Start the containers as before, using ``docker-compose``.
+
 .. _setup-sqlite_to_psql:
 
 Moving data from SQLite to PostgreSQL
@@ -724,10 +821,10 @@ configuring some options in paperless can help improve performance immensely:
 *   If you want to perform OCR on the device, consider using ``PAPERLESS_OCR_CLEAN=none``.
     This will speed up OCR times and use less memory at the expense of slightly worse
     OCR results.
-*   Set ``PAPERLESS_OPTIMIZE_THUMBNAILS`` to 'false' if you want faster consumption
-    times. Thumbnails will be about 20% larger.
 *   If using docker, consider setting ``PAPERLESS_WEBSERVER_WORKERS`` to
     1. This will save some memory.
+*   Consider setting ``PAPERLESS_ENABLE_NLTK`` to false, to disable the more
+    advanced language processing, which can take more memory and processing time.
 
 For details, refer to :ref:`configuration`.
 

docs/troubleshooting.rst

@@ -19,7 +19,7 @@ Check for the following issues:
 
     .. code:: shell-session
 
-        $ python3 manage.py qcluster
+        $ celery --app paperless worker
 
 *   Look at the output of paperless and inspect it for any errors.
 *   Go to the admin interface, and check if there are failed tasks. If so, the
@@ -125,7 +125,7 @@ If using docker-compose, this is achieved by the following configuration change
 .. code:: yaml
 
     gotenberg:
-        image: gotenberg/gotenberg:7.4
+        image: gotenberg/gotenberg:7.6
         restart: unless-stopped
         command:
             - "gotenberg"
@@ -235,3 +235,94 @@ You might find messages like these in your log files:
 This indicates that paperless failed to read PDF metadata from one of your documents. This happens when you
 open the affected documents in paperless for editing. Paperless will continue to work, and will simply not
 show the invalid metadata.
+
+Consumer fails with a FileNotFoundError
+#######################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.consumer] Error while consuming document SCN_0001.pdf: FileNotFoundError: [Errno 2] No such file or directory: '/tmp/ocrmypdf.io.yhk3zbv0/origin.pdf'
+    Traceback (most recent call last):
+      File "/app/paperless/src/paperless_tesseract/parsers.py", line 261, in parse
+        ocrmypdf.ocr(**args)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/api.py", line 337, in ocr
+        return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 385, in run_pipeline
+        exec_concurrent(context, executor)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 302, in exec_concurrent
+        pdf = post_process(pdf, context, executor)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 235, in post_process
+        pdf_out = metadata_fixup(pdf_out, context)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_pipeline.py", line 798, in metadata_fixup
+        with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
+      File "/usr/local/lib/python3.8/dist-packages/pikepdf/_methods.py", line 923, in open
+        pdf = Pdf._open(
+    FileNotFoundError: [Errno 2] No such file or directory: '/tmp/ocrmypdf.io.yhk3zbv0/origin.pdf'
+
+This probably indicates paperless tried to consume the same file twice.  This can happen for a number of reasons,
+depending on how documents are placed into the consume folder.  If paperless is using inotify (the default) to
+check for documents, try adjusting the :ref:`inotify configuration <configuration-inotify>`.  If polling is enabled,
+try adjusting the :ref:`polling configuration <configuration-polling>`.
+
+Consumer fails waiting for file to remain unmodified.
+#####################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.management.consumer] Timeout while waiting on file /usr/src/paperless/src/../consume/SCN_0001.pdf to remain unmodified.
+
+This indicates paperless timed out while waiting for the file to be completely written to the consume folder.
+Adjusting :ref:`polling configuration <configuration-polling>` values should resolve the issue.
+
+.. note::
+
+    The user will need to manually move the file out of the consume folder and
+    back in, for the initial failing file to be consumed.
+
+Consumer fails reporting "OS reports file as busy still".
+#########################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [WARNING] [paperless.management.consumer] Not consuming file /usr/src/paperless/src/../consume/SCN_0001.pdf: OS reports file as busy still
+
+This indicates paperless was unable to open the file, as the OS reported the file as still being in use.  To prevent a
+crash, paperless did not try to consume the file.  If paperless is using inotify (the default) to
+check for documents, try adjusting the :ref:`inotify configuration <configuration-inotify>`.  If polling is enabled,
+try adjusting the :ref:`polling configuration <configuration-polling>`.
+
+.. note::
+
+    The user will need to manually move the file out of the consume folder and
+    back in, for the initial failing file to be consumed.
+
+Log reports "Creating PaperlessTask failed".
+#########################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.management.consumer] Creating PaperlessTask failed: db locked
+
+You are likely using an sqlite based installation, with an increased number of workers and are running into sqlite's concurrency limitations.
+Uploading or consuming multiple files at once results in many workers attempting to access the database simultaneously.
+
+Consider changing to the PostgreSQL database if you will be processing many documents at once often.  Otherwise,
+try tweaking the ``PAPERLESS_DB_TIMEOUT`` setting to allow more time for the database to unlock.  This may have
+minor performance implications.
+
+
+gunicorn fails to start with "is not a valid port number"
+#########################################################
+
+You are likely running using Kubernetes, which automatically creates an environment variable named `${serviceName}_PORT`.
+This is the same environment variable which is used by Paperless to optionally change the port gunicorn listens on.
+
+To fix this, set `PAPERLESS_PORT` again to your desired port, or the default of 8000.

docs/usage_overview.rst

@@ -161,6 +161,9 @@ These are as follows:
     will not consume flagged mails.
 *   **Move to folder:** Moves consumed mails out of the way so that paperless wont
     consume them again.
+*   **Add custom Tag:** Adds a custom tag to mails with consumed documents (the IMAP
+    standard calls these "keywords"). Paperless will not consume mails already tagged.
+    Not all mail servers support this feature!
 
 .. caution::
 

gunicorn.conf.py

@@ -1,9 +1,17 @@
 import os
 
-bind = f'0.0.0.0:{os.getenv("PAPERLESS_PORT", 8000)}'
-workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 2))
+# See https://docs.gunicorn.org/en/stable/settings.html for
+# explanations of settings
+
+bind = f'{os.getenv("PAPERLESS_BIND_ADDR", "[::]")}:{os.getenv("PAPERLESS_PORT", 8000)}'
+
+workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 1))
 worker_class = "paperless.workers.ConfigurableWorker"
 timeout = 120
+preload_app = True
+
+# https://docs.gunicorn.org/en/stable/faq.html#blocking-os-fchmod
+worker_tmp_dir = "/dev/shm"
 
 
 def pre_fork(server, worker):

install-paperless-ngx.sh

@@ -118,12 +118,12 @@ ask "Current time zone" "$default_time_zone"
 TIME_ZONE=$ask_result
 
 echo ""
-echo "Database backend: PostgreSQL and SQLite are available. Use PostgreSQL"
+echo "Database backend: PostgreSQL, MariaDB, and SQLite are available. Use PostgreSQL"
 echo "if unsure. If you're running on a low-power device such as Raspberry"
 echo "Pi, use SQLite to save resources."
 echo ""
 
-ask "Database backend" "postgres" "postgres sqlite"
+ask "Database backend" "postgres" "postgres sqlite mariadb"
 DATABASE_BACKEND=$ask_result
 
 echo ""
@@ -214,9 +214,9 @@ echo ""
 ask_docker_folder "Data folder" ""
 DATA_FOLDER=$ask_result
 
-if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
+if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then
 	echo ""
-	echo "The database folder, where postgres stores its data."
+	echo "The database folder, where your database stores its data."
 	echo "Leave empty to have this managed by docker."
 	echo ""
 	echo "CAUTION: If specified, you must specify an absolute path starting with /"
@@ -224,7 +224,7 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
 	echo ""
 
 	ask_docker_folder "Database folder" ""
-	POSTGRES_FOLDER=$ask_result
+	DATABASE_FOLDER=$ask_result
 fi
 
 echo ""
@@ -278,13 +278,14 @@ if [[ -z $DATA_FOLDER ]] ; then
 else
 	echo "Data folder: $DATA_FOLDER"
 fi
-if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
-	if [[ -z $POSTGRES_FOLDER ]] ; then
-		echo "Database (postgres) folder: Managed by docker"
+if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then
+	if [[ -z $DATABASE_FOLDER ]] ; then
+		echo "Database folder: Managed by docker"
 	else
-		echo "Database (postgres) folder: $POSTGRES_FOLDER"
+		echo "Database folder: $DATABASE_FOLDER"
 	fi
 fi
+
 echo ""
 echo "URL: $URL"
 echo "Port: $PORT"
@@ -356,9 +357,16 @@ if [[ -n $DATA_FOLDER ]] ; then
 	sed -i "/^\s*data:/d" docker-compose.yml
 fi
 
-if [[ -n $POSTGRES_FOLDER ]] ; then
-	sed -i "s#- pgdata:/var/lib/postgresql/data#- $POSTGRES_FOLDER:/var/lib/postgresql/data#g" docker-compose.yml
-	sed -i "/^\s*pgdata:/d" docker-compose.yml
+# If the database folder was provided (not blank), replace the pgdata/dbdata volume with a bind mount
+# of the provided folder
+if [[ -n $DATABASE_FOLDER ]] ; then
+	if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
+		sed -i "s#- pgdata:/var/lib/postgresql/data#- $DATABASE_FOLDER:/var/lib/postgresql/data#g" docker-compose.yml
+		sed -i "/^\s*pgdata:/d" docker-compose.yml
+	elif [[ "$DATABASE_BACKEND" == "mariadb" ]]; then
+		sed -i "s#- dbdata:/var/lib/mysql#- $DATABASE_FOLDER:/var/lib/mysql#g" docker-compose.yml
+		sed -i "/^\s*dbdata:/d" docker-compose.yml
+	fi
 fi
 
 # remove trailing blank lines from end of file
@@ -375,4 +383,4 @@ ${DOCKER_COMPOSE_CMD} pull
 
 ${DOCKER_COMPOSE_CMD} run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
 
-${DOCKER_COMPOSE_CMD} up -d
+${DOCKER_COMPOSE_CMD} up --detach

paperless.conf.example

@@ -23,6 +23,7 @@
 #PAPERLESS_MEDIA_ROOT=../media
 #PAPERLESS_STATICDIR=../static
 #PAPERLESS_FILENAME_FORMAT=
+#PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=
 
 # Security and hosting
 
@@ -63,12 +64,12 @@
 #PAPERLESS_CONSUMER_IGNORE_PATTERNS=[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]
 #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
 #PAPERLESS_CONSUMER_ENABLE_BARCODES=false
-#PAPERLESS_CONSUMER_ENABLE_BARCODES=PATCHT
-#PAPERLESS_OPTIMIZE_THUMBNAILS=true
+#PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_FILENAME_DATE_ORDER=YMD
 #PAPERLESS_FILENAME_PARSE_TRANSFORMS=[]
+#PAPERLESS_NUMBER_OF_SUGGESTED_DATES=5
 #PAPERLESS_THUMBNAIL_FONT_NAME=
 #PAPERLESS_IGNORE_DATES=
 #PAPERLESS_ENABLE_UPDATE_CHECK=
@@ -83,4 +84,3 @@
 
 #PAPERLESS_CONVERT_BINARY=/usr/bin/convert
 #PAPERLESS_GS_BINARY=/usr/bin/gs
-#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng

requirements.txt

@@ -1,113 +0,0 @@
-#
-# These requirements were autogenerated by pipenv
-# To regenerate from the project's Pipfile, run:
-#
-#    pipenv lock --requirements
-#
-
--i https://pypi.python.org/simple
---extra-index-url https://www.piwheels.org/simple
-aioredis==1.3.1
-anyio==3.5.0; python_full_version >= '3.6.2'
-arrow==1.2.2; python_version >= '3.6'
-asgiref==3.5.1; python_version >= '3.7'
-async-timeout==4.0.2; python_version >= '3.6'
-attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-autobahn==22.3.2; python_version >= '3.7'
-automat==20.2.0
-backports.zoneinfo==0.2.1; python_version < '3.9'
-blessed==1.19.1; python_version >= '2.7'
-certifi==2021.10.8
-cffi==1.15.0
-channels-redis==3.4.0
-channels==3.0.4
-chardet==4.0.0; python_version >= '3.1'
-charset-normalizer==2.0.12; python_version >= '3'
-click==8.1.3; python_version >= '3.7'
-coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-concurrent-log-handler==0.9.20
-constantly==15.1.0
-cryptography==37.0.1; python_version >= '3.6'
-daphne==3.0.2; python_version >= '3.6'
-dateparser==1.1.1
-django-cors-headers==3.11.0
-django-extensions==3.1.5
-django-filter==21.1
-django-picklefield==3.0.1; python_version >= '3'
-django-q==1.3.9
-django==4.0.4
-djangorestframework==3.13.1
-filelock==3.6.0
-fuzzywuzzy[speedup]==0.18.0
-gunicorn==20.1.0
-h11==0.13.0; python_version >= '3.6'
-hiredis==2.0.0; python_version >= '3.6'
-httptools==0.4.0
-humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-hyperlink==21.0.0
-idna==3.3; python_version >= '3'
-imap-tools==0.54.0
-img2pdf==0.4.4
-importlib-resources==5.7.1; python_version < '3.9'
-incremental==21.3.0
-inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-inotifyrecursive==0.3.5
-joblib==1.1.0; python_version >= '3.6'
-langdetect==1.0.9
-lxml==4.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-msgpack==1.0.3
-numpy==1.22.3; python_version >= '3.8'
-ocrmypdf==13.4.3
-packaging==21.3; python_version >= '3.6'
-pathvalidate==2.5.0
-pdf2image==1.16.0
-pdfminer.six==20220319
-pikepdf==5.1.2
-pillow==9.1.0
-pluggy==1.0.0; python_version >= '3.6'
-portalocker==2.4.0; python_version >= '3'
-psycopg2==2.9.3
-pyasn1-modules==0.2.8
-pyasn1==0.4.8
-pycparser==2.21
-pyopenssl==22.0.0
-pyparsing==3.0.8; python_full_version >= '3.6.8'
-python-dateutil==2.8.2
-python-dotenv==0.20.0
-python-gnupg==0.4.8
-python-levenshtein==0.12.2
-python-magic==0.4.25
-pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-pytz==2022.1
-pyyaml==6.0
-pyzbar==0.1.9
-redis==3.5.3
-regex==2022.3.2; python_version >= '3.6'
-reportlab==3.6.9; python_version >= '3.7' and python_version < '4'
-requests==2.27.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-scikit-learn==1.0.2
-scipy==1.8.0; python_version < '3.11' and python_version >= '3.8'
-service-identity==21.1.0
-setuptools==62.1.0; python_version >= '3.7'
-six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-sniffio==1.2.0; python_version >= '3.5'
-sqlparse==0.4.2; python_version >= '3.5'
-threadpoolctl==3.1.0; python_version >= '3.6'
-tika==1.24
-tqdm==4.64.0
-twisted[tls]==22.4.0; python_full_version >= '3.6.7'
-txaio==22.2.1; python_version >= '3.6'
-typing-extensions==4.2.0; python_version >= '3.7'
-tzdata==2022.1; python_version >= '3.6'
-tzlocal==4.2; python_version >= '3.6'
-urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-uvicorn[standard]==0.17.6
-uvloop==0.16.0
-watchdog==2.1.7
-watchgod==0.8.2
-wcwidth==0.2.5
-websockets==10.3
-whitenoise==6.0.0
-whoosh==2.7.4
-zipp==3.8.0; python_version < '3.9'
-zope.interface==5.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

scripts/paperless-scheduler.service

@@ -1,12 +1,12 @@
 [Unit]
-Description=Paperless scheduler
+Description=Paperless Celery Beat
 Requires=redis.service
 
 [Service]
 User=paperless
 Group=paperless
 WorkingDirectory=/opt/paperless/src
-ExecStart=python3 manage.py qcluster
+ExecStart=celery --app paperless beat --loglevel INFO
 
 [Install]
 WantedBy=multi-user.target

scripts/paperless-task-queue.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Paperless Celery Workers
+Requires=redis.service
+
+[Service]
+User=paperless
+Group=paperless
+WorkingDirectory=/opt/paperless/src
+ExecStart=celery --app paperless worker --loglevel INFO
+
+[Install]
+WantedBy=multi-user.target

scripts/post-consumption-example.sh

@@ -1,21 +1,16 @@
 #!/usr/bin/env bash
 
-DOCUMENT_ID=${1}
-DOCUMENT_FILE_NAME=${2}
-DOCUMENT_SOURCE_PATH=${3}
-DOCUMENT_THUMBNAIL_PATH=${4}
-DOCUMENT_DOWNLOAD_URL=${5}
-DOCUMENT_THUMBNAIL_URL=${6}
-DOCUMENT_CORRESPONDENT=${7}
-DOCUMENT_TAGS=${8}
-
 echo "
 
 A document with an id of ${DOCUMENT_ID} was just consumed.  I know the
 following additional information about it:
 
 * Generated File Name: ${DOCUMENT_FILE_NAME}
+* Archive Path: ${DOCUMENT_ARCHIVE_PATH}
 * Source Path: ${DOCUMENT_SOURCE_PATH}
+* Created: ${DOCUMENT_CREATED}
+* Added: ${DOCUMENT_ADDED}
+* Modified: ${DOCUMENT_MODIFIED}
 * Thumbnail Path: ${DOCUMENT_THUMBNAIL_PATH}
 * Download URL: ${DOCUMENT_DOWNLOAD_URL}
 * Thumbnail URL: ${DOCUMENT_THUMBNAIL_URL}

scripts/start_services.sh

@@ -2,5 +2,5 @@
 
 docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
-docker run -p 3000:3000 -d gotenberg/gotenberg:7.4
+docker run -p 3000:3000 -d gotenberg/gotenberg:7.6
 docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/cypress.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from 'cypress'
+
+export default defineConfig({
+  videosFolder: 'cypress/videos',
+  screenshotsFolder: 'cypress/screenshots',
+  fixturesFolder: 'cypress/fixtures',
+  e2e: {
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.ts')(on, config)
+    },
+    baseUrl: 'http://localhost:4200',
+  },
+})

src-ui/cypress.json

@@ -1,9 +0,0 @@
-{
-  "integrationFolder": "cypress/integration",
-  "supportFile": "cypress/support/index.ts",
-  "videosFolder": "cypress/videos",
-  "screenshotsFolder": "cypress/screenshots",
-  "pluginsFile": "cypress/plugins/index.ts",
-  "fixturesFolder": "cypress/fixtures",
-  "baseUrl": "http://localhost:4200"
-}

src-ui/cypress/e2e/documents/document-detail.cy.ts

@@ -0,0 +1,94 @@
+describe('document-detail', () => {
+  beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
+    this.modifiedDocuments = []
+
+    cy.fixture('documents/documents.json').then((documentsJson) => {
+      cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
+        let response = { ...documentsJson }
+        response = response.results.find((d) => d.id == 1)
+        req.reply(response)
+      })
+    })
+
+    cy.intercept('PUT', 'http://localhost:8000/api/documents/1/', (req) => {
+      this.modifiedDocuments.push(req.body) // store this for later
+      req.reply({ result: 'OK' })
+    }).as('saveDoc')
+
+    cy.fixture('documents/1/comments.json').then((commentsJson) => {
+      cy.intercept(
+        'GET',
+        'http://localhost:8000/api/documents/1/comments/',
+        (req) => {
+          req.reply(commentsJson.filter((c) => c.id != 10)) // 3
+        }
+      )
+
+      cy.intercept(
+        'DELETE',
+        'http://localhost:8000/api/documents/1/comments/?id=9',
+        (req) => {
+          req.reply(commentsJson.filter((c) => c.id != 9 && c.id != 10)) // 2
+        }
+      )
+
+      cy.intercept(
+        'POST',
+        'http://localhost:8000/api/documents/1/comments/',
+        (req) => {
+          req.reply(commentsJson) // 4
+        }
+      )
+    })
+
+    cy.viewport(1024, 1024)
+    cy.visit('/documents/1/')
+  })
+
+  it('should activate / deactivate save button when changes are saved', () => {
+    cy.contains('button', 'Save').should('be.disabled')
+    cy.get('app-input-text[formcontrolname="title"]')
+      .type(' additional')
+      .wait(1500) // this delay is for frontend debounce
+    cy.contains('button', 'Save').should('not.be.disabled')
+  })
+
+  it('should warn on unsaved changes', () => {
+    cy.get('app-input-text[formcontrolname="title"]')
+      .type(' additional')
+      .wait(1500) // this delay is for frontend debounce
+    cy.get('button[title="Close"]').click()
+    cy.contains('You have unsaved changes')
+    cy.contains('button', 'Cancel').click().wait(150)
+    cy.contains('button', 'Save').click().wait('@saveDoc').wait(2000) // navigates away after saving
+    cy.contains('You have unsaved changes').should('not.exist')
+  })
+
+  it('should show a list of comments', () => {
+    cy.wait(1000).get('a').contains('Comments').click().wait(1000)
+    cy.get('app-document-comments').find('.card').its('length').should('eq', 3)
+  })
+
+  it('should support comment deletion', () => {
+    cy.wait(1000).get('a').contains('Comments').click().wait(1000)
+    cy.get('app-document-comments')
+      .find('.card')
+      .first()
+      .find('button')
+      .click({ force: true })
+      .wait(500)
+    cy.get('app-document-comments').find('.card').its('length').should('eq', 2)
+  })
+
+  it('should support comment insertion', () => {
+    cy.wait(1000).get('a').contains('Comments').click().wait(1000)
+    cy.get('app-document-comments')
+      .find('form textarea')
+      .type('Testing new comment')
+      .wait(500)
+    cy.get('app-document-comments').find('form button').click().wait(1500)
+    cy.get('app-document-comments').find('.card').its('length').should('eq', 4)
+  })
+})

src-ui/cypress/e2e/documents/documents-list.cy.ts

@@ -1,8 +1,9 @@
 describe('documents-list', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.bulkEdits = {}
 
-    // mock API methods
     cy.fixture('documents/documents.json').then((documentsJson) => {
       // bulk edit
       cy.intercept(
@@ -53,40 +54,25 @@ describe('documents-list', () => {
       })
     })
 
-    cy.intercept('http://localhost:8000/api/documents/1/thumb/', {
-      fixture: 'documents/lorem-ipsum.png',
-    })
-
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/document_types/*', {
-      fixture: 'document_types/doctypes.json',
-    })
-
+    cy.viewport(1280, 1024)
     cy.visit('/documents')
   })
 
   it('should show a list of documents rendered as cards with thumbnails', () => {
     cy.contains('3 documents')
-    cy.contains('lorem-ipsum')
+    cy.contains('lorem ipsum')
     cy.get('app-document-card-small:first-of-type img')
       .invoke('attr', 'src')
       .should('eq', 'http://localhost:8000/api/documents/1/thumb/')
   })
 
   it('should change to table "details" view', () => {
-    cy.get('div.btn-group-toggle input[value="details"]').parent().click()
+    cy.get('div.btn-group input[value="details"]').next().click()
     cy.get('table')
   })
 
   it('should change to large cards view', () => {
-    cy.get('div.btn-group-toggle input[value="largeCards"]').parent().click()
+    cy.get('div.btn-group input[value="largeCards"]').next().click()
     cy.get('app-document-card-large')
   })
 

src-ui/cypress/e2e/documents/query-params.cy.ts

@@ -0,0 +1,331 @@
+import { PaperlessDocument } from 'src/app/data/paperless-document'
+
+describe('documents query params', () => {
+  beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
+    cy.fixture('documents/documents.json').then((documentsJson) => {
+      // mock api filtering
+      cy.intercept('GET', 'http://localhost:8000/api/documents/*', (req) => {
+        let response = { ...documentsJson }
+
+        if (req.query.hasOwnProperty('ordering')) {
+          const sort_field = req.query['ordering'].toString().replace('-', '')
+          const reverse = req.query['ordering'].toString().indexOf('-') !== -1
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).sort((docA, docB) => {
+            let result = 0
+            switch (sort_field) {
+              case 'created':
+              case 'added':
+                result =
+                  new Date(docA[sort_field]) < new Date(docB[sort_field])
+                    ? -1
+                    : 1
+                break
+              case 'archive_serial_number':
+                result = docA[sort_field] < docB[sort_field] ? -1 : 1
+                break
+            }
+            if (reverse) result = -result
+            return result
+          })
+        }
+
+        if (req.query.hasOwnProperty('tags__id__in')) {
+          const tag_ids: Array<number> = req.query['tags__id__in']
+            .toString()
+            .split(',')
+            .map((v) => +v)
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) =>
+              d.tags.length > 0 &&
+              d.tags.filter((t) => tag_ids.includes(t)).length > 0
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('tags__id__none')) {
+          const tag_ids: Array<number> = req.query['tags__id__none']
+            .toString()
+            .split(',')
+            .map((v) => +v)
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.tags.filter((t) => tag_ids.includes(t)).length == 0)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('is_tagged') &&
+          req.query['is_tagged'] == '0'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.tags.length == 0)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('document_type__id')) {
+          const doctype_id = +req.query['document_type__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.document_type == doctype_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('document_type__isnull') &&
+          req.query['document_type__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.document_type == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('correspondent__id')) {
+          const correspondent_id = +req.query['correspondent__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.correspondent == correspondent_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('correspondent__isnull') &&
+          req.query['correspondent__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.correspondent == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('storage_path__id')) {
+          const storage_path_id = +req.query['storage_path__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.storage_path == storage_path_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('storage_path__isnull') &&
+          req.query['storage_path__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.storage_path == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('created__date__gt')) {
+          const date = new Date(req.query['created__date__gt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.created) > date)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('created__date__lt')) {
+          const date = new Date(req.query['created__date__lt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.created) < date)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('added__date__gt')) {
+          const date = new Date(req.query['added__date__gt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.added) > date)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('added__date__lt')) {
+          const date = new Date(req.query['added__date__lt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.added) < date)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('title_content')) {
+          const title_content_regexp = new RegExp(
+            req.query['title_content'].toString(),
+            'i'
+          )
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) =>
+              title_content_regexp.test(d.title) ||
+              title_content_regexp.test(d.content)
+          )
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('archive_serial_number')) {
+          const asn = +req.query['archive_serial_number']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.archive_serial_number == asn)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__isnull')) {
+          const isnull = req.query['storage_path__isnull'] == '1'
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) =>
+            isnull
+              ? d.archive_serial_number == undefined
+              : d.archive_serial_number != undefined
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__gt')) {
+          const asn = +req.query['archive_serial_number__gt']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) => d.archive_serial_number > 0 && d.archive_serial_number > asn
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__lt')) {
+          const asn = +req.query['archive_serial_number__lt']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) => d.archive_serial_number > 0 && d.archive_serial_number < asn
+          )
+          response.count = response.results.length
+        }
+
+        req.reply(response)
+      })
+    })
+  })
+
+  it('should show a list of documents sorted by created', () => {
+    cy.visit('/documents?sort=created')
+    cy.get('app-document-card-small').first().contains('No latin title')
+  })
+
+  it('should show a list of documents reverse sorted by created', () => {
+    cy.visit('/documents?sort=created&reverse=true')
+    cy.get('app-document-card-small').first().contains('sit amet')
+  })
+
+  it('should show a list of documents sorted by added', () => {
+    cy.visit('/documents?sort=added')
+    cy.get('app-document-card-small').first().contains('No latin title')
+  })
+
+  it('should show a list of documents reverse sorted by added', () => {
+    cy.visit('/documents?sort=added&reverse=true')
+    cy.get('app-document-card-small').first().contains('sit amet')
+  })
+
+  it('should show a list of documents filtered by any tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&tags__id__in=2,4,5')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by excluded tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&tags__id__none=2,4')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by no tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&is_tagged=0')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by document type', () => {
+    cy.visit('/documents?sort=created&reverse=true&document_type__id=1')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by no document type', () => {
+    cy.visit('/documents?sort=created&reverse=true&document_type__isnull=1')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by correspondent', () => {
+    cy.visit('/documents?sort=created&reverse=true&correspondent__id=9')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by no correspondent', () => {
+    cy.visit('/documents?sort=created&reverse=true&correspondent__isnull=1')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by storage path', () => {
+    cy.visit('/documents?sort=created&reverse=true&storage_path__id=2')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by no storage path', () => {
+    cy.visit('/documents?sort=created&reverse=true&storage_path__isnull=1')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by title or content', () => {
+    cy.visit('/documents?sort=created&reverse=true&title_content=lorem')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by asn', () => {
+    cy.visit('/documents?sort=created&reverse=true&archive_serial_number=12345')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by empty asn', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__isnull=1'
+    )
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by non-empty asn', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__isnull=0'
+    )
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by asn greater than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__gt=12346'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by asn less than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__lt=12346'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by created date greater than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&created__date__gt=2022-03-23'
+    )
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by created date less than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&created__date__lt=2022-03-23'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by added date greater than', () => {
+    cy.visit('/documents?sort=created&reverse=true&added__date__gt=2022-03-24')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by added date less than', () => {
+    cy.visit('/documents?sort=created&reverse=true&added__date__lt=2022-03-24')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by multiple filters', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&document_type__id=1&correspondent__id=9&tags__id__in=4,5'
+    )
+    cy.contains('2 documents')
+  })
+})

src-ui/cypress/e2e/manage/manage.cy.ts

@@ -1,12 +1,5 @@
 describe('manage', () => {
-  beforeEach(() => {
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-  })
+  // also uses global fixtures from cypress/support/e2e.ts
 
   it('should show a list of correspondents with bottom pagination as well', () => {
     cy.visit('/correspondents')

src-ui/cypress/e2e/settings/settings.cy.ts

@@ -1,50 +1,52 @@
 describe('settings', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.modifiedViews = []
 
     // mock API methods
-    cy.fixture('saved_views/savedviews.json').then((savedViewsJson) => {
-      // saved views PATCH
-      cy.intercept(
-        'PATCH',
-        'http://localhost:8000/api/saved_views/*',
-        (req) => {
-          this.modifiedViews.push(req.body) // store this for later
-          req.reply({ result: 'OK' })
-        }
-      )
+    cy.intercept('http://localhost:8000/api/ui_settings/', {
+      fixture: 'ui_settings/settings.json',
+    }).then(() => {
+      cy.fixture('saved_views/savedviews.json').then((savedViewsJson) => {
+        // saved views PATCH
+        cy.intercept(
+          'PATCH',
+          'http://localhost:8000/api/saved_views/*',
+          (req) => {
+            this.modifiedViews.push(req.body) // store this for later
+            req.reply({ result: 'OK' })
+          }
+        )
 
-      cy.intercept('GET', 'http://localhost:8000/api/saved_views/*', (req) => {
-        let response = { ...savedViewsJson }
-        if (this.modifiedViews.length) {
-          response.results = response.results.map((v) => {
-            if (this.modifiedViews.find((mv) => mv.id == v.id))
-              v = this.modifiedViews.find((mv) => mv.id == v.id)
-            return v
-          })
-        }
+        cy.intercept(
+          'GET',
+          'http://localhost:8000/api/saved_views/*',
+          (req) => {
+            let response = { ...savedViewsJson }
+            if (this.modifiedViews.length) {
+              response.results = response.results.map((v) => {
+                if (this.modifiedViews.find((mv) => mv.id == v.id))
+                  v = this.modifiedViews.find((mv) => mv.id == v.id)
+                return v
+              })
+            }
 
-        req.reply(response)
-      }).as('savedViews')
-    })
+            req.reply(response)
+          }
+        ).as('savedViews')
+      })
 
-    cy.fixture('documents/documents.json').then((documentsJson) => {
-      cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
-        let response = { ...documentsJson }
-        response = response.results.find((d) => d.id == 1)
-        req.reply(response)
+      cy.fixture('documents/documents.json').then((documentsJson) => {
+        cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
+          let response = { ...documentsJson }
+          response = response.results.find((d) => d.id == 1)
+          req.reply(response)
+        })
       })
     })
 
-    cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
-      fixture: 'documents/1/metadata.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
-      fixture: 'documents/1/suggestions.json',
-    })
-
-    cy.viewport(1024, 1024)
+    cy.viewport(1024, 1600)
     cy.visit('/settings')
     cy.wait('@savedViews')
   })

src-ui/cypress/e2e/tasks/tasks.cy.ts

@@ -0,0 +1,60 @@
+describe('tasks', () => {
+  beforeEach(() => {
+    this.dismissedTasks = new Set<number>()
+
+    cy.fixture('tasks/tasks.json').then((tasksViewsJson) => {
+      // acknowledge tasks POST
+      cy.intercept(
+        'POST',
+        'http://localhost:8000/api/acknowledge_tasks/',
+        (req) => {
+          req.body['tasks'].forEach((t) => this.dismissedTasks.add(t)) // store this for later
+          req.reply({ result: 'OK' })
+        }
+      )
+
+      cy.intercept('GET', 'http://localhost:8000/api/tasks/', (req) => {
+        let response = [...tasksViewsJson]
+        if (this.dismissedTasks.size) {
+          response = response.filter((t) => {
+            return !this.dismissedTasks.has(t.id)
+          })
+        }
+
+        req.reply(response)
+      }).as('tasks')
+    })
+
+    cy.visit('/tasks')
+    cy.wait('@tasks')
+  })
+
+  it('should show a list of dismissable tasks in tabs', () => {
+    cy.get('tbody').find('tr:visible').its('length').should('eq', 10) // double because collapsible result tr
+    cy.wait(500) // stabilizes the test, for some reason...
+    cy.get('tbody')
+      .find('button:visible')
+      .contains('Dismiss')
+      .first()
+      .click()
+      .wait('@tasks')
+      .wait(2000)
+      .then(() => {
+        cy.get('tbody').find('tr:visible').its('length').should('eq', 8) // double because collapsible result tr
+      })
+  })
+
+  it('should allow toggling all tasks in list and warn on dismiss', () => {
+    cy.get('thead').find('input[type="checkbox"]').first().click()
+    cy.get('body').find('button').contains('Dismiss selected').first().click()
+    cy.contains('Confirm')
+    cy.get('.modal')
+      .contains('button', 'Dismiss')
+      .click()
+      .wait('@tasks')
+      .wait(2000)
+      .then(() => {
+        cy.get('tbody').find('tr:visible').should('not.exist')
+      })
+  })
+})

src-ui/cypress/fixtures/documents/1/comments.json

@@ -0,0 +1,46 @@
+[
+    {
+        "id": 10,
+        "comment": "Testing new comment",
+        "created": "2022-08-08T04:24:55.176008Z",
+        "user": {
+            "id": 1,
+            "username": "user2",
+            "firstname": "",
+            "lastname": ""
+        }
+    },
+    {
+        "id": 9,
+        "comment": "Testing one more time",
+        "created": "2022-02-18T04:24:55.176008Z",
+        "user": {
+            "id": 2,
+            "username": "user1",
+            "firstname": "",
+            "lastname": ""
+        }
+    },
+    {
+        "id": 8,
+        "comment": "Another comment",
+        "created": "2021-11-08T04:24:47.925042Z",
+        "user": {
+            "id": 2,
+            "username": "user33",
+            "firstname": "",
+            "lastname": ""
+        }
+    },
+    {
+        "id": 7,
+        "comment": "Cupcake ipsum dolor sit amet cheesecake candy cookie tiramisu. Donut chocolate chupa chups macaroon brownie halvah pie cheesecake gummies. Sweet chocolate bar candy donut gummi bears bear claw liquorice bonbon shortbread.\n\nDonut chocolate bar candy wafer wafer tiramisu. Gummies chocolate cake muffin toffee carrot cake macaroon. Toffee toffee jelly beans danish lollipop cake.",
+        "created": "2021-02-08T02:37:49.724132Z",
+        "user": {
+            "id": 3,
+            "username": "admin",
+            "firstname": "",
+            "lastname": ""
+        }
+    }
+]

src-ui/cypress/fixtures/remote_version/remote_version.json

@@ -0,0 +1 @@
+{"version":"v1.7.1","update_available":false,"feature_is_set":true}

src-ui/cypress/fixtures/storage_paths/storage_paths.json

@@ -0,0 +1,17 @@
+{
+    "count": 1,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 2,
+            "slug": "year-title",
+            "name": "Year - Title",
+            "path": "{created_year}/{title}",
+            "match": "",
+            "matching_algorithm": 6,
+            "is_insensitive": true,
+            "document_count": 1
+        }
+    ]
+}

src-ui/cypress/fixtures/tags/tags.json

@@ -1 +1,103 @@
-{"count":8,"next":null,"previous":null,"results":[{"id":4,"slug":"another-sample-tag","name":"Another Sample Tag","color":"#a6cee3","text_color":"#000000","match":"","matching_algorithm":6,"is_insensitive":true,"is_inbox_tag":false,"document_count":3},{"id":7,"slug":"newone","name":"NewOne","color":"#9e4ad1","text_color":"#ffffff","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":2},{"id":6,"slug":"partial-tag","name":"Partial Tag","color":"#72dba7","text_color":"#000000","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":1},{"id":2,"slug":"tag-2","name":"Tag 2","color":"#612db7","text_color":"#ffffff","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":3},{"id":3,"slug":"tag-3","name":"Tag 3","color":"#b2df8a","text_color":"#000000","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":4},{"id":5,"slug":"tagwithpartial","name":"TagWithPartial","color":"#3b2db4","text_color":"#ffffff","match":"","matching_algorithm":6,"is_insensitive":true,"is_inbox_tag":false,"document_count":2},{"id":8,"slug":"test-another","name":"Test Another","color":"#3ccea5","text_color":"#000000","match":"","matching_algorithm":4,"is_insensitive":true,"is_inbox_tag":false,"document_count":0},{"id":1,"slug":"test-tag","name":"Test Tag","color":"#fb9a99","text_color":"#000000","match":"","matching_algorithm":1,"is_insensitive":true,"is_inbox_tag":false,"document_count":4}]}
+{
+    "count": 8,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 4,
+            "slug": "another-sample-tag",
+            "name": "Another Sample Tag",
+            "color": "#a6cee3",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 6,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 3
+        },
+        {
+            "id": 7,
+            "slug": "newone",
+            "name": "NewOne",
+            "color": "#9e4ad1",
+            "text_color": "#ffffff",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 2
+        },
+        {
+            "id": 6,
+            "slug": "partial-tag",
+            "name": "Partial Tag",
+            "color": "#72dba7",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 1
+        },
+        {
+            "id": 2,
+            "slug": "tag-2",
+            "name": "Tag 2",
+            "color": "#612db7",
+            "text_color": "#ffffff",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 3
+        },
+        {
+            "id": 3,
+            "slug": "tag-3",
+            "name": "Tag 3",
+            "color": "#b2df8a",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 4
+        },
+        {
+            "id": 5,
+            "slug": "tagwithpartial",
+            "name": "TagWithPartial",
+            "color": "#3b2db4",
+            "text_color": "#ffffff",
+            "match": "",
+            "matching_algorithm": 6,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 2
+        },
+        {
+            "id": 8,
+            "slug": "test-another",
+            "name": "Test Another",
+            "color": "#3ccea5",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 4,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 0
+        },
+        {
+            "id": 1,
+            "slug": "test-tag",
+            "name": "Test Tag",
+            "color": "#fb9a99",
+            "text_color": "#000000",
+            "match": "",
+            "matching_algorithm": 1,
+            "is_insensitive": true,
+            "is_inbox_tag": false,
+            "document_count": 4
+        }
+    ]
+}

src-ui/cypress/fixtures/ui_settings/settings.json

@@ -0,0 +1,34 @@
+{
+    "user_id": 1,
+    "username": "admin",
+    "display_name": "Admin",
+    "settings": {
+        "language": "",
+        "bulk_edit": {
+            "confirmation_dialogs": true,
+            "apply_on_close": false
+        },
+        "documentListSize": 50,
+        "dark_mode": {
+            "use_system": true,
+            "enabled": "false",
+            "thumb_inverted": "true"
+        },
+        "theme": {
+            "color": "#b198e5"
+        },
+        "document_details": {
+            "native_pdf_viewer": false
+        },
+        "date_display": {
+            "date_locale": "",
+            "date_format": "mediumDate"
+        },
+        "notifications": {
+            "consumer_new_documents": true,
+            "consumer_success": true,
+            "consumer_failed": true,
+            "consumer_suppress_on_dashboard": true
+        }
+    }
+}

src-ui/cypress/integration/document-detail.spec.ts

@@ -1,64 +0,0 @@
-describe('document-detail', () => {
-  beforeEach(() => {
-    this.modifiedDocuments = []
-
-    cy.fixture('documents/documents.json').then((documentsJson) => {
-      cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
-        let response = { ...documentsJson }
-        response = response.results.find((d) => d.id == 1)
-        req.reply(response)
-      })
-    })
-
-    cy.intercept('PUT', 'http://localhost:8000/api/documents/1/', (req) => {
-      this.modifiedDocuments.push(req.body) // store this for later
-      req.reply({ result: 'OK' })
-    }).as('saveDoc')
-
-    cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
-      fixture: 'documents/1/metadata.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
-      fixture: 'documents/1/suggestions.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/saved_views/*', {
-      fixture: 'saved_views/savedviews.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/document_types/*', {
-      fixture: 'document_types/doctypes.json',
-    })
-
-    cy.viewport(1024, 1024)
-    cy.visit('/documents/1/')
-  })
-
-  it('should activate / deactivate save button when changes are saved', () => {
-    cy.contains('button', 'Save').should('be.disabled')
-    cy.get('app-input-text[formcontrolname="title"]')
-      .type(' additional')
-      .wait(1500) // this delay is for frontend debounce
-    cy.contains('button', 'Save').should('not.be.disabled')
-  })
-
-  it('should warn on unsaved changes', () => {
-    cy.get('app-input-text[formcontrolname="title"]')
-      .type(' additional')
-      .wait(1500) // this delay is for frontend debounce
-    cy.get('button[title="Close"]').click()
-    cy.contains('You have unsaved changes')
-    cy.contains('button', 'Cancel').click().wait(150)
-    cy.contains('button', 'Save').click().wait('@saveDoc').wait(2000) // navigates away after saving
-    cy.contains('You have unsaved changes').should('not.exist')
-  })
-})

src-ui/cypress/support/e2e.ts

@@ -0,0 +1,43 @@
+// mock API methods
+
+beforeEach(() => {
+  cy.intercept('http://localhost:8000/api/ui_settings/', {
+    fixture: 'ui_settings/settings.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/remote_version/', {
+    fixture: 'remote_version/remote_version.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/saved_views/*', {
+    fixture: 'saved_views/savedviews.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/tags/*', {
+    fixture: 'tags/tags.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/correspondents/*', {
+    fixture: 'correspondents/correspondents.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/document_types/*', {
+    fixture: 'document_types/doctypes.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/storage_paths/*', {
+    fixture: 'storage_paths/storage_paths.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
+    fixture: 'documents/1/metadata.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
+    fixture: 'documents/1/suggestions.json',
+  })
+
+  cy.intercept('http://localhost:8000/api/documents/1/thumb/', {
+    fixture: 'documents/lorem-ipsum.png',
+  })
+})

src-ui/cypress/support/index.ts

@@ -1,17 +0,0 @@
-// ***********************************************************
-// This example support/index.js is processed and
-// loaded automatically before your test files.
-//
-// This is a great place to put global configuration and
-// behavior that modifies Cypress.
-//
-// You can change the location of this file or turn off
-// automatically serving support files with the
-// 'supportFile' configuration option.
-//
-// You can read more here:
-// https://on.cypress.io/configuration
-// ***********************************************************
-
-// When a command from ./commands is ready to use, import with `import './commands'` syntax
-// import './commands';

src-ui/package.json

@@ -13,48 +13,49 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/common": "~13.3.5",
-    "@angular/compiler": "~13.3.5",
-    "@angular/core": "~13.3.5",
-    "@angular/forms": "~13.3.5",
-    "@angular/localize": "~13.3.5",
-    "@angular/platform-browser": "~13.3.5",
-    "@angular/platform-browser-dynamic": "~13.3.5",
-    "@angular/router": "~13.3.5",
-    "@ng-bootstrap/ng-bootstrap": "^12.1.1",
-    "@ng-select/ng-select": "^8.1.1",
+    "@angular/common": "~14.2.8",
+    "@angular/compiler": "~14.2.8",
+    "@angular/core": "~14.2.8",
+    "@angular/forms": "~14.2.8",
+    "@angular/localize": "~14.2.8",
+    "@angular/platform-browser": "~14.2.8",
+    "@angular/platform-browser-dynamic": "~14.2.8",
+    "@angular/router": "~14.2.8",
+    "@ng-bootstrap/ng-bootstrap": "^13.0.0",
+    "@ng-select/ng-select": "^9.0.2",
     "@ngneat/dirty-check-forms": "^3.0.2",
-    "@popperjs/core": "^2.11.4",
-    "bootstrap": "^5.1.3",
+    "@popperjs/core": "^2.11.6",
+    "bootstrap": "^5.2.1",
     "file-saver": "^2.0.5",
-    "ng2-pdf-viewer": "^9.0.0",
-    "ngx-color": "^7.3.3",
-    "ngx-cookie-service": "^13.1.2",
-    "ngx-file-drop": "^13.0.0",
-    "rxjs": "~7.5.5",
+    "ng2-pdf-viewer": "^9.1.2",
+    "ngx-color": "^8.0.3",
+    "ngx-cookie-service": "^14.0.1",
+    "ngx-file-drop": "^14.0.1",
+    "ngx-ui-tour-ng-bootstrap": "^11.1.0",
+    "rxjs": "~7.5.7",
     "tslib": "^2.3.1",
-    "uuid": "^8.3.1",
-    "zone.js": "~0.11.4"
+    "uuid": "^9.0.0",
+    "zone.js": "~0.11.8"
   },
   "devDependencies": {
-    "@angular-builders/jest": "13.0.3",
-    "@angular-devkit/build-angular": "~13.3.4",
-    "@angular/cli": "~13.3.4",
-    "@angular/compiler-cli": "~13.3.5",
-    "@types/jest": "27.4.1",
-    "@types/node": "^17.0.30",
+    "@angular-builders/jest": "14.0.1",
+    "@angular-devkit/build-angular": "~14.2.7",
+    "@angular/cli": "~14.2.7",
+    "@angular/compiler-cli": "~14.2.8",
+    "@types/jest": "28.1.6",
+    "@types/node": "^18.7.23",
     "codelyzer": "^6.0.2",
-    "concurrently": "7.1.0",
-    "jest": "28.0.3",
-    "jest-environment-jsdom": "^28.0.2",
-    "jest-preset-angular": "^12.0.0-next.1",
-    "ts-node": "~10.7.0",
+    "concurrently": "7.4.0",
+    "jest": "28.1.3",
+    "jest-environment-jsdom": "^29.2.2",
+    "jest-preset-angular": "^12.2.2",
+    "ts-node": "~10.9.1",
     "tslint": "~6.1.3",
-    "typescript": "~4.6.3",
+    "typescript": "~4.8.4",
     "wait-on": "~6.0.1"
   },
   "optionalDependencies": {
-    "@cypress/schematic": "^1.6.0",
-    "cypress": "~9.6.0"
+    "@cypress/schematic": "^2.1.1",
+    "cypress": "~10.9.0"
   }
 }

src-ui/src/app/app-routing.module.ts

@@ -12,27 +12,42 @@ import { TagListComponent } from './components/manage/tag-list/tag-list.componen
 import { NotFoundComponent } from './components/not-found/not-found.component'
 import { DocumentAsnComponent } from './components/document-asn/document-asn.component'
 import { DirtyFormGuard } from './guards/dirty-form.guard'
+import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component'
+import { TasksComponent } from './components/manage/tasks/tasks.component'
+import { DirtyDocGuard } from './guards/dirty-doc.guard'
+import { DirtySavedViewGuard } from './guards/dirty-saved-view.guard'
 
 const routes: Routes = [
   { path: '', redirectTo: 'dashboard', pathMatch: 'full' },
   {
     path: '',
     component: AppFrameComponent,
+    canDeactivate: [DirtyDocGuard],
     children: [
       { path: 'dashboard', component: DashboardComponent },
-      { path: 'documents', component: DocumentListComponent },
-      { path: 'view/:id', component: DocumentListComponent },
+      {
+        path: 'documents',
+        component: DocumentListComponent,
+        canDeactivate: [DirtySavedViewGuard],
+      },
+      {
+        path: 'view/:id',
+        component: DocumentListComponent,
+        canDeactivate: [DirtySavedViewGuard],
+      },
       { path: 'documents/:id', component: DocumentDetailComponent },
       { path: 'asn/:id', component: DocumentAsnComponent },
       { path: 'tags', component: TagListComponent },
       { path: 'documenttypes', component: DocumentTypeListComponent },
       { path: 'correspondents', component: CorrespondentListComponent },
+      { path: 'storagepaths', component: StoragePathListComponent },
       { path: 'logs', component: LogsComponent },
       {
         path: 'settings',
         component: SettingsComponent,
         canDeactivate: [DirtyFormGuard],
       },
+      { path: 'tasks', component: TasksComponent },
     ],
   },
 

src-ui/src/app/app.component.html

@@ -11,3 +11,28 @@
         </div>
     </ng-template>
 </ngx-file-drop>
+
+<tour-step-template>
+    <ng-template #tourStep let-step="step">
+        <p class="tour-step-content" [innerHTML]="step?.content"></p>
+        <hr/>
+        <div class="d-flex justify-content-between align-items-center">
+            <span class="badge bg-light text-dark">{{ tourService.steps?.indexOf(step) + 1 }} / {{ tourService.steps?.length }}</span>
+            <div class="tour-step-navigation btn-toolbar" role="toolbar" aria-label="Controls">
+                <div class="btn-group btn-group-sm me-2" role="group" aria-label="Dismiss">
+                    <button class="btn btn-outline-danger" (click)="tourService.end()">
+                        {{ step?.endBtnTitle }}
+                    </button>
+                </div>
+                <div class="btn-group btn-group-sm align-self-end" role="group" aria-label="Previous / Next">
+                    <button *ngIf="tourService.hasPrev(step)" class="btn btn-outline-primary" (click)="tourService.prev()">
+                        « {{ step?.prevBtnTitle }}
+                    </button>
+                    <button *ngIf="tourService.hasNext(step)" class="btn btn-outline-primary" (click)="tourService.next()">
+                        {{ step?.nextBtnTitle }} »
+                    </button>
+                </div>
+            </div>
+        </div>
+    </ng-template>
+</tour-step-template>

src-ui/src/app/app.component.ts

@@ -1,11 +1,14 @@
-import { SettingsService, SETTINGS_KEYS } from './services/settings.service'
-import { Component, OnDestroy, OnInit } from '@angular/core'
+import { SettingsService } from './services/settings.service'
+import { SETTINGS_KEYS } from './data/paperless-uisettings'
+import { Component, OnDestroy, OnInit, Renderer2 } from '@angular/core'
 import { Router } from '@angular/router'
 import { Subscription } from 'rxjs'
 import { ConsumerStatusService } from './services/consumer-status.service'
 import { ToastService } from './services/toast.service'
 import { NgxFileDropEntry } from 'ngx-file-drop'
 import { UploadDocumentsService } from './services/upload-documents.service'
+import { TasksService } from './services/tasks.service'
+import { TourService } from 'ngx-ui-tour-ng-bootstrap'
 
 @Component({
   selector: 'app-root',
@@ -26,7 +29,10 @@ export class AppComponent implements OnInit, OnDestroy {
     private consumerStatusService: ConsumerStatusService,
     private toastService: ToastService,
     private router: Router,
-    private uploadDocumentsService: UploadDocumentsService
+    private uploadDocumentsService: UploadDocumentsService,
+    private tasksService: TasksService,
+    public tourService: TourService,
+    private renderer: Renderer2
   ) {
     let anyWindow = window as any
     anyWindow.pdfWorkerSrc = 'assets/js/pdf.worker.min.js'
@@ -64,6 +70,7 @@ export class AppComponent implements OnInit, OnDestroy {
     this.successSubscription = this.consumerStatusService
       .onDocumentConsumptionFinished()
       .subscribe((status) => {
+        this.tasksService.reload()
         if (
           this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)
         ) {
@@ -82,6 +89,7 @@ export class AppComponent implements OnInit, OnDestroy {
     this.failedSubscription = this.consumerStatusService
       .onDocumentConsumptionFailed()
       .subscribe((status) => {
+        this.tasksService.reload()
         if (
           this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)
         ) {
@@ -94,6 +102,7 @@ export class AppComponent implements OnInit, OnDestroy {
     this.newDocumentSubscription = this.consumerStatusService
       .onDocumentDetected()
       .subscribe((status) => {
+        this.tasksService.reload()
         if (
           this.showNotification(
             SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT
@@ -106,6 +115,87 @@ export class AppComponent implements OnInit, OnDestroy {
           })
         }
       })
+
+    this.tourService.initialize([
+      {
+        anchorId: 'tour.dashboard',
+        content: $localize`The dashboard can be used to show saved views, such as an 'Inbox'. Those settings are found under Settings > Saved Views once you have created some.`,
+        route: '/dashboard',
+        enableBackdrop: true,
+        delayAfterNavigation: 500,
+      },
+      {
+        anchorId: 'tour.upload-widget',
+        content: $localize`Drag-and-drop documents here to start uploading or place them in the consume folder. You can also drag-and-drop documents anywhere on all other pages of the web app. Once you do, Paperless-ngx will start training its machine learning algorithms.`,
+        route: '/dashboard',
+        enableBackdrop: true,
+      },
+      {
+        anchorId: 'tour.documents',
+        content: $localize`The documents list shows all of your documents and allows for filtering as well as bulk-editing. There are three different view styles: list, small cards and large cards. A list of documents currently opened for editing is shown in the sidebar.`,
+        route: '/documents?sort=created&reverse=1&page=1',
+        delayAfterNavigation: 500,
+        placement: 'bottom',
+        enableBackdrop: true,
+        disableScrollToAnchor: true,
+      },
+      {
+        anchorId: 'tour.documents-filter-editor',
+        content: $localize`The filtering tools allow you to quickly find documents using various searches, dates, tags, etc.`,
+        route: '/documents?sort=created&reverse=1&page=1',
+        placement: 'bottom',
+        enableBackdrop: true,
+      },
+      {
+        anchorId: 'tour.documents-views',
+        content: $localize`Any combination of filters can be saved as a 'view' which can then be displayed on the dashboard and / or sidebar.`,
+        route: '/documents?sort=created&reverse=1&page=1',
+        enableBackdrop: true,
+      },
+      {
+        anchorId: 'tour.tags',
+        content: $localize`Tags, correspondents, document types and storage paths can all be managed using these pages. They can also be created from the document edit view.`,
+        route: '/tags',
+        enableBackdrop: true,
+      },
+      {
+        anchorId: 'tour.file-tasks',
+        content: $localize`File Tasks shows you documents that have been consumed, are waiting to be, or may have failed during the process.`,
+        route: '/tasks',
+        enableBackdrop: true,
+      },
+      {
+        anchorId: 'tour.settings',
+        content: $localize`Check out the settings for various tweaks to the web app or to toggle settings for saved views.`,
+        route: '/settings',
+        enableBackdrop: true,
+      },
+      {
+        anchorId: 'tour.admin',
+        content: $localize`The Admin area contains more advanced controls as well as the settings for automatic e-mail fetching.`,
+        enableBackdrop: true,
+      },
+      {
+        anchorId: 'tour.outro',
+        title: $localize`Thank you! 🙏`,
+        content:
+          $localize`There are <em>tons</em> more features and info we didn't cover here, but this should get you started. Check out the documentation or visit the project on GitHub to learn more or to report issues.` +
+          '<br/><br/>' +
+          $localize`Lastly, on behalf of every contributor to this community-supported project, thank you for using Paperless-ngx!`,
+        route: '/dashboard',
+      },
+    ])
+
+    this.tourService.start$.subscribe(() => {
+      this.renderer.addClass(document.body, 'tour-active')
+    })
+
+    this.tourService.end$.subscribe(() => {
+      // animation time
+      setTimeout(() => {
+        this.renderer.removeClass(document.body, 'tour-active')
+      }, 500)
+    })
   }
 
   public get dragDropEnabled(): boolean {

src-ui/src/app/app.module.ts

@@ -1,5 +1,5 @@
 import { BrowserModule } from '@angular/platform-browser'
-import { NgModule } from '@angular/core'
+import { APP_INITIALIZER, NgModule } from '@angular/core'
 import { AppRoutingModule } from './app-routing.module'
 import { AppComponent } from './app.component'
 import {
@@ -24,6 +24,7 @@ import { CorrespondentEditDialogComponent } from './components/common/edit-dialo
 import { TagEditDialogComponent } from './components/common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
 import { DocumentTypeEditDialogComponent } from './components/common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
 import { TagComponent } from './components/common/tag/tag.component'
+import { ClearableBadge } from './components/common/clearable-badge/clearable-badge.component'
 import { PageHeaderComponent } from './components/common/page-header/page-header.component'
 import { AppFrameComponent } from './components/app-frame/app-frame.component'
 import { ToastsComponent } from './components/common/toasts/toasts.component'
@@ -61,12 +62,20 @@ import { SafeUrlPipe } from './pipes/safeurl.pipe'
 import { SafeHtmlPipe } from './pipes/safehtml.pipe'
 import { CustomDatePipe } from './pipes/custom-date.pipe'
 import { DateComponent } from './components/common/input/date/date.component'
-import { ISODateTimeAdapter } from './utils/ngb-iso-date-time-adapter'
+import { ISODateAdapter } from './utils/ngb-iso-date-adapter'
 import { LocalizedDateParserFormatter } from './utils/ngb-date-parser-formatter'
 import { ApiVersionInterceptor } from './interceptors/api-version.interceptor'
 import { ColorSliderModule } from 'ngx-color/slider'
 import { ColorComponent } from './components/common/input/color/color.component'
 import { DocumentAsnComponent } from './components/document-asn/document-asn.component'
+import { DocumentCommentsComponent } from './components/document-comments/document-comments.component'
+import { DirtyDocGuard } from './guards/dirty-doc.guard'
+import { DirtySavedViewGuard } from './guards/dirty-saved-view.guard'
+import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component'
+import { StoragePathEditDialogComponent } from './components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
+import { SettingsService } from './services/settings.service'
+import { TasksComponent } from './components/manage/tasks/tasks.component'
+import { TourNgBootstrapModule } from 'ngx-ui-tour-ng-bootstrap'
 
 import localeBe from '@angular/common/locales/be'
 import localeCs from '@angular/common/locales/cs'
@@ -109,6 +118,12 @@ registerLocaleData(localeSv)
 registerLocaleData(localeTr)
 registerLocaleData(localeZh)
 
+function initializeApp(settings: SettingsService) {
+  return () => {
+    return settings.initializeSettings()
+  }
+}
+
 @NgModule({
   declarations: [
     AppComponent,
@@ -118,6 +133,7 @@ registerLocaleData(localeZh)
     TagListComponent,
     DocumentTypeListComponent,
     CorrespondentListComponent,
+    StoragePathListComponent,
     LogsComponent,
     SettingsComponent,
     NotFoundComponent,
@@ -125,7 +141,9 @@ registerLocaleData(localeZh)
     ConfirmDialogComponent,
     TagEditDialogComponent,
     DocumentTypeEditDialogComponent,
+    StoragePathEditDialogComponent,
     TagComponent,
+    ClearableBadge,
     PageHeaderComponent,
     AppFrameComponent,
     ToastsComponent,
@@ -160,6 +178,8 @@ registerLocaleData(localeZh)
     DateComponent,
     ColorComponent,
     DocumentAsnComponent,
+    DocumentCommentsComponent,
+    TasksComponent,
   ],
   imports: [
     BrowserModule,
@@ -172,8 +192,15 @@ registerLocaleData(localeZh)
     PdfViewerModule,
     NgSelectModule,
     ColorSliderModule,
+    TourNgBootstrapModule.forRoot(),
   ],
   providers: [
+    {
+      provide: APP_INITIALIZER,
+      useFactory: initializeApp,
+      deps: [SettingsService],
+      multi: true,
+    },
     DatePipe,
     CookieService,
     {
@@ -188,8 +215,10 @@ registerLocaleData(localeZh)
     },
     FilterPipe,
     DocumentTitlePipe,
-    { provide: NgbDateAdapter, useClass: ISODateTimeAdapter },
+    { provide: NgbDateAdapter, useClass: ISODateAdapter },
     { provide: NgbDateParserFormatter, useClass: LocalizedDateParserFormatter },
+    DirtyDocGuard,
+    DirtySavedViewGuard,
   ],
   bootstrap: [AppComponent],
 })

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -4,11 +4,11 @@
     (click)="isMenuCollapsed = !isMenuCollapsed">
     <span class="navbar-toggler-icon"></span>
   </button>
-  <a class="navbar-brand col-auto col-md-3 col-lg-2 me-0 px-3 py-3 order-sm-0" routerLink="/dashboard">
+  <a class="navbar-brand col-auto col-md-3 col-lg-2 me-0 px-3 py-3 order-sm-0" [ngClass]="slimSidebarEnabled ? 'slim' : 'col-auto col-md-3 col-lg-2'" routerLink="/dashboard" tourAnchor="tour.intro">
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 198.43 238.91" width="1em" class="me-2" fill="currentColor">
       <path d="M194.7,0C164.22,70.94,17.64,79.74,64.55,194.06c.58,1.47-10.85,17-18.47,29.9-1.76-6.45-3.81-13.48-3.52-14.07,38.11-45.14-27.26-70.65-30.78-107.58C-4.64,131.62-10.5,182.92,39,212.53c.3,0,2.64,11.14,3.81,16.71a58.55,58.55,0,0,0-2.93,6.45c-1.17,2.93,7.62,2.64,7.62,3.22.88-.29,21.7-36.93,22.28-37.23C187.67,174.72,208.48,68.6,194.7,0ZM134.61,74.75C79.5,124,70.12,160.64,71.88,178.53,53.41,134.85,107.64,86.77,134.61,74.75ZM28.2,145.11c10.55,9.67,28.14,39.28,13.19,56.57C44.91,193.77,46.08,175.89,28.2,145.11Z" transform="translate(0 0)"/>
     </svg>
-    <ng-container i18n="app title">Paperless-ngx</ng-container>
+    <span class="ms-2" [class.visually-hidden]="slimSidebarEnabled" i18n="app title">Paperless-ngx</span>
   </a>
   <div class="search-form-container flex-grow-1 py-2 pb-3 pb-sm-2 px-3 ps-md-4 me-sm-auto order-3 order-sm-1">
     <form (ngSubmit)="search()" class="form-inline flex-grow-1">
@@ -16,22 +16,27 @@
         <use xlink:href="assets/bootstrap-icons.svg#search"/>
       </svg>
       <input class="form-control form-control-sm" type="text" placeholder="Search documents" aria-label="Search"
-        [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (selectItem)="itemSelected($event)" i18n-placeholder>
+        [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (keyup)="searchFieldKeyup($event)" (selectItem)="itemSelected($event)" i18n-placeholder>
+      <button *ngIf="!searchFieldEmpty" class="btn btn-link btn-sm px-0 position-absolute top-0 end-0" (click)="resetSearchField()">
+        <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-x me-1" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+          <path fill-rule="evenodd" d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
+        </svg>
+      </button>
     </form>
   </div>
   <ul ngbNav class="order-sm-3">
     <li ngbDropdown class="nav-item dropdown">
-      <button class="btn text-light" id="userDropdown" ngbDropdownToggle>
-        <span *ngIf="displayName" class="navbar-text small me-2 text-light d-none d-sm-inline">
-          {{displayName}}
+      <button class="btn border-0" id="userDropdown" ngbDropdownToggle>
+        <span class="small me-2 d-none d-sm-inline">
+          {{this.settingsService.displayName}}
         </span>
         <svg width="1.3em" height="1.3em" fill="currentColor">
           <use xlink:href="assets/bootstrap-icons.svg#person-circle"/>
         </svg>
       </button>
       <div ngbDropdownMenu class="dropdown-menu-end shadow me-2" aria-labelledby="userDropdown">
-        <div *ngIf="displayName" class="d-sm-none">
-          <p class="small mb-0 px-3 text-muted" i18n>Logged in as {{displayName}}</p>
+        <div class="d-sm-none">
+          <p class="small mb-0 px-3 text-muted" i18n>Logged in as {{this.settingsService.displayName}}</p>
           <div class="dropdown-divider"></div>
         </div>
         <a ngbDropdownItem class="nav-link" routerLink="settings" (click)="closeMenu()">
@@ -51,47 +56,54 @@
 
 <div class="container-fluid">
   <div class="row">
-    <nav id="sidebarMenu" class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse" [ngbCollapse]="isMenuCollapsed">
+    <nav id="sidebarMenu" class="d-md-block bg-light sidebar collapse" [ngClass]="slimSidebarEnabled ? 'slim' : 'col-md-3 col-lg-2'" [class.animating]="slimSidebarAnimating" [ngbCollapse]="isMenuCollapsed">
+      <button class="btn btn-sm btn-dark sidebar-slim-toggler" (click)="toggleSlimSidebar()">
+        <svg class="sidebaricon-sm" fill="currentColor">
+          <use *ngIf="slimSidebarEnabled" xlink:href="assets/bootstrap-icons.svg#chevron-double-right"/>
+          <use *ngIf="!slimSidebarEnabled" xlink:href="assets/bootstrap-icons.svg#chevron-double-left"/>
+        </svg>
+      </button>
       <div class="sidebar-sticky pt-3 d-flex flex-column justify-space-around">
         <ul class="nav flex-column">
           <li class="nav-item">
-            <a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Dashboard" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#house"/>
-              </svg>&nbsp;<ng-container i18n>Dashboard</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Dashboard</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" routerLink="documents" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="documents" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Documents" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#files"/>
-              </svg>&nbsp;<ng-container i18n>Documents</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Documents</ng-container></span>
             </a>
           </li>
         </ul>
 
-        <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.sidebarViews.length > 0'>
-          <ng-container i18n>Saved views</ng-container>
+        <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.loading || savedViewService.sidebarViews.length > 0'>
+          <span i18n>Saved views</span>
+          <div *ngIf="savedViewService.loading" class="spinner-border spinner-border-sm fw-normal ms-2" role="status"></div>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item w-100" *ngFor="let view of savedViewService.sidebarViews">
-            <a class="nav-link text-truncate" routerLink="view/{{view.id}}" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" [class.text-truncate]="!slimSidebarEnabled" routerLink="view/{{view.id}}" routerLinkActive="active" (click)="closeMenu()" [ngbPopover]="view.name" [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#funnel"/>
-              </svg>&nbsp;{{view.name}}
+              </svg><span>&nbsp;{{view.name}}</span>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='openDocuments.length > 0'>
-          <ng-container i18n>Open documents</ng-container>
+          <span i18n>Open documents</span>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item w-100" *ngFor='let d of openDocuments'>
-            <a class="nav-link text-truncate" routerLink="documents/{{d.id}}" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" [class.text-truncate]="!slimSidebarEnabled" routerLink="documents/{{d.id}}" routerLinkActive="active" (click)="closeMenu()" [ngbPopover]="d.title | documentTitle" [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#file-text"/>
-              </svg>&nbsp;{{d.title | documentTitle}}
+              </svg><span>&nbsp;{{d.title | documentTitle}}</span>
               <span class="close" (click)="closeDocument(d); $event.preventDefault()">
                 <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="bi bi-x" viewBox="0 0 16 16">
                   <use xlink:href="assets/bootstrap-icons.svg#x"/>
@@ -100,81 +112,96 @@
             </a>
           </li>
           <li class="nav-item w-100" *ngIf="openDocuments.length >= 1">
-            <a class="nav-link text-truncate" [routerLink]="[]" (click)="closeAll()">
+            <a class="nav-link" [class.text-truncate]="!slimSidebarEnabled" [routerLink]="[]" (click)="closeAll()" ngbPopover="Close all" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#x"/>
-              </svg>&nbsp;<ng-container i18n>Close all</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Close all</ng-container></span>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
-          <ng-container i18n>Manage</ng-container>
+          <span i18n>Manage</span>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item">
-            <a class="nav-link" routerLink="correspondents" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="correspondents" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Correspondents" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#person"/>
-              </svg>&nbsp;<ng-container i18n>Correspondents</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Correspondents</ng-container></span>
             </a>
           </li>
-          <li class="nav-item">
-            <a class="nav-link" routerLink="tags" routerLinkActive="active" (click)="closeMenu()">
+          <li class="nav-item" tourAnchor="tour.tags">
+            <a class="nav-link" routerLink="tags" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Tags" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#tags"/>
-              </svg>&nbsp;<ng-container i18n>Tags</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Tags</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" routerLink="documenttypes" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="documenttypes" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Document types" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#hash"/>
-              </svg>&nbsp;<ng-container i18n>Document types</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Document types</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" routerLink="logs" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="storagepaths" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Storage paths" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
+              <svg class="sidebaricon" fill="currentColor">
+                <use xlink:href="assets/bootstrap-icons.svg#folder"/>
+              </svg><span>&nbsp;<ng-container i18n>Storage paths</ng-container></span>
+            </a>
+          </li>
+          <li class="nav-item" tourAnchor="tour.file-tasks">
+            <a class="nav-link" routerLink="tasks" routerLinkActive="active" (click)="closeMenu()" ngbPopover="File Tasks" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
+              <span *ngIf="tasksService.failedFileTasks.length > 0 && slimSidebarEnabled" class="badge bg-danger position-absolute top-0 end-0">{{tasksService.failedFileTasks.length}}</span>
+              <svg class="sidebaricon" fill="currentColor">
+                <use xlink:href="assets/bootstrap-icons.svg#list-task"/>
+              </svg><span>&nbsp;<ng-container i18n>File Tasks<span *ngIf="tasksService.failedFileTasks.length > 0"><span class="badge bg-danger ms-2">{{tasksService.failedFileTasks.length}}</span></span></ng-container></span>
+            </a>
+          </li>
+          <li class="nav-item">
+            <a class="nav-link" routerLink="logs" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Logs" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#text-left"/>
-              </svg>&nbsp;<ng-container i18n>Logs</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Logs</ng-container></span>
             </a>
           </li>
-          <li class="nav-item">
-            <a class="nav-link" routerLink="settings" routerLinkActive="active" (click)="closeMenu()">
+          <li class="nav-item" tourAnchor="tour.settings">
+            <a class="nav-link" routerLink="settings" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Settings" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#gear"/>
-              </svg>&nbsp;<ng-container i18n>Settings</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Settings</ng-container></span>
             </a>
           </li>
-          <li class="nav-item">
-            <a class="nav-link" href="admin/">
+          <li class="nav-item" tourAnchor="tour.admin">
+            <a class="nav-link" href="admin/" ngbPopover="Admin" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#toggles"/>
-              </svg>&nbsp;<ng-container i18n>Admin</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Admin</ng-container></span>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-auto pt-4 mb-1 text-muted">
-          <ng-container i18n>Info</ng-container>
+          <span i18n>Info</span>
         </h6>
         <ul class="nav flex-column mb-2">
-          <li class="nav-item">
-            <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://paperless-ngx.readthedocs.io/en/latest/">
+          <li class="nav-item" tourAnchor="tour.outro">
+            <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://paperless-ngx.readthedocs.io/en/latest/" ngbPopover="Documentation" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#question-circle"/>
-              </svg>&nbsp;<ng-container i18n>Documentation</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Documentation</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
             <div class="d-flex w-100 flex-wrap">
-              <a class="nav-link pe-2 pb-1" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx">
+              <a class="nav-link pe-2 pb-1" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx" ngbPopover="GitHub" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
                 <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="sidebaricon" viewBox="0 0 16 16">
                   <use xlink:href="assets/bootstrap-icons.svg#github" />
-                </svg>&nbsp;<ng-container i18n>GitHub</ng-container>
+                </svg><span>&nbsp;<ng-container i18n>GitHub</ng-container></span>
               </a>
-              <a class="nav-link-additional small text-muted ms-3" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests" title="Suggest an idea">
+              <a class="nav-link-additional small text-muted ms-3" [class.visually-hidden]="slimSidebarEnabled" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests" title="Suggest an idea" i18n-title>
                 <svg xmlns="http://www.w3.org/2000/svg" width="1.1em" height="1.1em" fill="currentColor" class="me-1" viewBox="0 0 16 16">
                   <use xlink:href="assets/bootstrap-icons.svg#lightbulb" />
                 </svg>
@@ -182,17 +209,28 @@
               </a>
             </div>
           </li>
-          <li class="nav-item mt-2">
+          <li class="nav-item mt-2" [class.visually-hidden]="slimSidebarEnabled">
             <div class="px-3 py-2 text-muted small d-flex align-items-center flex-wrap">
               <div class="me-3">{{ versionString }}</div>
-              <div *ngIf="appRemoteVersion" class="version-check">
+              <div *ngIf="!settingsService.updateCheckingIsSet || appRemoteVersion" class="version-check">
                 <ng-template #updateAvailablePopContent>
-                  <span class="small">Paperless-ngx v{{ appRemoteVersion.version }} <ng-container i18n>is available.</ng-container><br/><ng-container i18n>Click to view.</ng-container></span>
+                  <span class="small">Paperless-ngx {{ appRemoteVersion.version }} <ng-container i18n>is available.</ng-container><br/><ng-container i18n>Click to view.</ng-container></span>
                 </ng-template>
                 <ng-template #updateCheckingNotEnabledPopContent>
-                  <span class="small"><ng-container i18n>Checking for updates is disabled.</ng-container><br/><ng-container i18n>Click for more information.</ng-container></span>
+                  <p class="small mb-2">
+                    <ng-container i18n>Paperless-ngx can automatically check for updates</ng-container>
+                  </p>
+                  <div class="btn-group btn-group-xs flex-fill w-100">
+                    <button class="btn btn-outline-primary" (click)="setUpdateChecking(true)">Enable</button>
+                    <button class="btn btn-outline-secondary" (click)="setUpdateChecking(false)">Disable</button>
+                  </div>
+                  <p class="small mb-0 mt-2">
+                    <a class="small text-decoration-none fst-italic" routerLink="/settings" fragment="update-checking" i18n>
+                      How does this work?
+                    </a>
+                  </p>
                 </ng-template>
-                <ng-container *ngIf="appRemoteVersion.feature_is_set; else updateCheckNotSet">
+                <ng-container *ngIf="settingsService.updateCheckingIsSet; else updateCheckNotSet">
                   <a *ngIf="appRemoteVersion.update_available" class="small text-decoration-none" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/releases"
                   [ngbPopover]="updateAvailablePopContent" popoverClass="shadow" triggers="mouseenter:mouseleave" container="body">
                     <svg fill="currentColor" class="me-1" width="1.2em" height="1.2em" style="vertical-align: text-top;" viewBox="0 0 16 16">
@@ -202,8 +240,8 @@
                   </a>
                 </ng-container>
                 <ng-template #updateCheckNotSet>
-                  <a class="small text-decoration-none" target="_blank" rel="noopener noreferrer" href="https://paperless-ngx.readthedocs.io/en/latest/configuration.html#update-checking"
-                  [ngbPopover]="updateCheckingNotEnabledPopContent" popoverClass="shadow" triggers="mouseenter:mouseleave" container="body">
+                  <a class="small text-decoration-none" routerLink="/settings" fragment="update-checking"
+                  [ngbPopover]="updateCheckingNotEnabledPopContent" popoverClass="shadow" triggers="mouseenter" container="body">
                     <svg fill="currentColor" class="me-1" width="1.2em" height="1.2em" style="vertical-align: text-top;" viewBox="0 0 16 16">
                       <use xlink:href="assets/bootstrap-icons.svg#info-circle" />
                     </svg>
@@ -216,7 +254,7 @@
       </div>
     </nav>
 
-    <main role="main" class="col-md-9 ms-sm-auto col-lg-10 px-md-4">
+    <main role="main" class="ms-sm-auto px-md-4" [ngClass]="slimSidebarEnabled ? 'col-slim' : 'col-md-9 col-lg-10'">
       <router-outlet></router-outlet>
     </main>
   </div>

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -1,3 +1,6 @@
+@import "node_modules/bootstrap/scss/functions";
+@import "node_modules/bootstrap/scss/variables";
+
 /*
  * Sidebar
  */
@@ -9,6 +12,22 @@
   z-index: 995; /* Behind the navbar */
   padding: 50px 0 0; /* Height of navbar */
   box-shadow: inset -1px 0 0 rgba(0, 0, 0, .1);
+
+  .sidebar-heading .spinner-border {
+    width: 0.8em;
+    height: 0.8em;
+  }
+
+  // These come from the col-md-3 col-lg-2 classes for regular sidebar, needed for animation
+  @media (min-width: 768px) {
+    max-width: 25%;
+  }
+
+  @media (min-width: 992px) {
+    max-width: 16.66666667%;
+  }
+
+  transition: all .2s ease;
 }
 @media (max-width: 767.98px) {
   .sidebar {
@@ -16,6 +35,90 @@
   }
 }
 
+main {
+  transition: all .2s ease;
+}
+
+.sidebar-slim-toggler {
+  display: none; // hide on mobile
+}
+
+.sidebar li.nav-item span,
+.sidebar .sidebar-heading span {
+  transition: all .1s ease;
+}
+
+@media(min-width: 768px) {
+  .sidebar.slim {
+    max-width: 50px;
+
+    li.nav-item span.badge {
+      display: inline-block;
+      margin-right: 2px;
+    }
+  }
+
+  .sidebar.slim:not(.animating) {
+    li.nav-item span,
+    .sidebar-heading span {
+      display: none;
+    }
+  }
+
+  .sidebar.animating {
+    li.nav-item span,
+    .sidebar-heading span {
+      display: unset;
+      position: absolute;
+      opacity: 0;
+      overflow: hidden;
+    }
+  }
+
+  .sidebar:not(.slim):not(.animating) {
+    li.nav-item span,
+    .sidebar-heading span {
+      position: unset;
+      opacity: 1;
+      overflow: auto;
+    }
+  }
+
+  .sidebar.slim,
+  .sidebar.animating {
+    .text-truncate {
+      text-overflow: unset !important;
+      word-wrap: break-word !important;
+    }
+  }
+
+  .sidebar.slim {
+    li.nav-item span.badge {
+      display: inline-block;
+      margin-right: 2px;
+    }
+  }
+
+  .col-slim {
+    padding-left: calc(50px + $grid-gutter-width) !important;
+  }
+
+  .sidebar-slim-toggler {
+    display: block;
+    position: absolute;
+    right: -12px;
+    top: 60px;
+    z-index: 996;
+    --bs-btn-padding-x: 0.35rem;
+    --bs-btn-padding-y: 0.125rem;
+  }
+}
+
+::ng-deep .popover-slim .popover-body {
+  --bs-popover-body-padding-x: .5rem;
+  --bs-popover-body-padding-y: .5rem;
+}
+
 .sidebar-sticky {
   position: relative;
   top: 0;
@@ -72,7 +175,7 @@
 
   .close {
     display: none;
-    position: absolute;
+    position: absolute !important;
     cursor: pointer;
     opacity: 1;
     top: 0;
@@ -140,17 +243,18 @@
 
   form {
     position: relative;
+
+    > svg {
+      position: absolute;
+      left: 0.6rem;
+      top: 0.5rem;
+      color: rgba(255, 255, 255, 0.6);
+    }
   }
 
-  svg {
-    position: absolute;
-    left: 0.6rem;
-    top: 0.5rem;
-    color: rgba(255, 255, 255, 0.6);
-  }
 
   &:focus-within {
-    svg {
+    form > svg {
       display: none;
     }
 

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -1,7 +1,7 @@
-import { Component } from '@angular/core'
+import { Component, HostListener, OnInit } from '@angular/core'
 import { FormControl } from '@angular/forms'
-import { ActivatedRoute, Router, Params } from '@angular/router'
-import { from, Observable, Subscription, BehaviorSubject } from 'rxjs'
+import { ActivatedRoute, Router } from '@angular/router'
+import { from, Observable } from 'rxjs'
 import {
   debounceTime,
   distinctUntilChanged,
@@ -15,37 +15,42 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service'
 import { SearchService } from 'src/app/services/rest/search.service'
 import { environment } from 'src/environments/environment'
 import { DocumentDetailComponent } from '../document-detail/document-detail.component'
-import { Meta } from '@angular/platform-browser'
 import { DocumentListViewService } from 'src/app/services/document-list-view.service'
 import { FILTER_FULLTEXT_QUERY } from 'src/app/data/filter-rule-type'
 import {
   RemoteVersionService,
   AppRemoteVersion,
 } from 'src/app/services/rest/remote-version.service'
-import { QueryParamsService } from 'src/app/services/query-params.service'
+import { SettingsService } from 'src/app/services/settings.service'
+import { TasksService } from 'src/app/services/tasks.service'
+import { ComponentCanDeactivate } from 'src/app/guards/dirty-doc.guard'
+import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings'
+import { ToastService } from 'src/app/services/toast.service'
 
 @Component({
   selector: 'app-app-frame',
   templateUrl: './app-frame.component.html',
   styleUrls: ['./app-frame.component.scss'],
 })
-export class AppFrameComponent {
+export class AppFrameComponent implements OnInit, ComponentCanDeactivate {
   constructor(
     public router: Router,
     private activatedRoute: ActivatedRoute,
     private openDocumentsService: OpenDocumentsService,
     private searchService: SearchService,
     public savedViewService: SavedViewService,
-    private list: DocumentListViewService,
-    private meta: Meta,
     private remoteVersionService: RemoteVersionService,
-    private queryParamsService: QueryParamsService
-  ) {
-    this.remoteVersionService
-      .checkForUpdates()
-      .subscribe((appRemoteVersion: AppRemoteVersion) => {
-        this.appRemoteVersion = appRemoteVersion
-      })
+    private list: DocumentListViewService,
+    public settingsService: SettingsService,
+    public tasksService: TasksService,
+    private readonly toastService: ToastService
+  ) {}
+
+  ngOnInit(): void {
+    if (this.settingsService.get(SETTINGS_KEYS.UPDATE_CHECKING_ENABLED)) {
+      this.checkForUpdates()
+    }
+    this.tasksService.reload()
   }
 
   versionString = `${environment.appTitle} ${environment.version}`
@@ -53,16 +58,64 @@ export class AppFrameComponent {
 
   isMenuCollapsed: boolean = true
 
+  slimSidebarAnimating: boolean = false
+
+  toggleSlimSidebar(): void {
+    this.slimSidebarAnimating = true
+    this.slimSidebarEnabled = !this.slimSidebarEnabled
+    setTimeout(() => {
+      this.slimSidebarAnimating = false
+    }, 200) // slightly longer than css animation for slim sidebar
+  }
+
+  get slimSidebarEnabled(): boolean {
+    return this.settingsService.get(SETTINGS_KEYS.SLIM_SIDEBAR)
+  }
+
+  set slimSidebarEnabled(enabled: boolean) {
+    this.settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, enabled)
+    this.settingsService
+      .storeSettings()
+      .pipe(first())
+      .subscribe({
+        error: (error) => {
+          this.toastService.showError(
+            $localize`An error occurred while saving settings.`
+          )
+          console.log(error)
+        },
+      })
+  }
+
   closeMenu() {
     this.isMenuCollapsed = true
   }
 
   searchField = new FormControl('')
 
+  get searchFieldEmpty(): boolean {
+    return this.searchField.value.trim().length == 0
+  }
+
+  resetSearchField() {
+    this.searchField.reset('')
+  }
+
+  searchFieldKeyup(event: KeyboardEvent) {
+    if (event.key == 'Escape') {
+      this.resetSearchField()
+    }
+  }
+
   get openDocuments(): PaperlessDocument[] {
     return this.openDocumentsService.getOpenDocuments()
   }
 
+  @HostListener('window:beforeunload')
+  canDeactivate(): Observable<boolean> | boolean {
+    return !this.openDocumentsService.hasDirty()
+  }
+
   searchAutoComplete = (text$: Observable<string>) =>
     text$.pipe(
       debounceTime(200),
@@ -94,7 +147,7 @@ export class AppFrameComponent {
 
   search() {
     this.closeMenu()
-    this.queryParamsService.navigateWithFilterRules([
+    this.list.quickFilter([
       {
         rule_type: FILTER_FULLTEXT_QUERY,
         value: (this.searchField.value as string).trim(),
@@ -144,16 +197,29 @@ export class AppFrameComponent {
       })
   }
 
-  get displayName() {
-    // TODO: taken from dashboard component, is this the best way to pass around username?
-    let tagFullName = this.meta.getTag('name=full_name')
-    let tagUsername = this.meta.getTag('name=username')
-    if (tagFullName && tagFullName.content) {
-      return tagFullName.content
-    } else if (tagUsername && tagUsername.content) {
-      return tagUsername.content
-    } else {
-      return null
+  private checkForUpdates() {
+    this.remoteVersionService
+      .checkForUpdates()
+      .subscribe((appRemoteVersion: AppRemoteVersion) => {
+        this.appRemoteVersion = appRemoteVersion
+      })
+  }
+
+  setUpdateChecking(enable: boolean) {
+    this.settingsService.set(SETTINGS_KEYS.UPDATE_CHECKING_ENABLED, enable)
+    this.settingsService
+      .storeSettings()
+      .pipe(first())
+      .subscribe({
+        error: (error) => {
+          this.toastService.showError(
+            $localize`An error occurred while saving update checking settings.`
+          )
+          console.log(error)
+        },
+      })
+    if (enable) {
+      this.checkForUpdates()
     }
   }
 }

src-ui/src/app/components/common/clearable-badge/clearable-badge.component.html

@@ -0,0 +1,9 @@
+<button *ngIf="active" class="position-absolute top-0 start-100 translate-middle badge bg-secondary border border-light rounded-pill p-1" title="Clear" i18n-title (click)="onClick($event)">
+    <svg *ngIf="!isNumbered && selected" width="1em" height="1em" class="check m-0 p-0 opacity-75" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+        <use xlink:href="assets/bootstrap-icons.svg#check-lg"/>
+    </svg>
+    <div *ngIf="isNumbered" class="number">{{number}}<span class="visually-hidden">selected</span></div>
+    <svg width=".9em" height="1em" class="x m-0 p-0 opacity-75" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+        <use xlink:href="assets/bootstrap-icons.svg#x-lg"/>
+    </svg>
+</button>

src-ui/src/app/components/common/clearable-badge/clearable-badge.component.scss

@@ -0,0 +1,28 @@
+.badge {
+    min-width: 20px;
+    min-height: 20px;
+}
+
+.x {
+    display: none;
+}
+
+.number {
+    min-width: 1em;
+    min-height: 1em;
+    display: inline-block;
+}
+
+button:hover {
+    .check,
+    .number {
+        opacity: 0 !important;
+    }
+
+    .x {
+        display: inline-block;
+        position: absolute;
+        top: 5px;
+        left: calc(50% - 4px);
+    }
+}