Merge pull request #2025 from paperless-ngx/fix-redo-ocr-message

Add info that re-do OCR doesnt automatically refresh content

.build-config.json

@@ -1,6 +1,6 @@
 {
   "qpdf": {
-      "version": "10.6.3"
+      "version": "11.1.1"
     },
   "jbig2enc": {
       "version": "0.29",

.editorconfig

@@ -27,6 +27,9 @@ indent_style = space
 [*.md]
 indent_style = space
 
+[Pipfile.lock]
+indent_style = space
+
 # Tests don't get a line width restriction.  It's still a good idea to follow
 # the 79 character rule, but in the interests of clarity, tests often need to
 # violate it.

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -13,6 +13,7 @@ body:
         - [The troubleshooting documentation](https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html).
         - [The installation instructions](https://paperless-ngx.readthedocs.io/en/latest/setup.html#installation).
         - [Existing issues and discussions](https://github.com/paperless-ngx/paperless-ngx/search?q=&type=issues).
+        - Disable any customer container initialization scripts, if using any
 
         If you encounter issues while installing or configuring Paperless-ngx, please post in the ["Support" section of the discussions](https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=support).
   - type: textarea
@@ -41,7 +42,15 @@ body:
     id: logs
     attributes:
       label: Webserver logs
-      description: If available, post any logs from the web server related to your issue.
+      description: Logs from the web server related to your issue.
+      render: bash
+    validations:
+      required: true
+  - type: textarea
+    id: logs_browser
+    attributes:
+      label: Browser logs
+      description: Logs from the web browser related to your issue, if needed
       render: bash
   - type: input
     id: version
@@ -63,9 +72,11 @@ body:
     attributes:
       label: Installation method
       options:
-        - Docker
+        - Docker - official image
+        - Docker - linuxserver.io image
         - Bare metal
         - Other (please describe above)
+      description: Note there are significant differences from the official image and linuxserver.io, please check if your issue is specific to the third-party image.
     validations:
       required: true
   - type: input

.github/release-drafter.yml

@@ -1,3 +1,14 @@
+autolabeler:
+  - label: "bug"
+    branch:
+      - '/^fix/'
+    title:
+      - "/^fix/i"
+  - label: "enhancement"
+    branch:
+      - '/^feature/'
+    title:
+      - "/^feature/i"
 categories:
   - title: 'Breaking Changes'
     labels:
@@ -15,9 +26,15 @@ categories:
       - 'chore'
       - 'deployment'
       - 'translation'
+      - 'ci-cd'
   - title: 'Dependencies'
     collapse-after: 3
     label: 'dependencies'
+  - title: 'All App Changes'
+    labels:
+      - 'frontend'
+      - 'backend'
+    collapse-after: 0
 include-labels:
   - 'enhancement'
   - 'bug'
@@ -25,11 +42,12 @@ include-labels:
   - 'deployment'
   - 'translation'
   - 'dependencies'
-replacers: # Changes "Feature: Update checker" to "Update checker"
-  - search: '/Feature:|Feat:|\[feature\]/gi'
-    replace: ''
+  - 'documentation'
+  - 'frontend'
+  - 'backend'
+  - 'ci-cd'
 category-template: '### $TITLE'
-change-template: '- $TITLE [@$AUTHOR](https://github.com/$AUTHOR) ([#$NUMBER]($URL))'
+change-template: '- $TITLE @$AUTHOR ([#$NUMBER]($URL))'
 change-title-escapes: '\<*_&#@'
 template: |
   ## paperless-ngx $RESOLVED_VERSION

.github/scripts/cleanup-tags.py

@@ -1,144 +1,312 @@
+#!/usr/bin/env python3
+import json
 import logging
 import os
+import shutil
+import subprocess
 from argparse import ArgumentParser
+from typing import Dict
 from typing import Final
 from typing import List
-from urllib.parse import quote
+from typing import Optional
 
-import requests
 from common import get_log_level
+from github import ContainerPackage
+from github import GithubBranchApi
+from github import GithubContainerRegistryApi
 
 logger = logging.getLogger("cleanup-tags")
 
 
-class GithubContainerRegistry:
+class DockerManifest2:
+    """
+    Data class wrapping the Docker Image Manifest Version 2.
+
+    See https://docs.docker.com/registry/spec/manifest-v2-2/
+    """
+
+    def __init__(self, data: Dict) -> None:
+        self._data = data
+        # This is the sha256: digest string.  Corresponds to GitHub API name
+        # if the package is an untagged package
+        self.digest = self._data["digest"]
+        platform_data_os = self._data["platform"]["os"]
+        platform_arch = self._data["platform"]["architecture"]
+        platform_variant = self._data["platform"].get(
+            "variant",
+            "",
+        )
+        self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
+
+
+class RegistryTagsCleaner:
+    """
+    This is the base class for the image registry cleaning.  Given a package
+    name, it will keep all images which are tagged and all untagged images
+    referred to by a manifest.  This results in only images which have been untagged
+    and cannot be referenced except by their SHA in being removed.  None of these
+    images should be referenced, so it is fine to delete them.
+    """
+
     def __init__(
-        self,
-        session: requests.Session,
-        token: str,
-        owner_or_org: str,
-    ):
-        self._session: requests.Session = session
-        self._token = token
-        self._owner_or_org = owner_or_org
-        # https://docs.github.com/en/rest/branches/branches
-        self._BRANCHES_ENDPOINT = "https://api.github.com/repos/{OWNER}/{REPO}/branches"
-        if self._owner_or_org == "paperless-ngx":
-            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-an-organization
-            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
-            # https://docs.github.com/en/rest/packages#delete-package-version-for-an-organization
-            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
-        else:
-            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-the-authenticated-user
-            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
-            # https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
-            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
-
-    def __enter__(self):
-        self._session.headers.update(
-            {
-                "Accept": "application/vnd.github.v3+json",
-                "Authorization": f"token {self._token}",
-            },
-        )
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if "Accept" in self._session.headers:
-            del self._session.headers["Accept"]
-        if "Authorization" in self._session.headers:
-            del self._session.headers["Authorization"]
-
-    def _read_all_pages(self, endpoint):
-        internal_data = []
-
-        while True:
-            resp = self._session.get(endpoint)
-            if resp.status_code == 200:
-                internal_data += resp.json()
-                if "next" in resp.links:
-                    endpoint = resp.links["next"]["url"]
-                else:
-                    logger.debug("Exiting pagination loop")
-                    break
-            else:
-                logger.warning(f"Request to {endpoint} return HTTP {resp.status_code}")
-                break
-
-        return internal_data
-
-    def get_branches(self, repo: str):
-        endpoint = self._BRANCHES_ENDPOINT.format(OWNER=self._owner_or_org, REPO=repo)
-        internal_data = self._read_all_pages(endpoint)
-        return internal_data
-
-    def filter_branches_by_name_pattern(self, branch_data, pattern: str):
-        matches = {}
-
-        for branch in branch_data:
-            if branch["name"].startswith(pattern):
-                matches[branch["name"]] = branch
-
-        return matches
-
-    def get_package_versions(
         self,
         package_name: str,
-        package_type: str = "container",
-    ) -> List:
-        package_name = quote(package_name, safe="")
-        endpoint = self._PACKAGES_VERSIONS_ENDPOINT.format(
-            ORG=self._owner_or_org,
-            PACKAGE_TYPE=package_type,
-            PACKAGE_NAME=package_name,
+        repo_owner: str,
+        repo_name: str,
+        package_api: GithubContainerRegistryApi,
+        branch_api: Optional[GithubBranchApi],
+    ):
+        self.actually_delete = False
+        self.package_api = package_api
+        self.branch_api = branch_api
+        self.package_name = package_name
+        self.repo_owner = repo_owner
+        self.repo_name = repo_name
+        self.tags_to_delete: List[str] = []
+        self.tags_to_keep: List[str] = []
+
+        # Get the information about all versions of the given package
+        # These are active, not deleted, the default returned from the API
+        self.all_package_versions = self.package_api.get_active_package_versions(
+            self.package_name,
         )
 
-        internal_data = self._read_all_pages(endpoint)
-
-        return internal_data
-
-    def filter_packages_by_tag_pattern(self, package_data, pattern: str):
-        matches = {}
-
-        for package in package_data:
-            if "metadata" in package and "container" in package["metadata"]:
-                container_metadata = package["metadata"]["container"]
-                if "tags" in container_metadata:
-                    container_tags = container_metadata["tags"]
-                    for tag in container_tags:
-                        if tag.startswith(pattern):
-                            matches[tag] = package
-                            break
-
-        return matches
-
-    def filter_packages_untagged(self, package_data):
-        matches = {}
-
-        for package in package_data:
-            if "metadata" in package and "container" in package["metadata"]:
-                container_metadata = package["metadata"]["container"]
-                if "tags" in container_metadata:
-                    container_tags = container_metadata["tags"]
-                    if not len(container_tags):
-                        matches[package["name"]] = package
-
-        return matches
-
-    def delete_package_version(self, package_name, package_data):
-        package_name = quote(package_name, safe="")
-        endpoint = self._PACKAGE_VERSION_DELETE_ENDPOINT.format(
-            ORG=self._owner_or_org,
-            PACKAGE_TYPE=package_data["metadata"]["package_type"],
-            PACKAGE_NAME=package_name,
-            PACKAGE_VERSION_ID=package_data["id"],
+        # Get a mapping from a tag like "1.7.0" or "feature-xyz" to the ContainerPackage
+        # tagged with it.  It makes certain lookups easy
+        self.all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
+        for pkg in self.all_package_versions:
+            for tag in pkg.tags:
+                self.all_pkgs_tags_to_version[tag] = pkg
+        logger.info(
+            f"Located {len(self.all_package_versions)} versions of package {self.package_name}",
         )
-        resp = self._session.delete(endpoint)
-        if resp.status_code != 204:
-            logger.warning(
-                f"Request to delete {endpoint} returned HTTP {resp.status_code}",
+
+        self.decide_what_tags_to_keep()
+
+    def clean(self):
+        """
+        This method will delete image versions, based on the selected tags to delete
+        """
+        for tag_to_delete in self.tags_to_delete:
+            package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
+
+            if self.actually_delete:
+                logger.info(
+                    f"Deleting {tag_to_delete} (id {package_version_info.id})",
+                )
+                self.package_api.delete_package_version(
+                    package_version_info,
+                )
+
+            else:
+                logger.info(
+                    f"Would delete {tag_to_delete} (id {package_version_info.id})",
+                )
+        else:
+            logger.info("No tags to delete")
+
+    def clean_untagged(self, is_manifest_image: bool):
+        """
+        This method will delete untagged images, that is those which are not named.  It
+        handles if the image tag is actually a manifest, which points to images that look otherwise
+        untagged.
+        """
+
+        def _clean_untagged_manifest():
+            """
+
+            Handles the deletion of untagged images, but where the package is a manifest, ie a multi
+            arch image, which means some "untagged" images need to exist still.
+
+            Ok, bear with me, these are annoying.
+
+            Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
+            These images are untagged, but pointed to, and so should not be removed (or every pull fails).
+
+            So for each image getting kept, parse the manifest to find the digest(s) it points to.  Then
+            remove those from the list of untagged images.  The final result is the untagged, not pointed to
+            version which should be safe to remove.
+
+            Example:
+                Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
+                    amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
+                    armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
+                    arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
+                each of which appears as untagged image, but isn't really.
+
+                So from the list of untagged packages, remove those digests.  Once all tags which
+                are being kept are checked, the remaining untagged packages are actually untagged
+                with no referrals in a manifest to them.
+            """
+            # Simplify the untagged data, mapping name (which is a digest) to the version
+            # At the moment, these are the images which APPEAR untagged.
+            untagged_versions = {}
+            for x in self.all_package_versions:
+                if x.untagged:
+                    untagged_versions[x.name] = x
+
+            skips = 0
+
+            # Parse manifests to locate digests pointed to
+            for tag in sorted(self.tags_to_keep):
+                full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
+                logger.info(f"Checking manifest for {full_name}")
+                try:
+                    proc = subprocess.run(
+                        [
+                            shutil.which("docker"),
+                            "manifest",
+                            "inspect",
+                            full_name,
+                        ],
+                        capture_output=True,
+                    )
+
+                    manifest_list = json.loads(proc.stdout)
+                    for manifest_data in manifest_list["manifests"]:
+                        manifest = DockerManifest2(manifest_data)
+
+                        if manifest.digest in untagged_versions:
+                            logger.info(
+                                f"Skipping deletion of {manifest.digest},"
+                                f" referred to by {full_name}"
+                                f" for {manifest.platform}",
+                            )
+                            del untagged_versions[manifest.digest]
+                            skips += 1
+
+                except Exception as err:
+                    self.actually_delete = False
+                    logger.exception(err)
+                    return
+
+            logger.info(
+                f"Skipping deletion of {skips} packages referred to by a manifest",
             )
 
+            # Delete the untagged and not pointed at packages
+            logger.info(f"Deleting untagged packages of {self.package_name}")
+            for to_delete_name in untagged_versions:
+                to_delete_version = untagged_versions[to_delete_name]
+
+                if self.actually_delete:
+                    logger.info(
+                        f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
+                    )
+                    self.package_api.delete_package_version(
+                        to_delete_version,
+                    )
+                else:
+                    logger.info(
+                        f"Would delete {to_delete_name} (id {to_delete_version.id})",
+                    )
+
+        def _clean_untagged_non_manifest():
+            """
+            If the package is not a multi-arch manifest, images without tags are safe to delete.
+            """
+
+            for package in self.all_package_versions:
+                if package.untagged:
+                    if self.actually_delete:
+                        logger.info(
+                            f"Deleting id {package.id} named {package.name}",
+                        )
+                        self.package_api.delete_package_version(
+                            package,
+                        )
+                    else:
+                        logger.info(
+                            f"Would delete {package.name} (id {package.id})",
+                        )
+                else:
+                    logger.info(
+                        f"Not deleting tag {package.tags[0]} of package {self.package_name}",
+                    )
+
+        logger.info("Beginning untagged image cleaning")
+
+        if is_manifest_image:
+            _clean_untagged_manifest()
+        else:
+            _clean_untagged_non_manifest()
+
+    def decide_what_tags_to_keep(self):
+        """
+        This method holds the logic to delete what tags to keep and there fore
+        what tags to delete.
+
+        By default, any image with at least 1 tag will be kept
+        """
+        # By default, keep anything which is tagged
+        self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
+
+
+class MainImageTagsCleaner(RegistryTagsCleaner):
+    def decide_what_tags_to_keep(self):
+        """
+        Overrides the default logic for deciding what images to keep.  Images tagged as "feature-"
+        will be removed, if the corresponding branch no longer exists.
+        """
+
+        # Default to everything gets kept still
+        super().decide_what_tags_to_keep()
+
+        # Locate the feature branches
+        feature_branches = {}
+        for branch in self.branch_api.get_branches(
+            repo=self.repo_name,
+        ):
+            if branch.name.startswith("feature-"):
+                logger.debug(f"Found feature branch {branch.name}")
+                feature_branches[branch.name] = branch
+
+        logger.info(f"Located {len(feature_branches)} feature branches")
+
+        if not len(feature_branches):
+            # Our work here is done, delete nothing
+            return
+
+        # Filter to packages which are tagged with feature-*
+        packages_tagged_feature: List[ContainerPackage] = []
+        for package in self.all_package_versions:
+            if package.tag_matches("feature-"):
+                packages_tagged_feature.append(package)
+
+        # Map tags like "feature-xyz" to a ContainerPackage
+        feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
+        for pkg in packages_tagged_feature:
+            for tag in pkg.tags:
+                feature_pkgs_tags_to_versions[tag] = pkg
+
+        logger.info(
+            f'Located {len(feature_pkgs_tags_to_versions)} versions of package {self.package_name} tagged "feature-"',
+        )
+
+        # All the feature tags minus all the feature branches leaves us feature tags
+        # with no corresponding branch
+        self.tags_to_delete = list(
+            set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
+        )
+
+        # All the tags minus the set of going to be deleted tags leaves us the
+        # tags which will be kept around
+        self.tags_to_keep = list(
+            set(self.all_pkgs_tags_to_version.keys()) - set(self.tags_to_delete),
+        )
+        logger.info(
+            f"Located {len(self.tags_to_delete)} versions of package {self.package_name} to delete",
+        )
+
+
+class LibraryTagsCleaner(RegistryTagsCleaner):
+    """
+    Exists for the off change that someday, the installer library images
+    will need their own logic
+    """
+
+    pass
+
 
 def _main():
     parser = ArgumentParser(
@@ -146,6 +314,7 @@ def _main():
         " tags which no longer have an associated feature branch",
     )
 
+    # Requires an affirmative command to actually do a delete
     parser.add_argument(
         "--delete",
         action="store_true",
@@ -153,7 +322,8 @@ def _main():
         help="If provided, actually delete the container tags",
     )
 
-    # TODO There's a lot of untagged images, do those need to stay for anything?
+    # When a tagged image is updated, the previous version remains, but it no longer tagged
+    # Add this option to remove them as well
     parser.add_argument(
         "--untagged",
         action="store_true",
@@ -161,12 +331,28 @@ def _main():
         help="If provided, delete untagged containers as well",
     )
 
+    # If given, the package is assumed to be a multi-arch manifest.  Cache packages are
+    # not multi-arch, all other types are
+    parser.add_argument(
+        "--is-manifest",
+        action="store_true",
+        default=False,
+        help="If provided, the package is assumed to be a multi-arch manifest following schema v2",
+    )
+
+    # Allows configuration of log level for debugging
     parser.add_argument(
         "--loglevel",
         default="info",
         help="Configures the logging level",
     )
 
+    # Get the name of the package being processed this round
+    parser.add_argument(
+        "package",
+        help="The package to process",
+    )
+
     args = parser.parse_args()
 
     logging.basicConfig(
@@ -175,79 +361,41 @@ def _main():
         format="%(asctime)s %(levelname)-8s %(message)s",
     )
 
+    # Must be provided in the environment
     repo_owner: Final[str] = os.environ["GITHUB_REPOSITORY_OWNER"]
     repo: Final[str] = os.environ["GITHUB_REPOSITORY"]
-    gh_token: Final[str] = os.environ["GITHUB_TOKEN"]
+    gh_token: Final[str] = os.environ["TOKEN"]
 
-    with requests.session() as sess:
-        with GithubContainerRegistry(sess, gh_token, repo_owner) as gh_api:
-            all_branches = gh_api.get_branches("paperless-ngx")
-            logger.info(f"Located {len(all_branches)} branches of {repo_owner}/{repo} ")
-
-            feature_branches = gh_api.filter_branches_by_name_pattern(
-                all_branches,
-                "feature-",
-            )
-            logger.info(f"Located {len(feature_branches)} feature branches")
-
-            for package_name in ["paperless-ngx", "paperless-ngx/builder/cache/app"]:
-
-                all_package_versions = gh_api.get_package_versions(package_name)
-                logger.info(
-                    f"Located {len(all_package_versions)} versions of package {package_name}",
+    # Find all branches named feature-*
+    # Note: Only relevant to the main application, but simpler to
+    # leave in for all packages
+    with GithubBranchApi(gh_token) as branch_api:
+        with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
+            if args.package in {"paperless-ngx", "paperless-ngx/builder/cache/app"}:
+                cleaner = MainImageTagsCleaner(
+                    args.package,
+                    repo_owner,
+                    repo,
+                    container_api,
+                    branch_api,
+                )
+            else:
+                cleaner = LibraryTagsCleaner(
+                    args.package,
+                    repo_owner,
+                    repo,
+                    container_api,
+                    None,
                 )
 
-                packages_tagged_feature = gh_api.filter_packages_by_tag_pattern(
-                    all_package_versions,
-                    "feature-",
-                )
-                logger.info(
-                    f'Located {len(packages_tagged_feature)} versions of package {package_name} tagged "feature-"',
-                )
+            # Set if actually doing a delete vs dry run
+            cleaner.actually_delete = args.delete
 
-                untagged_packages = gh_api.filter_packages_untagged(
-                    all_package_versions,
-                )
-                logger.info(
-                    f"Located {len(untagged_packages)} untagged versions of package {package_name}",
-                )
+            # Clean images with tags
+            cleaner.clean()
 
-                to_delete = list(
-                    set(packages_tagged_feature.keys()) - set(feature_branches.keys()),
-                )
-                logger.info(
-                    f"Located {len(to_delete)} versions of package {package_name} to delete",
-                )
-
-                for tag_to_delete in to_delete:
-                    package_version_info = packages_tagged_feature[tag_to_delete]
-
-                    if args.delete:
-                        logger.info(
-                            f"Deleting {tag_to_delete} (id {package_version_info['id']})",
-                        )
-                        gh_api.delete_package_version(
-                            package_name,
-                            package_version_info,
-                        )
-
-                    else:
-                        logger.info(
-                            f"Would delete {tag_to_delete} (id {package_version_info['id']})",
-                        )
-
-                if args.untagged:
-                    logger.info(f"Deleting untagged packages of {package_name}")
-                    for to_delete_name in untagged_packages:
-                        to_delete_version = untagged_packages[to_delete_name]
-                        logger.info(f"Deleting id {to_delete_version['id']}")
-                        if args.delete:
-                            gh_api.delete_package_version(
-                                package_name,
-                                to_delete_version,
-                            )
-                else:
-                    logger.info("Leaving untagged images untouched")
+            # Clean images which are untagged
+            cleaner.clean_untagged(args.is_manifest)
 
 
 if __name__ == "__main__":

.github/scripts/common.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import logging
-from argparse import ArgumentError
 
 
 def get_image_tag(
@@ -30,6 +29,11 @@ def get_cache_image_tag(
 
 
 def get_log_level(args) -> int:
+    """
+    Returns a logging level, based
+    :param args:
+    :return:
+    """
     levels = {
         "critical": logging.CRITICAL,
         "error": logging.ERROR,

.github/scripts/github.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python3
+"""
+This module contains some useful classes for interacting with the Github API.
+The full documentation for the API can be found here: https://docs.github.com/en/rest
+
+Mostly, this focusses on two areas, repo branches and repo packages, as the use case
+is cleaning up container images which are no longer referred to.
+
+"""
+import functools
+import logging
+import re
+import urllib.parse
+from typing import Dict
+from typing import List
+from typing import Optional
+
+import httpx
+
+logger = logging.getLogger("github-api")
+
+
+class _GithubApiBase:
+    """
+    A base class for interacting with the Github API.  It
+    will handle the session and setting authorization headers.
+    """
+
+    def __init__(self, token: str) -> None:
+        self._token = token
+        self._client: Optional[httpx.Client] = None
+
+    def __enter__(self) -> "_GithubApiBase":
+        """
+        Sets up the required headers for auth and response
+        type from the API
+        """
+        self._client = httpx.Client()
+        self._client.headers.update(
+            {
+                "Accept": "application/vnd.github.v3+json",
+                "Authorization": f"token {self._token}",
+            },
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """
+        Ensures the authorization token is cleaned up no matter
+        the reason for the exit
+        """
+        if "Accept" in self._client.headers:
+            del self._client.headers["Accept"]
+        if "Authorization" in self._client.headers:
+            del self._client.headers["Authorization"]
+
+        # Close the session as well
+        self._client.close()
+        self._client = None
+
+    def _read_all_pages(self, endpoint):
+        """
+        Helper function to read all pages of an endpoint, utilizing the
+        next.url until exhausted.  Assumes the endpoint returns a list
+        """
+        internal_data = []
+
+        while True:
+            resp = self._client.get(endpoint)
+            if resp.status_code == 200:
+                internal_data += resp.json()
+                if "next" in resp.links:
+                    endpoint = resp.links["next"]["url"]
+                else:
+                    logger.debug("Exiting pagination loop")
+                    break
+            else:
+                logger.warning(f"Request to {endpoint} return HTTP {resp.status_code}")
+                resp.raise_for_status()
+
+        return internal_data
+
+
+class _EndpointResponse:
+    """
+    For all endpoint JSON responses, store the full
+    response data, for ease of extending later, if need be.
+    """
+
+    def __init__(self, data: Dict) -> None:
+        self._data = data
+
+
+class GithubBranch(_EndpointResponse):
+    """
+    Simple wrapper for a repository branch, only extracts name information
+    for now.
+    """
+
+    def __init__(self, data: Dict) -> None:
+        super().__init__(data)
+        self.name = self._data["name"]
+
+
+class GithubBranchApi(_GithubApiBase):
+    """
+    Wrapper around branch API.
+
+    See https://docs.github.com/en/rest/branches/branches
+
+    """
+
+    def __init__(self, token: str) -> None:
+        super().__init__(token)
+
+        self._ENDPOINT = "https://api.github.com/repos/{REPO}/branches"
+
+    def get_branches(self, repo: str) -> List[GithubBranch]:
+        """
+        Returns all current branches of the given repository owned by the given
+        owner or organization.
+        """
+        # The environment GITHUB_REPOSITORY already contains the owner in the correct location
+        endpoint = self._ENDPOINT.format(REPO=repo)
+        internal_data = self._read_all_pages(endpoint)
+        return [GithubBranch(branch) for branch in internal_data]
+
+
+class ContainerPackage(_EndpointResponse):
+    """
+    Data class wrapping the JSON response from the package related
+    endpoints
+    """
+
+    def __init__(self, data: Dict):
+        super().__init__(data)
+        # This is a numerical ID, required for interactions with this
+        # specific package, including deletion of it or restoration
+        self.id: int = self._data["id"]
+
+        # A string name.  This might be an actual name or it could be a
+        # digest string like "sha256:"
+        self.name: str = self._data["name"]
+
+        # URL to the package, including its ID, can be used for deletion
+        # or restoration without needing to build up a URL ourselves
+        self.url: str = self._data["url"]
+
+        # The list of tags applied to this image. Maybe an empty list
+        self.tags: List[str] = self._data["metadata"]["container"]["tags"]
+
+    @functools.cached_property
+    def untagged(self) -> bool:
+        """
+        Returns True if the image has no tags applied to it, False otherwise
+        """
+        return len(self.tags) == 0
+
+    @functools.cache
+    def tag_matches(self, pattern: str) -> bool:
+        """
+        Returns True if the image has at least one tag which matches the given regex,
+        False otherwise
+        """
+        for tag in self.tags:
+            if re.match(pattern, tag) is not None:
+                return True
+        return False
+
+    def __repr__(self):
+        return f"Package {self.name}"
+
+
+class GithubContainerRegistryApi(_GithubApiBase):
+    """
+    Class wrapper to deal with the Github packages API.  This class only deals with
+    container type packages, the only type published by paperless-ngx.
+    """
+
+    def __init__(self, token: str, owner_or_org: str) -> None:
+        super().__init__(token)
+        self._owner_or_org = owner_or_org
+        if self._owner_or_org == "paperless-ngx":
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-an-organization
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-package-version-for-an-organization
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+        else:
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-the-authenticated-user
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+        self._PACKAGE_VERSION_RESTORE_ENDPOINT = (
+            f"{self._PACKAGE_VERSION_DELETE_ENDPOINT}/restore"
+        )
+
+    def get_active_package_versions(
+        self,
+        package_name: str,
+    ) -> List[ContainerPackage]:
+        """
+        Returns all the versions of a given package (container images) from
+        the API
+        """
+
+        package_type: str = "container"
+        # Need to quote this for slashes in the name
+        package_name = urllib.parse.quote(package_name, safe="")
+
+        endpoint = self._PACKAGES_VERSIONS_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_type,
+            PACKAGE_NAME=package_name,
+        )
+
+        pkgs = []
+
+        for data in self._read_all_pages(endpoint):
+            pkgs.append(ContainerPackage(data))
+
+        return pkgs
+
+    def get_deleted_package_versions(
+        self,
+        package_name: str,
+    ) -> List[ContainerPackage]:
+        package_type: str = "container"
+        # Need to quote this for slashes in the name
+        package_name = urllib.parse.quote(package_name, safe="")
+
+        endpoint = (
+            self._PACKAGES_VERSIONS_ENDPOINT.format(
+                ORG=self._owner_or_org,
+                PACKAGE_TYPE=package_type,
+                PACKAGE_NAME=package_name,
+            )
+            + "?state=deleted"
+        )
+
+        pkgs = []
+
+        for data in self._read_all_pages(endpoint):
+            pkgs.append(ContainerPackage(data))
+
+        return pkgs
+
+    def delete_package_version(self, package_data: ContainerPackage):
+        """
+        Deletes the given package version from the GHCR
+        """
+        resp = self._client.delete(package_data.url)
+        if resp.status_code != 204:
+            logger.warning(
+                f"Request to delete {package_data.url} returned HTTP {resp.status_code}",
+            )
+
+    def restore_package_version(
+        self,
+        package_name: str,
+        package_data: ContainerPackage,
+    ):
+        package_type: str = "container"
+        endpoint = self._PACKAGE_VERSION_RESTORE_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_type,
+            PACKAGE_NAME=package_name,
+            PACKAGE_VERSION_ID=package_data.id,
+        )
+
+        resp = self._client.post(endpoint)
+        if resp.status_code != 204:
+            logger.warning(
+                f"Request to delete {endpoint} returned HTTP {resp.status_code}",
+            )

.github/stale.yml

@@ -1,15 +1,23 @@
 # Number of days of inactivity before an issue becomes stale
 daysUntilStale: 30
+
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
-onlyLabels:
-  - unconfirmed
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: [cant-reproduce]
+
 # Label to use when marking an issue as stale
 staleLabel: stale
+
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
   This issue has been automatically marked as stale because it has not had
   recent activity. It will be closed if no further activity occurs. Thank you
   for your contributions.
+
 # Comment to post when closing a stale issue. Set to `false` to disable
 closeComment: false
+
+# See https://github.com/marketplace/stale for more info on the app
+# and https://github.com/probot/stale for the configuration docs

.github/workflows/ci.yml

@@ -14,16 +14,37 @@ on:
       - 'translations**'
 
 jobs:
+  pre-commit:
+    name: Linting Checks
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout repository
+        uses: actions/checkout@v3
+
+      -
+        name: Install tools
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      -
+        name: Check files
+        uses: pre-commit/action@v3.0.0
+
   documentation:
     name: "Build Documentation"
     runs-on: ubuntu-20.04
+    needs:
+      - pre-commit
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
       -
         name: Install pipenv
-        run: pipx install pipenv
+        run: |
+          pipx install pipenv==2022.10.12
       -
         name: Set up Python
         uses: actions/setup-python@v4
@@ -35,6 +56,10 @@ jobs:
         name: Install dependencies
         run: |
           pipenv sync --dev
+      -
+        name: List installed Python dependencies
+        run: |
+          pipenv run pip list
       -
         name: Make documentation
         run: |
@@ -47,11 +72,107 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  ci-backend:
-    uses: ./.github/workflows/reusable-ci-backend.yml
+  tests-backend:
+    name: "Tests (${{ matrix.python-version }})"
+    runs-on: ubuntu-20.04
+    needs:
+      - pre-commit
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+      fail-fast: false
+    services:
+      tika:
+        image: ghcr.io/paperless-ngx/tika:latest
+        ports:
+          - "9998:9998/tcp"
+      gotenberg:
+        image: docker.io/gotenberg/gotenberg:7.6
+        ports:
+          - "3000:3000/tcp"
+    env:
+      # Enable Tika end to end testing
+      TIKA_LIVE: 1
+      # Enable paperless_mail testing against real server
+      PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
+      PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
+      PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Install pipenv
+        run: |
+          pipx install pipenv==2022.10.12
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: List installed Python dependencies
+        run: |
+          pipenv run pip list
+      -
+        name: Tests
+        run: |
+          cd src/
+          pipenv run pytest -rfEp
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v34
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          pipenv run coveralls --service=github
 
-  ci-frontend:
-    uses: ./.github/workflows/reusable-ci-frontend.yml
+  tests-frontend:
+    name: "Tests Frontend"
+    runs-on: ubuntu-20.04
+    needs:
+      - pre-commit
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      -
+        name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci
 
   prepare-docker-build:
     name: Prepare Docker Pipeline Data
@@ -65,15 +186,15 @@ jobs:
       cancel-in-progress: false
     needs:
       - documentation
-      - ci-backend
-      - ci-frontend
+      - tests-backend
+      - tests-frontend
     steps:
       -
         name: Set ghcr repository name
         id: set-ghcr-repository
         run: |
           ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
-          echo ::set-output name=repository::${ghcr_name}
+          echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
       -
         name: Checkout
         uses: actions/checkout@v3
@@ -90,7 +211,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=qpdf-json::${build_json}
+          echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup psycopg2 image
         id: psycopg2-setup
@@ -99,7 +220,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=psycopg2-json::${build_json}
+          echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup pikepdf image
         id: pikepdf-setup
@@ -108,7 +229,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=pikepdf-json::${build_json}
+          echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup jbig2enc image
         id: jbig2enc-setup
@@ -117,7 +238,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=jbig2enc-json::${build_json}
+          echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
 
     outputs:
 
@@ -152,10 +273,10 @@ jobs:
         run: |
           if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
             echo "Enabling DockerHub image push"
-            echo ::set-output name=enable::"true"
+            echo "enable=true" >> $GITHUB_OUTPUT
           else
             echo "Not pushing to DockerHub"
-            echo ::set-output name=enable::"false"
+            echo "enable=false" >> $GITHUB_OUTPUT
           fi
       -
         name: Gather Docker metadata
@@ -243,18 +364,27 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
+      -
+        name: Install pipenv
+        run: |
+          pip3 install --upgrade pip setuptools wheel pipx
+          pipx install pipenv
       -
         name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
       -
-        name: Install dependencies
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Install system dependencies
         run: |
           sudo apt-get update -qq
           sudo apt-get install -qq --no-install-recommends gettext liblept5
-          pip3 install --upgrade pip setuptools wheel
-          pip3 install -r requirements.txt
       -
         name: Download frontend artifact
         uses: actions/download-artifact@v3
@@ -267,34 +397,38 @@ jobs:
         with:
           name: documentation
           path: docs/_build/html/
+      -
+        name: Generate requirements file
+        run: |
+          pipenv requirements > requirements.txt
+      -
+        name: Compile messages
+        run: |
+          cd src/
+          pipenv run python3 manage.py compilemessages
+      -
+        name: Collect static files
+        run: |
+          cd src/
+          pipenv run python3 manage.py collectstatic --no-input
       -
         name: Move files
         run: |
           mkdir dist
           mkdir dist/paperless-ngx
           mkdir dist/paperless-ngx/scripts
-          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ngx/
+          cp .dockerignore .env Dockerfile Pipfile Pipfile.lock requirements.txt LICENSE README.md dist/paperless-ngx/
           cp paperless.conf.example dist/paperless-ngx/paperless.conf
           cp gunicorn.conf.py dist/paperless-ngx/gunicorn.conf.py
-          cp docker/ dist/paperless-ngx/docker -r
+          cp -r docker/ dist/paperless-ngx/docker
           cp scripts/*.service scripts/*.sh dist/paperless-ngx/scripts/
-          cp src/ dist/paperless-ngx/src -r
-          cp docs/_build/html/ dist/paperless-ngx/docs -r
-      -
-        name: Compile messages
-        run: |
-          cd dist/paperless-ngx/src
-          python3 manage.py compilemessages
-      -
-        name: Collect static files
-        run: |
-          cd dist/paperless-ngx/src
-          python3 manage.py collectstatic --no-input
+          cp -r src/ dist/paperless-ngx/src
+          cp -r docs/_build/html/ dist/paperless-ngx/docs
+          mv static dist/paperless-ngx
       -
         name: Make release package
         run: |
           cd dist
-          find . -name __pycache__ | xargs rm -r
           tar -cJf paperless-ngx.tar.xz paperless-ngx/
       -
         name: Upload release artifact
@@ -305,6 +439,10 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
+    outputs:
+      prerelease: ${{ steps.get_version.outputs.prerelease }}
+      changelog: ${{ steps.create-release.outputs.body }}
+      version: ${{ steps.get_version.outputs.version }}
     needs:
       - build-release
     if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
@@ -319,16 +457,16 @@ jobs:
         name: Get version
         id: get_version
         run: |
-          echo ::set-output name=version::${{ github.ref_name }}
+          echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
           if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
-            echo ::set-output name=prerelease::true
+            echo "prerelease=true" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=prerelease::false
+            echo "prerelease=false" >> $GITHUB_OUTPUT
           fi
       -
         name: Create Release and Changelog
         id: create-release
-        uses: release-drafter/release-drafter@v5
+        uses: paperless-ngx/release-drafter@master
         with:
           name: Paperless-ngx ${{ steps.get_version.outputs.version }}
           tag: ${{ steps.get_version.outputs.version }}
@@ -348,21 +486,65 @@ jobs:
           asset_path: ./paperless-ngx.tar.xz
           asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
           asset_content_type: application/x-xz
+
+  append-changelog:
+    runs-on: ubuntu-20.04
+    needs:
+      - publish-release
+    if: needs.publish-release.outputs.prerelease == 'false'
+    steps:
       -
         name: Checkout
         uses: actions/checkout@v3
         with:
           ref: main
+      -
+        name: Install pipenv
+        run: |
+          pip3 install --upgrade pip setuptools wheel pipx
+          pipx install pipenv
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
       -
         name: Append Changelog to docs
         id: append-Changelog
         working-directory: docs
         run: |
-          echo -e "# Changelog\n\n${{ steps.create-release.outputs.body }}\n" > changelog-new.md
+          git branch ${{ needs.publish-release.outputs.version }}-changelog
+          git checkout ${{ needs.publish-release.outputs.version }}-changelog
+          echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md
+          echo "Manually linking usernames"
+          sed -i -r 's|@(.+?) \(\[#|[@\1](https://github.com/\1) ([#|ig' changelog-new.md
           CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
           echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
           mv changelog-new.md changelog.md
+          pipenv run pre-commit --files changelog.md
           git config --global user.name "github-actions"
           git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git commit -am "Changelog ${{ steps.get_version.outputs.version }} - GHA"
-          git push origin HEAD:main
+          git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA"
+          git push origin ${{ needs.publish-release.outputs.version }}-changelog
+      -
+        name: Create Pull Request
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const { repo, owner } = context.repo;
+            const result = await github.rest.pulls.create({
+              title: '[Documentation] Add ${{ needs.publish-release.outputs.version }} changelog',
+              owner,
+              repo,
+              head: '${{ needs.publish-release.outputs.version }}-changelog',
+              base: 'main',
+              body: 'This PR is auto-generated by CI.'
+            });
+            github.rest.issues.addLabels({
+              owner,
+              repo,
+              issue_number: result.data.number,
+              labels: ['documentation']
+            });

.github/workflows/cleanup-tags.yml

@@ -1,3 +1,8 @@
+# This workflow runs on certain conditions to check for and potentially
+# delete container images from the GHCR which no longer have an associated
+# code branch.
+# Requires a PAT with the correct scope set in the secrets
+
 name: Cleanup Image Tags
 
 on:
@@ -11,38 +16,80 @@ on:
     paths:
       - ".github/workflows/cleanup-tags.yml"
       - ".github/scripts/cleanup-tags.py"
+      - ".github/scripts/github.py"
       - ".github/scripts/common.py"
 
-env:
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+concurrency:
+  group: registry-tags-cleanup
+  cancel-in-progress: false
 
 jobs:
-  cleanup:
-    name: Cleanup Image Tags
-    runs-on: ubuntu-20.04
-    permissions:
-      packages: write
+  cleanup-images:
+    name: Cleanup Image Tags for ${{ matrix.primary-name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - primary-name: "paperless-ngx"
+            cache-name: "paperless-ngx/builder/cache/app"
+
+          - primary-name: "paperless-ngx/builder/qpdf"
+            cache-name: "paperless-ngx/builder/cache/qpdf"
+
+          - primary-name: "paperless-ngx/builder/pikepdf"
+            cache-name: "paperless-ngx/builder/cache/pikepdf"
+
+          - primary-name: "paperless-ngx/builder/jbig2enc"
+            cache-name: "paperless-ngx/builder/cache/jbig2enc"
+
+          - primary-name: "paperless-ngx/builder/psycopg2"
+            cache-name: "paperless-ngx/builder/cache/psycopg2"
+    env:
+      # Requires a personal access token with the OAuth scope delete:packages
+      TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
       -
         name: Login to Github Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       -
-        name: Install requests
+        name: Install httpx
         run: |
-          python -m pip install requests
+          python -m pip install httpx
+      #
+      # Clean up primary package
+      #
       -
-        name: Cleanup feature tags
+        name: Cleanup for package "${{ matrix.primary-name }}"
+        if: "${{ env.TOKEN != '' }}"
         run: |
-          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --delete
+          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --is-manifest --delete "${{ matrix.primary-name }}"
+      #
+      # Clean up registry cache package
+      #
+      -
+        name: Cleanup for package "${{ matrix.cache-name }}"
+        if: "${{ env.TOKEN != '' }}"
+        run: |
+          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
+      #
+      # Verify tags which are left still pull
+      #
+      -
+        name: Check all tags still pull
+        run: |
+          ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
+          echo "Pulling all tags of ${ghcr_name}"
+          docker pull --quiet --all-tags ${ghcr_name}
+          docker image list

.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/installer-library.yml

@@ -41,7 +41,7 @@ jobs:
         id: set-ghcr-repository
         run: |
           ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
-          echo ::set-output name=repository::${ghcr_name}
+          echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
       -
         name: Checkout
         uses: actions/checkout@v3
@@ -50,6 +50,11 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.9"
+      -
+        name: Install jq
+        run: |
+          sudo apt-get update
+          sudo apt-get install jq
       -
         name: Setup qpdf image
         id: qpdf-setup
@@ -58,7 +63,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=qpdf-json::${build_json}
+          echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup psycopg2 image
         id: psycopg2-setup
@@ -67,7 +72,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=psycopg2-json::${build_json}
+          echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup pikepdf image
         id: pikepdf-setup
@@ -76,7 +81,7 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=pikepdf-json::${build_json}
+          echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
       -
         name: Setup jbig2enc image
         id: jbig2enc-setup
@@ -85,7 +90,19 @@ jobs:
 
           echo ${build_json}
 
-          echo ::set-output name=jbig2enc-json::${build_json}
+          echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
+      -
+        name: Setup other versions
+        id: cache-bust-setup
+        run: |
+          pillow_version=$(jq ".default.pillow.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+          lxml_version=$(jq ".default.lxml.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+
+          echo "Pillow is ${pillow_version}"
+          echo "lxml is ${lxml_version}"
+
+          echo "pillow-version=${pillow_version}" >> $GITHUB_OUTPUT
+          echo "lxml-version=${lxml_version}" >> $GITHUB_OUTPUT
 
     outputs:
 
@@ -97,7 +114,11 @@ jobs:
 
       psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
 
-      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json }}
+
+      pillow-version: ${{ steps.cache-bust-setup.outputs.pillow-version }}
+
+      lxml-version: ${{ steps.cache-bust-setup.outputs.lxml-version }}
 
   build-qpdf-debs:
     name: qpdf
@@ -145,3 +166,5 @@ jobs:
         REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }}
         QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
         PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+        PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
+        LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}

.github/workflows/project-actions.yml

@@ -13,6 +13,9 @@ on:
       - main
       - dev
 
+permissions:
+  contents: read
+
 env:
   todo: Todo
   done: Done
@@ -24,8 +27,8 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'issues' && (github.event.action == 'opened' || github.event.action == 'reopened')
     steps:
-      - name: Set issue status to ${{ env.todo }}
-        uses: leonsteinhaeuser/project-beta-automations@v1.2.1
+      - name: Add issue to project and set status to ${{ env.todo }}
+        uses: leonsteinhaeuser/project-beta-automations@v2.0.1
         with:
           gh_token: ${{ secrets.GH_TOKEN }}
           organization: paperless-ngx
@@ -35,13 +38,20 @@ jobs:
   pr_opened_or_reopened:
     name: pr_opened_or_reopened
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened')
+    permissions:
+      # write permission is required for autolabeler
+      pull-requests: write
+    if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened') && github.event.pull_request.user.login != 'dependabot'
     steps:
-      - name: Set PR status to ${{ env.in_progress }}
-        uses: leonsteinhaeuser/project-beta-automations@v1.2.1
+      - name: Add PR to project and set status to "Needs Review"
+        uses: leonsteinhaeuser/project-beta-automations@v2.0.1
         with:
           gh_token: ${{ secrets.GH_TOKEN }}
           organization: paperless-ngx
           project_id: 2
           resource_node_id: ${{ github.event.pull_request.node_id }}
-          status_value: ${{ env.in_progress }} # Target status
+          status_value: "Needs Review" # Target status
+      - name: Label PR with release-drafter
+        uses: release-drafter/release-drafter@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/reusable-ci-backend.yml

@@ -1,108 +0,0 @@
-name: Backend CI Jobs
-
-on:
-  workflow_call:
-
-jobs:
-
-  code-checks-backend:
-    name: "Code Style Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Install checkers
-        run: |
-          pipx install reorder-python-imports
-          pipx install yesqa
-          pipx install add-trailing-comma
-          pipx install flake8
-      -
-        name: Run reorder-python-imports
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-      -
-        name: Run yesqa
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-      -
-        name: Run add-trailing-comma
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
-      # black is placed after add-trailing-comma because it may format differently
-      # if a trailing comma is added
-      -
-        name: Run black
-        uses: psf/black@stable
-        with:
-          options: "--check --diff"
-          version: "22.3.0"
-      -
-        name: Run flake8 checks
-        run: |
-          cd src/
-          flake8 --max-line-length=88 --ignore=E203,W503
-
-  tests-backend:
-    name: "Tests (${{ matrix.python-version }})"
-    runs-on: ubuntu-20.04
-    needs:
-      - code-checks-backend
-    strategy:
-      matrix:
-        python-version: ['3.8', '3.9', '3.10']
-      fail-fast: false
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      -
-        name: Install pipenv
-        run: pipx install pipenv
-      -
-        name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
-      -
-        name: Install system dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
-      -
-        name: Install Python dependencies
-        run: |
-          pipenv sync --dev
-      -
-        name: Tests
-        run: |
-          cd src/
-          pipenv run pytest
-      -
-        name: Get changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v23.1
-        with:
-          files: |
-            src/**
-      -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github

.github/workflows/reusable-ci-frontend.yml

@@ -1,42 +0,0 @@
-name: Frontend CI Jobs
-
-on:
-  workflow_call:
-
-jobs:
-
-  code-checks-frontend:
-    name: "Code Style Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-      -
-        name: Install prettier
-        run: |
-          npm install prettier
-      -
-        name: Run prettier
-        run:
-          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
-  tests-frontend:
-    name: "Tests"
-    runs-on: ubuntu-20.04
-    needs:
-      - code-checks-frontend
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-      - run: cd src-ui && npm ci
-      - run: cd src-ui && npm run test
-      - run: cd src-ui && npm run e2e:ci

.gitignore

@@ -93,3 +93,6 @@ scripts/nuke
 
 # mac os
 .DS_Store
+
+# celery schedule file
+celerybeat-schedule*

.hadolint.yml

@@ -0,0 +1,8 @@
+failure-threshold: warning
+ignored:
+  # https://github.com/hadolint/hadolint/wiki/DL3008
+  - DL3008
+  # https://github.com/hadolint/hadolint/wiki/DL3013
+  - DL3013
+  # https://github.com/hadolint/hadolint/wiki/DL3003
+  - DL3003

.pre-commit-config.yaml

@@ -37,33 +37,33 @@ repos:
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.8.1
+    rev: v3.9.0
     hooks:
       - id: reorder-python-imports
         exclude: "(migrations)"
   - repo: https://github.com/asottile/yesqa
-    rev: "v1.3.0"
+    rev: "v1.4.0"
     hooks:
       - id: yesqa
         exclude: "(migrations)"
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: "v2.2.3"
+    rev: "v2.3.0"
     hooks:
       - id: add-trailing-comma
         exclude: "(migrations)"
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+  - repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
     hooks:
       - id: flake8
         files: ^src/
         args:
           - "--config=./src/setup.cfg"
   - repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 22.10.0
     hooks:
       - id: black
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.37.1
+    rev: v3.2.2
     hooks:
       - id: pyupgrade
         exclude: "(migrations)"
@@ -74,13 +74,6 @@ repos:
     rev: v2.10.0
     hooks:
       - id: hadolint
-        args:
-          - --ignore
-          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
-          - --ignore
-          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
-          - --ignore
-          - DL3003      # https://github.com/hadolint/hadolint/wiki/DL3003  (seems excessive to use WORKDIR so much)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1

CODEOWNERS

@@ -7,4 +7,3 @@
 /src/ @paperless-ngx/backend
 Pipfile* @paperless-ngx/backend
 *.py @paperless-ngx/backend
-requirements.txt @paperless-ngx/backend

Dockerfile

@@ -30,6 +30,25 @@ RUN set -eux \
 RUN set -eux \
   && ./node_modules/.bin/ng build --configuration production
 
+FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base
+
+# This stage generates the requirements.txt file using pipenv
+# This stage runs once for the native platform, as the outputs are not
+# dependent on target arch
+# This way, pipenv dependencies are not left in the final image
+# nor can pipenv mess up the final image somehow
+# Inputs: None
+
+WORKDIR /usr/src/pipenv
+
+COPY Pipfile* ./
+
+RUN set -eux \
+  && echo "Installing pipenv" \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv \
+  && echo "Generating requirement.txt" \
+    && pipenv requirements > requirements.txt
+
 FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
@@ -81,6 +100,7 @@ ARG RUNTIME_PACKAGES="\
   python3-pip \
   python3-setuptools \
   postgresql-client \
+  mariadb-client \
   # For Numpy
   libatlas3-base \
   # OCRmyPDF dependencies
@@ -90,6 +110,10 @@ ARG RUNTIME_PACKAGES="\
   tesseract-ocr-fra \
   tesseract-ocr-ita \
   tesseract-ocr-spa \
+  # Suggested for OCRmyPDF
+  pngquant \
+  # Suggested for pikepdf
+  jbig2dec \
   tzdata \
   unpaper \
   # Mime type detection
@@ -117,19 +141,19 @@ COPY gunicorn.conf.py .
 # setup docker-specific things
 # Use mounts to avoid copying installer files into the image
 # These change sometimes, but rarely
-ARG DOCKER_SRC=/usr/src/paperless/src/docker/
-WORKDIR ${DOCKER_SRC}
+WORKDIR /usr/src/paperless/src/docker/
 
 COPY [ \
-	"docker/imagemagick-policy.xml", \
-	"docker/supervisord.conf", \
-	"docker/docker-entrypoint.sh", \
-	"docker/docker-prepare.sh", \
-	"docker/paperless_cmd.sh", \
-	"docker/wait-for-redis.py", \
-	"docker/management_script.sh", \
-	"docker/install_management_commands.sh", \
-	"${DOCKER_SRC}" \
+  "docker/imagemagick-policy.xml", \
+  "docker/supervisord.conf", \
+  "docker/docker-entrypoint.sh", \
+  "docker/docker-prepare.sh", \
+  "docker/paperless_cmd.sh", \
+  "docker/wait-for-redis.py", \
+  "docker/management_script.sh", \
+  "docker/flower-conditional.sh", \
+  "docker/install_management_commands.sh", \
+  "/usr/src/paperless/src/docker/" \
 ]
 
 RUN set -eux \
@@ -147,6 +171,8 @@ RUN set -eux \
     && chmod 755 /sbin/wait-for-redis.py \
     && mv paperless_cmd.sh /usr/local/bin/paperless_cmd.sh \
     && chmod 755 /usr/local/bin/paperless_cmd.sh \
+    && mv flower-conditional.sh /usr/local/bin/flower-conditional.sh \
+    && chmod 755 /usr/local/bin/flower-conditional.sh \
   && echo "Installing managment commands" \
     && chmod +x install_management_commands.sh \
     && ./install_management_commands.sh
@@ -159,28 +185,31 @@ RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
     --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
   set -eux \
   && echo "Installing qpdf" \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf29_*.deb \
     && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
   && echo "Installing pikepdf and dependencies" \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \
     && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \
     && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \
     && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \
     && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \
-    && python -m pip list \
+    && python3 -m pip list \
   && echo "Installing psycopg2" \
     && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
-    && python -m pip list
+    && python3 -m pip list
+
+WORKDIR /usr/src/paperless/src/
 
 # Python dependencies
 # Change pretty frequently
-COPY requirements.txt ../
+COPY --from=pipenv-base /usr/src/pipenv/requirements.txt ./
 
 # Packages needed only for building a few quick Python
 # dependencies
 ARG BUILD_PACKAGES="\
   build-essential \
   git \
+  default-libmysqlclient-dev \
   python3-dev"
 
 RUN set -eux \
@@ -189,7 +218,7 @@ RUN set -eux \
     && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
     && python3 -m pip install --no-cache-dir --upgrade wheel \
   && echo "Installing Python requirements" \
-    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir --requirement requirements.txt \
   && echo "Cleaning up image" \
     && apt-get -y purge ${BUILD_PACKAGES} \
     && apt-get -y autoremove --purge \
@@ -200,8 +229,6 @@ RUN set -eux \
     && rm -rf /var/cache/apt/archives/* \
     && truncate -s 0 /var/log/*log
 
-WORKDIR /usr/src/paperless/src/
-
 # copy backend
 COPY ./src ./
 

Pipfile

@@ -10,38 +10,42 @@ name = "piwheels"
 
 [packages]
 dateparser = "~=1.1"
-django = "~=4.0"
+django = "~=4.1"
 django-cors-headers = "*"
 django-extensions = "*"
 django-filter = "~=22.1"
-django-q = {editable = true, ref = "paperless-main", git = "https://github.com/paperless-ngx/django-q.git"}
-djangorestframework = "~=3.13"
+djangorestframework = "~=3.14"
 filelock = "*"
-fuzzywuzzy = {extras = ["speedup"], version = "*"}
 gunicorn = "*"
 imap-tools = "*"
 langdetect = "*"
 pathvalidate = "*"
-pillow = "~=9.2"
-pikepdf = "~=5.1"
+pillow = "~=9.3"
+pikepdf = "*"
 python-gnupg = "*"
 python-dotenv = "*"
 python-dateutil = "*"
 python-magic = "*"
 psycopg2 = "*"
-redis = "*"
-scikit-learn="~=1.1"
-whitenoise = "~=6.2.0"
-watchdog = "~=2.1.9"
-whoosh="~=2.7.4"
+rapidfuzz = "*"
+redis = {extras = ["hiredis"], version = "*"}
+scikit-learn = "~=1.1"
+# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
+scipy = "==1.8.1"
+numpy = "*"
+whitenoise = "~=6.2"
+watchdog = "~=2.1"
+whoosh="~=2.7"
 inotifyrecursive = "~=0.3"
-ocrmypdf = "~=13.4"
+ocrmypdf = "~=14.0"
 tqdm = "*"
 tika = "*"
 # TODO: This will sadly also install daphne+dependencies,
 #  which an ASGI server we don't need. Adds about 15MB image size.
 channels = "~=3.0"
-channels-redis = "*"
+# Locked version until https://github.com/django/channels_redis/issues/332
+# is resolved
+channels-redis = "==3.4.1"
 uvicorn = {extras = ["standard"], version = "*"}
 concurrent-log-handler = "*"
 "pdfminer.six" = "*"
@@ -49,7 +53,13 @@ concurrent-log-handler = "*"
 "importlib-resources" = {version = "*", markers = "python_version < '3.9'"}
 zipp = {version = "*", markers = "python_version < '3.9'"}
 pyzbar = "*"
+mysqlclient = "*"
+celery = {extras = ["redis"], version = "*"}
+django-celery-results = "*"
+setproctitle = "*"
+nltk = "*"
 pdf2image = "*"
+flower = "*"
 
 [dev-packages]
 coveralls = "*"
@@ -61,7 +71,7 @@ pytest-django = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-sphinx = "~=5.0.2"
+sphinx = "~=5.3"
 sphinx_rtd_theme = "*"
 tox = "*"
 black = "*"

README.md

@@ -2,7 +2,7 @@
 [![Crowdin](https://badges.crowdin.net/paperless-ngx/localized.svg)](https://crowdin.com/project/paperless-ngx)
 [![Documentation Status](https://readthedocs.org/projects/paperless-ngx/badge/?version=latest)](https://paperless-ngx.readthedocs.io/en/latest/?badge=latest)
 [![Coverage Status](https://coveralls.io/repos/github/paperless-ngx/paperless-ngx/badge.svg?branch=master)](https://coveralls.io/github/paperless-ngx/paperless-ngx?branch=master)
-[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/#paperless:adnidor.de)
+[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/%23paperlessngx%3Amatrix.org)
 
 <p align="center">
 <img src="https://github.com/paperless-ngx/paperless-ngx/raw/main/resources/logo/web/png/Black%20logo%20-%20no%20background.png#gh-light-mode-only" width="50%" />
@@ -105,6 +105,7 @@ Paperless has been around a while now, and people are starting to build stuff on
 - [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ng.
 - [Paperless Share](https://github.com/qcasey/paperless_share). Share any files from your Android application with paperless. Very simple, but works with all of the mobile scanning apps out there that allow you to share scanned documents.
 - [Scan to Paperless](https://github.com/sbrunner/scan-to-paperless): Scan and prepare (crop, deskew, OCR, ...) your documents for Paperless.
+- [Paperless Mobile](https://github.com/astubenbord/paperless-mobile): A modern, feature rich mobile application for Paperless.
 
 These projects also exist, but their status and compatibility with paperless-ngx is unknown.
 

build-docker-image.sh

@@ -23,6 +23,8 @@ fi
 # Parse what we can from Pipfile.lock
 pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
 psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+pillow_version=$(jq ".default.pillow.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+lxml_version=$(jq ".default.lxml.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
 # Read this from the other config file
 qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
 jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
@@ -40,4 +42,6 @@ docker build --file "$1" \
 	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
 	--build-arg QPDF_VERSION="${qpdf_version}" \
 	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PILLOW_VERSION="${pillow_version}" \
+	--build-arg LXML_VERSION="${lxml_version}" \
 	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" .

docker-builders/Dockerfile.frontend

@@ -1,14 +0,0 @@
-# This Dockerfile compiles the frontend
-# Inputs: None
-
-FROM node:16-bullseye-slim AS compile-frontend
-
-COPY ./src /src/src
-COPY ./src-ui /src/src-ui
-
-WORKDIR /src/src-ui
-RUN set -eux \
-  && npm update npm -g \
-  && npm ci --omit=optional
-RUN set -eux \
-  && ./node_modules/.bin/ng build --configuration production

docker-builders/Dockerfile.jbig2enc

@@ -7,6 +7,7 @@ FROM debian:bullseye-slim as main
 LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG JBIG2ENC_VERSION
 
 ARG BUILD_PACKAGES="\
   build-essential \
@@ -19,21 +20,16 @@ ARG BUILD_PACKAGES="\
 
 WORKDIR /usr/src/jbig2enc
 
-# As this is an base image for a multi-stage final image
-# the added size of the install is basically irrelevant
-RUN apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
-  && rm -rf /var/lib/apt/lists/*
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-ARG JBIG2ENC_VERSION
-
 RUN set -eux \
-  && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
-RUN set -eux \
-  && ./autogen.sh
-RUN set -eux \
-  && ./configure && make
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && echo "Building jbig2enc" \
+    && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc . \
+    && ./autogen.sh \
+    && ./configure \
+    && make \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker-builders/Dockerfile.pikepdf

@@ -17,6 +17,11 @@ FROM python:3.9-slim-bullseye as main
 LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG PIKEPDF_VERSION
+# These are not used, but will still bust the cache if one changes
+# Otherwise, the main image will try to build thing (and fail)
+ARG PILLOW_VERSION
+ARG LXML_VERSION
 
 ARG BUILD_PACKAGES="\
   build-essential \
@@ -55,34 +60,33 @@ COPY --from=qpdf-builder /usr/src/qpdf/*.deb ./
 # the added size of the install is basically irrelevant
 
 RUN set -eux \
-  && apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
-  && dpkg --install libqpdf28_*.deb \
-  && dpkg --install libqpdf-dev_*.deb \
-  && python3 -m pip install --no-cache-dir --upgrade \
-    pip \
-    wheel \
-    # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
-    pybind11 \
-  && rm -rf /var/lib/apt/lists/*
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-ARG PIKEPDF_VERSION
-
-RUN set -eux \
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && echo "Installing qpdf" \
+    && dpkg --install libqpdf29_*.deb \
+    && dpkg --install libqpdf-dev_*.deb \
+  && echo "Installing Python tools" \
+    && python3 -m pip install --no-cache-dir --upgrade \
+      pip \
+      wheel \
+      # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
+      pybind11 \
   && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \
-  && mkdir wheels \
-  && python3 -m pip wheel \
-    # Build the package at the required version
-    pikepdf==${PIKEPDF_VERSION} \
-    # Output the *.whl into this directory
-    --wheel-dir wheels \
-    # Do not use a binary packge for the package being built
-    --no-binary=pikepdf \
-    # Do use binary packages for dependencies
-    --prefer-binary \
-    --no-cache-dir \
-  && ls -ahl wheels
+    && mkdir wheels \
+    && python3 -m pip wheel \
+      # Build the package at the required version
+      pikepdf==${PIKEPDF_VERSION} \
+      # Output the *.whl into this directory
+      --wheel-dir wheels \
+      # Do not use a binary packge for the package being built
+      --no-binary=pikepdf \
+      # Do use binary packages for dependencies
+      --prefer-binary \
+      # Don't cache build files
+      --no-cache-dir \
+    && ls -ahl wheels \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker-builders/Dockerfile.psycopg2

@@ -6,6 +6,7 @@ FROM python:3.9-slim-bullseye as main
 
 LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
 
+ARG PSYCOPG2_VERSION
 ARG DEBIAN_FRONTEND=noninteractive
 
 ARG BUILD_PACKAGES="\
@@ -21,29 +22,27 @@ WORKDIR /usr/src
 # the added size of the install is basically irrelevant
 
 RUN set -eux \
-  && apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
-  && rm -rf /var/lib/apt/lists/* \
-  && python3 -m pip install --no-cache-dir --upgrade pip wheel
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-ARG PSYCOPG2_VERSION
-
-RUN set -eux \
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && echo "Installing Python tools" \
+    && python3 -m pip install --no-cache-dir --upgrade pip wheel \
   && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \
-  && cd /usr/src \
-  && mkdir wheels \
-  && python3 -m pip wheel \
-    # Build the package at the required version
-    psycopg2==${PSYCOPG2_VERSION} \
-    # Output the *.whl into this directory
-    --wheel-dir wheels \
-    # Do not use a binary packge for the package being built
-    --no-binary=psycopg2 \
-    # Do use binary packages for dependencies
-    --prefer-binary \
-    --no-cache-dir \
-  && ls -ahl wheels/
+    && cd /usr/src \
+    && mkdir wheels \
+    && python3 -m pip wheel \
+      # Build the package at the required version
+      psycopg2==${PSYCOPG2_VERSION} \
+      # Output the *.whl into this directory
+      --wheel-dir wheels \
+      # Do not use a binary packge for the package being built
+      --no-binary=psycopg2 \
+      # Do use binary packages for dependencies
+      --prefer-binary \
+      # Don't cache build files
+      --no-cache-dir \
+    && ls -ahl wheels/ \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker-builders/Dockerfile.qpdf

@@ -1,8 +1,15 @@
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+#    - QPDF_VERSION - the version of qpdf to build a .deb.
+#                     Must be present as a deb-src in bookworm
+
 FROM debian:bullseye-slim as main
 
 LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
 
 ARG DEBIAN_FRONTEND=noninteractive
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
 
 ARG BUILD_PACKAGES="\
   build-essential \
@@ -15,39 +22,27 @@ ARG BUILD_PACKAGES="\
   libjpeg62-turbo-dev \
   libgnutls28-dev \
   packaging-dev \
+  cmake \
   zlib1g-dev"
 
 WORKDIR /usr/src
 
-# As this is an base image for a multi-stage final image
-# the added size of the install is basically irrelevant
-
-RUN set -eux \
-  && apt-get update --quiet \
-  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
-  && rm -rf /var/lib/apt/lists/*
-
-# Layers after this point change according to required version
-# For better caching, seperate the basic installs from
-# the building
-
-# This must match to pikepdf's minimum at least
-ARG QPDF_VERSION
-
-# In order to get the required version of qpdf, it is backported from bookwork
-# and then built from source
 RUN set -eux \
+  && echo "Installing build tools" \
+    && apt-get update --quiet \
+    && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && echo "Getting qpdf src" \
+    && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
+    && apt-get update \
+    && mkdir qpdf \
+    && cd qpdf \
+    && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
   && echo "Building qpdf" \
-  && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
-  && apt-get update \
-  && mkdir qpdf \
-  && cd qpdf \
-  && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
-  && rm -rf /var/lib/apt/lists/* \
-  && cd qpdf-$QPDF_VERSION \
-  # We don't need to build the tests (also don't run them)
-  && rm -rf libtests \
-  && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
-  && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
-  && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
-  && ls -ahl ../*.deb
+    && cd qpdf-$QPDF_VERSION \
+    && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+    && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean \
+    && ls -ahl ../*.deb \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && rm -rf /var/lib/apt/lists/*

docker/compose/docker-compose.env

@@ -36,3 +36,7 @@
 # The default language to use for OCR. Set this to the language most of your
 # documents are written in.
 #PAPERLESS_OCR_LANGUAGE=eng
+
+# Set if accessing paperless via a domain subpath e.g. https://domain.com/PATHPREFIX and using a reverse-proxy like traefik or nginx
+#PAPERLESS_FORCE_SCRIPT_NAME=/PATHPREFIX
+#PAPERLESS_STATIC_URL=/PATHPREFIX/static/ # trailing slash required

docker/compose/docker-compose.mariadb-tika.yml

@@ -0,0 +1,102 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), MariaDB is used as the database server.
+# - Apache Tika and Gotenberg servers are started with paperless and paperless
+#   is configured to use these services. These provide support for consuming
+#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
+#   parts.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: docker.io/library/redis:7
+    restart: unless-stopped
+    volumes:
+      - redisdata:/data
+
+  db:
+    image: docker.io/library/mariadb:10
+    restart: unless-stopped
+    volumes:
+      - dbdata:/var/lib/mysql
+    environment:
+      MARIADB_HOST: paperless
+      MARIADB_DATABASE: paperless
+      MARIADB_USER: paperless
+      MARIADB_PASSWORD: paperless
+      MARIADB_ROOT_PASSWORD: paperless
+    ports:
+      - "3306:3306"
+
+  webserver:
+    image: ghcr.io/paperless-ngx/paperless-ngx:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+      - gotenberg
+      - tika
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBENGINE: mariadb
+      PAPERLESS_DBHOST: db
+      PAPERLESS_DBUSER: paperless # only needed if non-default username
+      PAPERLESS_DBPASS: paperless # only needed if non-default password
+      PAPERLESS_DBPORT: 3306
+      PAPERLESS_TIKA_ENABLED: 1
+      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
+      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
+
+  gotenberg:
+    image: docker.io/gotenberg/gotenberg:7.6
+    restart: unless-stopped
+    command:
+      - "gotenberg"
+      - "--chromium-disable-routes=true"
+
+  tika:
+    image: ghcr.io/paperless-ngx/tika:latest
+    restart: unless-stopped
+
+volumes:
+  data:
+  media:
+  dbdata:
+  redisdata:

docker/compose/docker-compose.mariadb.yml

@@ -0,0 +1,83 @@
+# docker-compose file for running paperless from the Docker Hub.
+# This file contains everything paperless needs to run.
+# Paperless supports amd64, arm and arm64 hardware.
+#
+# All compose files of paperless configure paperless in the following way:
+#
+# - Paperless is (re)started on system boot, if it was running before shutdown.
+# - Docker volumes for storing data are managed by Docker.
+# - Folders for importing and exporting files are created in the same directory
+#   as this file and mounted to the correct folders inside the container.
+# - Paperless listens on port 8000.
+#
+# In addition to that, this docker-compose file adds the following optional
+# configurations:
+#
+# - Instead of SQLite (default), MariaDB is used as the database server.
+#
+# To install and update paperless with this file, do the following:
+#
+# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
+#   and '.env' into a folder.
+# - Run 'docker-compose pull'.
+# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker-compose up -d'.
+#
+# For more extensive installation and update instructions, refer to the
+# documentation.
+
+version: "3.4"
+services:
+  broker:
+    image: docker.io/library/redis:7
+    restart: unless-stopped
+    volumes:
+      - redisdata:/data
+
+  db:
+    image: docker.io/library/mariadb:10
+    restart: unless-stopped
+    volumes:
+      - dbdata:/var/lib/mysql
+    environment:
+      MARIADB_HOST: paperless
+      MARIADB_DATABASE: paperless
+      MARIADB_USER: paperless
+      MARIADB_PASSWORD: paperless
+      MARIADB_ROOT_PASSWORD: paperless
+    ports:
+      - "3306:3306"
+
+  webserver:
+    image: ghcr.io/paperless-ngx/paperless-ngx:latest
+    restart: unless-stopped
+    depends_on:
+      - db
+      - broker
+    ports:
+      - 8000:8000
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - data:/usr/src/paperless/data
+      - media:/usr/src/paperless/media
+      - ./export:/usr/src/paperless/export
+      - ./consume:/usr/src/paperless/consume
+    env_file: docker-compose.env
+    environment:
+      PAPERLESS_REDIS: redis://broker:6379
+      PAPERLESS_DBENGINE: mariadb
+      PAPERLESS_DBHOST: db
+      PAPERLESS_DBUSER: paperless # only needed if non-default username
+      PAPERLESS_DBPASS: paperless # only needed if non-default password
+      PAPERLESS_DBPORT: 3306
+
+
+volumes:
+  data:
+  media:
+  dbdata:
+  redisdata:

docker/compose/docker-compose.portainer.yml

@@ -31,7 +31,7 @@
 version: "3.4"
 services:
   broker:
-    image: docker.io/library/redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data

docker/compose/docker-compose.postgres-tika.yml

@@ -33,7 +33,7 @@
 version: "3.4"
 services:
   broker:
-    image: docker.io/library/redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data
@@ -77,7 +77,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.6
     restart: unless-stopped
     command:
       - "gotenberg"

docker/compose/docker-compose.postgres.yml

@@ -29,7 +29,7 @@
 version: "3.4"
 services:
   broker:
-    image: docker.io/library/redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data

docker/compose/docker-compose.sqlite-tika.yml

@@ -33,7 +33,7 @@
 version: "3.4"
 services:
   broker:
-    image: docker.io/library/redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data
@@ -65,7 +65,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.4
+    image: docker.io/gotenberg/gotenberg:7.6
     restart: unless-stopped
     command:
       - "gotenberg"

docker/compose/docker-compose.sqlite.yml

@@ -26,7 +26,7 @@
 version: "3.4"
 services:
   broker:
-    image: docker.io/library/redis:6.0
+    image: docker.io/library/redis:7
     restart: unless-stopped
     volumes:
       - redisdata:/data

docker/docker-entrypoint.sh

@@ -9,8 +9,8 @@ set -e
 # fill in the value of "$XYZ_DB_PASSWORD" from a file, especially for Docker's
 # secrets feature
 file_env() {
-	local var="$1"
-	local fileVar="${var}_FILE"
+	local -r var="$1"
+	local -r fileVar="${var}_FILE"
 
 	# Basic validation
 	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
@@ -35,14 +35,14 @@ file_env() {
 
 # Source: https://github.com/sameersbn/docker-gitlab/
 map_uidgid() {
-	USERMAP_ORIG_UID=$(id -u paperless)
-	USERMAP_ORIG_GID=$(id -g paperless)
-	USERMAP_NEW_UID=${USERMAP_UID:-$USERMAP_ORIG_UID}
-	USERMAP_NEW_GID=${USERMAP_GID:-${USERMAP_ORIG_GID:-$USERMAP_NEW_UID}}
-	if [[ ${USERMAP_NEW_UID} != "${USERMAP_ORIG_UID}" || ${USERMAP_NEW_GID} != "${USERMAP_ORIG_GID}" ]]; then
-		echo "Mapping UID and GID for paperless:paperless to $USERMAP_NEW_UID:$USERMAP_NEW_GID"
-		usermod -o -u "${USERMAP_NEW_UID}" paperless
-		groupmod -o -g "${USERMAP_NEW_GID}" paperless
+	local -r usermap_original_uid=$(id -u paperless)
+	local -r usermap_original_gid=$(id -g paperless)
+	local -r usermap_new_uid=${USERMAP_UID:-$usermap_original_uid}
+	local -r usermap_new_gid=${USERMAP_GID:-${usermap_original_gid:-$usermap_new_uid}}
+	if [[ ${usermap_new_uid} != "${usermap_original_uid}" || ${usermap_new_gid} != "${usermap_original_gid}" ]]; then
+		echo "Mapping UID and GID for paperless:paperless to $usermap_new_uid:$usermap_new_gid"
+		usermod -o -u "${usermap_new_uid}" paperless
+		groupmod -o -g "${usermap_new_gid}" paperless
 	fi
 }
 
@@ -50,6 +50,31 @@ map_folders() {
 	# Export these so they can be used in docker-prepare.sh
 	export DATA_DIR="${PAPERLESS_DATA_DIR:-/usr/src/paperless/data}"
 	export MEDIA_ROOT_DIR="${PAPERLESS_MEDIA_ROOT:-/usr/src/paperless/media}"
+	export CONSUME_DIR="${PAPERLESS_CONSUMPTION_DIR:-/usr/src/paperless/consume}"
+}
+
+nltk_data () {
+	# Store the NLTK data outside the Docker container
+	local -r nltk_data_dir="${DATA_DIR}/nltk"
+	local -r truthy_things=("yes y 1 t true")
+
+	# If not set, or it looks truthy
+	if [[ -z "${PAPERLESS_ENABLE_NLTK}" ]] || [[ "${truthy_things[*]}" =~ ${PAPERLESS_ENABLE_NLTK,} ]]; then
+
+		# Download or update the snowball stemmer data
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" snowball_data
+
+		# Download or update the stopwords corpus
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" stopwords
+
+		# Download or update the punkt tokenizer data
+		python3 -W ignore::RuntimeWarning -m nltk.downloader -d "${nltk_data_dir}" punkt
+
+	else
+		echo "Skipping NLTK data download"
+
+	fi
+
 }
 
 initialize() {
@@ -62,7 +87,8 @@ initialize() {
 		PAPERLESS_AUTO_LOGIN_USERNAME \
 		PAPERLESS_ADMIN_USER \
 		PAPERLESS_ADMIN_MAIL \
-		PAPERLESS_ADMIN_PASSWORD; do
+		PAPERLESS_ADMIN_PASSWORD \
+		PAPERLESS_REDIS; do
 		# Check for a version of this var with _FILE appended
 		# and convert the contents to the env var value
 		file_env ${env_var}
@@ -74,35 +100,44 @@ initialize() {
 	# Check for overrides of certain folders
 	map_folders
 
-	local export_dir="/usr/src/paperless/export"
+	local -r export_dir="/usr/src/paperless/export"
 
-	for dir in "${export_dir}" "${DATA_DIR}" "${DATA_DIR}/index" "${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails"; do
+	for dir in \
+		"${export_dir}" \
+		"${DATA_DIR}" "${DATA_DIR}/index" \
+		"${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails" \
+		"${CONSUME_DIR}"; do
 		if [[ ! -d "${dir}" ]]; then
 			echo "Creating directory ${dir}"
 			mkdir "${dir}"
 		fi
 	done
 
-	local tmp_dir="/tmp/paperless"
+	local -r tmp_dir="/tmp/paperless"
 	echo "Creating directory ${tmp_dir}"
 	mkdir -p "${tmp_dir}"
 
+	nltk_data
+
 	set +e
 	echo "Adjusting permissions of paperless files. This may take a while."
 	chown -R paperless:paperless ${tmp_dir}
-	for dir in "${export_dir}" "${DATA_DIR}" "${MEDIA_ROOT_DIR}"; do
+	for dir in \
+		"${export_dir}" \
+		"${DATA_DIR}" \
+		"${MEDIA_ROOT_DIR}" \
+		"${CONSUME_DIR}"; do
 		find "${dir}" -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
 	done
 	set -e
 
-	${gosu_cmd[@]} /sbin/docker-prepare.sh
+	"${gosu_cmd[@]}" /sbin/docker-prepare.sh
 }
 
 install_languages() {
 	echo "Installing languages..."
 
-	local langs="$1"
-	read -ra langs <<<"$langs"
+	read -ra langs <<<"$1"
 
 	# Check that it is not empty
 	if [ ${#langs[@]} -eq 0 ]; then
@@ -138,7 +173,7 @@ install_languages() {
 echo "Paperless-ngx docker container starting..."
 
 gosu_cmd=(gosu paperless)
-if [ $(id -u) == $(id -u paperless) ]; then
+if [ "$(id -u)" == "$(id -u paperless)" ]; then
 	gosu_cmd=()
 fi
 
@@ -151,7 +186,7 @@ initialize
 
 if [[ "$1" != "/"* ]]; then
 	echo Executing management command "$@"
-	exec ${gosu_cmd[@]} python3 manage.py "$@"
+	exec "${gosu_cmd[@]}" python3 manage.py "$@"
 else
 	echo Executing "$@"
 	exec "$@"

docker/docker-prepare.sh

@@ -4,12 +4,12 @@ set -e
 
 wait_for_postgres() {
 	local attempt_num=1
-	local max_attempts=5
+	local -r max_attempts=5
 
 	echo "Waiting for PostgreSQL to start..."
 
-	local host="${PAPERLESS_DBHOST:-localhost}"
-	local port="${PAPERLESS_DBPORT:-5432}"
+	local -r host="${PAPERLESS_DBHOST:-localhost}"
+	local -r port="${PAPERLESS_DBPORT:-5432}"
 
 	# Disable warning, host and port can't have spaces
 	# shellcheck disable=SC2086
@@ -28,6 +28,30 @@ wait_for_postgres() {
 	done
 }
 
+wait_for_mariadb() {
+	echo "Waiting for MariaDB to start..."
+
+	local -r host="${PAPERLESS_DBHOST:=localhost}"
+	local -r port="${PAPERLESS_DBPORT:=3306}"
+
+	local attempt_num=1
+	local -r max_attempts=5
+
+	while ! true > /dev/tcp/$host/$port; do
+
+		if [ $attempt_num -eq $max_attempts ]; then
+			echo "Unable to connect to database."
+			exit 1
+		else
+			echo "Attempt $attempt_num failed! Trying again in 5 seconds..."
+
+		fi
+
+		attempt_num=$(("$attempt_num" + 1))
+		sleep 5
+	done
+}
+
 wait_for_redis() {
 	# We use a Python script to send the Redis ping
 	# instead of installing redis-tools just for 1 thing
@@ -49,8 +73,8 @@ migrations() {
 
 search_index() {
 
-	local index_version=1
-	local index_version_file=${DATA_DIR}/.index_version
+	local -r index_version=1
+	local -r index_version_file=${DATA_DIR}/.index_version
 
 	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
 		echo "Search index out of date. Updating..."
@@ -65,8 +89,50 @@ superuser() {
 	fi
 }
 
+custom_container_init() {
+	# Mostly borrowed from the LinuxServer.io base image
+	# https://github.com/linuxserver/docker-baseimage-ubuntu/tree/bionic/root/etc/cont-init.d
+	local -r custom_script_dir="/custom-cont-init.d"
+	# Tamper checking.
+	# Don't run files which are owned by anyone except root
+	# Don't run files which are writeable by others
+	if [ -d "${custom_script_dir}" ]; then
+		if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 ! -user root)" ]; then
+			echo "**** Potential tampering with custom scripts detected ****"
+			echo "**** The folder '${custom_script_dir}' must be owned by root ****"
+			return 0
+		fi
+		if [ -n "$(/usr/bin/find "${custom_script_dir}" -maxdepth 1 -perm -o+w)" ]; then
+			echo "**** The folder '${custom_script_dir}' or some of contents have write permissions for others, which is a security risk. ****"
+			echo "**** Please review the permissions and their contents to make sure they are owned by root, and can only be modified by root. ****"
+			return 0
+		fi
+
+		# Make sure custom init directory has files in it
+		if [ -n "$(/bin/ls -A "${custom_script_dir}" 2>/dev/null)" ]; then
+			echo "[custom-init] files found in ${custom_script_dir} executing"
+			# Loop over files in the directory
+			for SCRIPT in "${custom_script_dir}"/*; do
+				NAME="$(basename "${SCRIPT}")"
+				if [ -f "${SCRIPT}" ]; then
+					echo "[custom-init] ${NAME}: executing..."
+					/bin/bash "${SCRIPT}"
+					echo "[custom-init] ${NAME}: exited $?"
+				elif [ ! -f "${SCRIPT}" ]; then
+					echo "[custom-init] ${NAME}: is not a file"
+				fi
+			done
+		else
+			echo "[custom-init] no custom files found exiting..."
+		fi
+
+	fi
+}
+
 do_work() {
-	if [[ -n "${PAPERLESS_DBHOST}" ]]; then
+	if [[ "${PAPERLESS_DBENGINE}" == "mariadb" ]]; then
+		wait_for_mariadb
+	elif [[ -n "${PAPERLESS_DBHOST}" ]]; then
 		wait_for_postgres
 	fi
 
@@ -78,6 +144,9 @@ do_work() {
 
 	superuser
 
+	# Leave this last thing
+	custom_container_init
+
 }
 
 do_work

docker/flower-conditional.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "Checking if we should start flower..."
+
+if [[ -n  "${PAPERLESS_ENABLE_FLOWER}" ]]; then
+	celery --app paperless flower
+fi

docker/paperless_cmd.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 rootless_args=()
-if [ $(id -u) == $(id -u paperless) ]; then
+if [ "$(id -u)" == "$(id -u paperless)" ]; then
 	rootless_args=(
 		--user
 		paperless
@@ -12,4 +12,4 @@ if [ $(id -u) == $(id -u paperless) ]; then
 	)
 fi
 
-/usr/local/bin/supervisord -c /etc/supervisord.conf ${rootless_args[@]}
+exec /usr/local/bin/supervisord -c /etc/supervisord.conf "${rootless_args[@]}"

docker/supervisord.conf

@@ -10,7 +10,7 @@ user=root
 [program:gunicorn]
 command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.asgi:application
 user=paperless
-
+priority = 1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
@@ -19,17 +19,41 @@ stderr_logfile_maxbytes=0
 [program:consumer]
 command=python3 manage.py document_consumer
 user=paperless
-
+stopsignal=INT
+priority = 20
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:scheduler]
-command=python3 manage.py qcluster
+[program:celery]
+
+command = celery --app paperless worker --loglevel INFO
 user=paperless
 stopasgroup = true
-
+stopwaitsecs = 60
+priority = 5
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:celery-beat]
+
+command = celery --app paperless beat --loglevel INFO
+user=paperless
+stopasgroup = true
+priority = 10
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:celery-flower]
+command = /usr/local/bin/flower-conditional.sh
+user = paperless
+startsecs = 0
+priority = 40
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr

docker/wait-for-redis.py

@@ -18,7 +18,7 @@ if __name__ == "__main__":
 
     REDIS_URL: Final[str] = os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")
 
-    print(f"Waiting for Redis: {REDIS_URL}", flush=True)
+    print(f"Waiting for Redis...", flush=True)
 
     attempt = 0
     with Redis.from_url(url=REDIS_URL) as client:
@@ -37,8 +37,8 @@ if __name__ == "__main__":
                 attempt += 1
 
     if attempt >= MAX_RETRY_COUNT:
-        print(f"Failed to connect to: {REDIS_URL}")
+        print(f"Failed to connect to redis using environment variable PAPERLESS_REDIS.")
         sys.exit(os.EX_UNAVAILABLE)
     else:
-        print(f"Connected to Redis broker: {REDIS_URL}")
+        print(f"Connected to Redis broker.")
         sys.exit(os.EX_OK)

docs/Dockerfile

@@ -1,17 +0,0 @@
-FROM python:3.5.1
-
-# Install Sphinx and Pygments
-RUN pip install --no-cache-dir Sphinx Pygments \
-  # Setup directories, copy data
-  && mkdir /build
-
-COPY . /build
-WORKDIR /build/docs
-
-# Build documentation
-RUN make html
-
-# Start webserver
-WORKDIR /build/docs/_build/html
-EXPOSE 8000/tcp
-CMD ["python3", "-m", "http.server"]

docs/_static/css/custom.css

@@ -439,7 +439,8 @@ a.image-reference img {
 }
 
 .rst-content code.literal,
-.rst-content tt.literal {
+.rst-content tt.literal,
+html.writer-html5 .rst-content dl.footnote code {
   border-color: var(--color-border);
   background-color: var(--color-border);
   color: var(--color-text-code-inline)

docs/administration.rst

@@ -35,13 +35,15 @@ Options available to docker installations:
     ``/var/lib/docker/volumes`` on the host and you need to be root in order
     to access them.
 
-    Paperless uses 3 volumes:
+    Paperless uses 4 volumes:
 
     *   ``paperless_media``: This is where your documents are stored.
     *   ``paperless_data``: This is where auxillary data is stored. This
         folder also contains the SQLite database, if you use it.
     *   ``paperless_pgdata``: Exists only if you use PostgreSQL and contains
         the database.
+    *   ``paperless_dbdata``: Exists only if you use MariaDB and contains
+        the database.
 
 Options available to bare-metal and non-docker installations:
 
@@ -49,7 +51,7 @@ Options available to bare-metal and non-docker installations:
     crashes at some point or your disk fails, you can simply copy the folder back
     into place and it works.
 
-    When using PostgreSQL, you'll also have to backup the database.
+    When using PostgreSQL or MariaDB, you'll also have to backup the database.
 
 .. _migrating-restoring:
 
@@ -310,6 +312,7 @@ there are tools for it.
     -c, --correspondent
     -T, --tags
     -t, --document_type
+    -s, --storage_path
     -i, --inbox-only
     --use-first
     -f, --overwrite
@@ -318,7 +321,7 @@ Run this after changing or adding matching rules. It'll loop over all
 of the documents in your database and attempt to match documents
 according to the new rules.
 
-Specify any combination of ``-c``, ``-T`` and ``-t`` to have the
+Specify any combination of ``-c``, ``-T``, ``-t`` and ``-s`` to have the
 retagger perform matching of the specified metadata type. If you don't
 specify any of these options, the document retagger won't do anything.
 
@@ -444,6 +447,14 @@ command:
 
 The command takes no arguments and processes all your mail accounts and rules.
 
+ .. note::
+
+    As of October 2022 Microsoft no longer supports IMAP authentication for Exchange
+    servers, thus Exchange is no longer supported until a solution is implemented in
+    the Python IMAP library used by Paperless. See  `learn.microsoft.com`_
+
+.. _learn.microsoft.com: https://learn.microsoft.com/en-us/exchange/clients-and-mobile-in-exchange-online/deprecation-of-basic-authentication-exchange-online
+
 .. _utilities-archiver:
 
 Creating archived documents

docs/advanced_usage.rst

@@ -149,6 +149,9 @@ which will in turn call `pdf2pdfocr.py`_ on your document, which will then
 overwrite the file with an OCR'd version of the file and exit.  At which point,
 the consumption process will begin with the newly modified file.
 
+The script's stdout and stderr will be logged line by line to the webserver log, along
+with the exit code of the script.
+
 .. _pdf2pdfocr.py: https://github.com/LeoFCardoso/pdf2pdfocr
 
 .. _advanced-post_consume_script:
@@ -171,12 +174,17 @@ into paperless. It receives the following environment variables:
 * ``DOCUMENT_THUMBNAIL_URL``
 * ``DOCUMENT_CORRESPONDENT``
 * ``DOCUMENT_TAGS``
+* ``DOCUMENT_ORIGINAL_FILENAME``
 
 The script can be in any language, but for a simple shell script
 example, you can take a look at `post-consumption-example.sh`_ in this project.
 
 The post consumption script cannot cancel the consumption process.
 
+The script's stdout and stderr will be logged line by line to the webserver log, along
+with the exit code of the script.
+
+
 Docker
 ------
 Assumed you have ``/home/foo/paperless-ngx/scripts/post-consumption-example.sh``.
@@ -217,7 +225,8 @@ using the identifier which it has assigned to each document. You will end up get
 files like ``0000123.pdf`` in your media directory. This isn't necessarily a bad
 thing, because you normally don't have to access these files manually. However, if
 you wish to name your files differently, you can do that by adjusting the
-``PAPERLESS_FILENAME_FORMAT`` configuration option.
+``PAPERLESS_FILENAME_FORMAT`` configuration option. Paperless adds the correct
+file extension e.g. ``.pdf``, ``.jpg`` automatically.
 
 This variable allows you to configure the filename (folders are allowed) using
 placeholders. For example, configuring this to
@@ -256,12 +265,18 @@ Paperless provides the following placeholders within filenames:
 * ``{tag_list}``: A comma separated list of all tags assigned to the document.
 * ``{title}``: The title of the document.
 * ``{created}``: The full date (ISO format) the document was created.
-* ``{created_year}``: Year created only.
+* ``{created_year}``: Year created only, formatted as the year with century.
+* ``{created_year_short}``: Year created only, formatted as the year without century, zero padded.
 * ``{created_month}``: Month created only (number 01-12).
+* ``{created_month_name}``: Month created name, as per locale
+* ``{created_month_name_short}``: Month created abbreviated name, as per locale
 * ``{created_day}``: Day created only (number 01-31).
 * ``{added}``: The full date (ISO format) the document was added to paperless.
 * ``{added_year}``: Year added only.
+* ``{added_year_short}``: Year added only, formatted as the year without century, zero padded.
 * ``{added_month}``: Month added only (number 01-12).
+* ``{added_month_name}``: Month added name, as per locale
+* ``{added_month_name_short}``: Month added abbreviated name, as per locale
 * ``{added_day}``: Day added only (number 01-31).
 
 
@@ -362,3 +377,71 @@ For simplicity, `By Year` defines the same structure as in the previous example
 
     If you adjust the format of an existing storage path, old documents don't get relocated automatically.
     You need to run the :ref:`document renamer <utilities-renamer>` to adjust their pathes.
+
+.. _advanced-celery-monitoring:
+
+Celery Monitoring
+#################
+
+The monitoring tool `Flower <https://flower.readthedocs.io/en/latest/index.html>`_ can be used to view more
+detailed information about the health of the celery workers used for asynchronous tasks.  This includes details
+on currently running, queued and completed tasks, timing and more.  Flower can also be used with Prometheus, as it
+exports metrics.  For details on its capabilities, refer to the Flower documentation.
+
+To configure Flower further, create a `flowerconfig.py` and place it into the `src/paperless` directory.  For
+a Docker installation, you can use volumes to accomplish this:
+
+.. code:: yaml
+
+    services:
+      # ...
+      webserver:
+        # ...
+        volumes:
+          - /path/to/my/flowerconfig.py:/usr/src/paperless/src/paperless/flowerconfig.py:ro
+
+Custom Container Initialization
+###############################
+
+The Docker image includes the ability to run custom user scripts during startup.  This could be
+utilized for installing additional tools or Python packages, for example.
+
+To utilize this, mount a folder containing your scripts to the custom initialization directory, `/custom-cont-init.d`
+and place scripts you wish to run inside.  For security, the folder and its contents must be owned by `root`.
+Additionally, scripts must only be writable by `root`.
+
+Your scripts will be run directly before the webserver completes startup.  Scripts will be run by the `root` user.
+This is an advanced functionality with which you could break functionality or lose data.
+
+For example, using Docker Compose:
+
+
+.. code:: yaml
+
+    services:
+      # ...
+      webserver:
+        # ...
+        volumes:
+          - /path/to/my/scripts:/custom-cont-init.d:ro
+
+.. _advanced-mysql-caveats:
+
+MySQL Caveats
+#############
+
+Case Sensitivity
+================
+
+The database interface does not provide a method to configure a MySQL database to
+be case sensitive.  This would prevent a user from creating a tag ``Name`` and ``NAME``
+as they are considered the same.
+
+Per Django documentation, to enable this requires manual intervention.  To enable
+case sensetive tables, you can execute the following command against each table:
+
+``ALTER TABLE <table_name> CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;``
+
+You can also set the default for new tables (this does NOT affect existing tables) with:
+
+``ALTER DATABASE <db_name> CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;``

docs/changelog.md

@@ -1,5 +1,335 @@
 # Changelog
 
+## paperless-ngx 1.9.2
+
+### Bug Fixes
+
+- Bugfix: Allow PAPERLESS_OCR_CLEAN=none [@shamoon](https://github.com/shamoon) ([#1670](https://github.com/paperless-ngx/paperless-ngx/pull/1670))
+
+### All App Changes
+
+- Chore: Bumps version numbers to 1.9.2 [@stumpylog](https://github.com/stumpylog) ([#1666](https://github.com/paperless-ngx/paperless-ngx/pull/1666))
+
+## paperless-ngx 1.9.1
+
+### Notes
+
+- Version 1.9.1 incorrectly displays the version string as 1.9.0
+
+### Bug Fixes
+
+- Bugfix: Fixes missing OCR mode skip_noarchive [@stumpylog](https://github.com/stumpylog) ([#1645](https://github.com/paperless-ngx/paperless-ngx/pull/1645))
+- Fix reset button padding on small screens [@shamoon](https://github.com/shamoon) ([#1646](https://github.com/paperless-ngx/paperless-ngx/pull/1646))
+
+### Documentation
+
+- Improve docs re [@janis-ax](https://github.com/janis-ax) ([#1625](https://github.com/paperless-ngx/paperless-ngx/pull/1625))
+- [Documentation] Add v1.9.0 changelog [@github-actions](https://github.com/github-actions) ([#1639](https://github.com/paperless-ngx/paperless-ngx/pull/1639))
+
+### All App Changes
+
+- Bugfix: Fixes missing OCR mode skip_noarchive [@stumpylog](https://github.com/stumpylog) ([#1645](https://github.com/paperless-ngx/paperless-ngx/pull/1645))
+- Fix reset button padding on small screens [@shamoon](https://github.com/shamoon) ([#1646](https://github.com/paperless-ngx/paperless-ngx/pull/1646))
+
+## paperless-ngx 1.9.0
+
+### Features
+
+- Feature: Faster, less memory barcode handling [@stumpylog](https://github.com/stumpylog) ([#1594](https://github.com/paperless-ngx/paperless-ngx/pull/1594))
+- Feature: Display django-q process names [@stumpylog](https://github.com/stumpylog) ([#1567](https://github.com/paperless-ngx/paperless-ngx/pull/1567))
+- Feature: Add MariaDB support [@bckelly1](https://github.com/bckelly1) ([#543](https://github.com/paperless-ngx/paperless-ngx/pull/543))
+- Feature: Simplify IMAP login for UTF-8 [@stumpylog](https://github.com/stumpylog) ([#1492](https://github.com/paperless-ngx/paperless-ngx/pull/1492))
+- Feature: Even better re-do of OCR [@stumpylog](https://github.com/stumpylog) ([#1451](https://github.com/paperless-ngx/paperless-ngx/pull/1451))
+- Feature: document comments [@tim-vogel](https://github.com/tim-vogel) ([#1375](https://github.com/paperless-ngx/paperless-ngx/pull/1375))
+- Adding date suggestions to the documents details view [@Eckii24](https://github.com/Eckii24) ([#1367](https://github.com/paperless-ngx/paperless-ngx/pull/1367))
+- Feature: Event driven consumer [@stumpylog](https://github.com/stumpylog) ([#1421](https://github.com/paperless-ngx/paperless-ngx/pull/1421))
+- Feature: Adds storage paths to re-tagger command [@stumpylog](https://github.com/stumpylog) ([#1446](https://github.com/paperless-ngx/paperless-ngx/pull/1446))
+- Feature: Preserve original filename in metadata [@GwynHannay](https://github.com/GwynHannay) ([#1440](https://github.com/paperless-ngx/paperless-ngx/pull/1440))
+- Handle tags for gmail email accounts [@sisao](https://github.com/sisao) ([#1433](https://github.com/paperless-ngx/paperless-ngx/pull/1433))
+- Update redis image [@tribut](https://github.com/tribut) ([#1436](https://github.com/paperless-ngx/paperless-ngx/pull/1436))
+- PAPERLESS_REDIS may be set via docker secrets [@DennisGaida](https://github.com/DennisGaida) ([#1405](https://github.com/paperless-ngx/paperless-ngx/pull/1405))
+
+### Bug Fixes
+
+- paperless_cmd.sh: use exec to run supervisord [@lemmi](https://github.com/lemmi) ([#1617](https://github.com/paperless-ngx/paperless-ngx/pull/1617))
+- Fix: Double barcode separation creates empty file [@stumpylog](https://github.com/stumpylog) ([#1596](https://github.com/paperless-ngx/paperless-ngx/pull/1596))
+- Fix: Resolve issue with slow classifier [@stumpylog](https://github.com/stumpylog) ([#1576](https://github.com/paperless-ngx/paperless-ngx/pull/1576))
+- Fix document comments not updating on document navigation [@shamoon](https://github.com/shamoon) ([#1566](https://github.com/paperless-ngx/paperless-ngx/pull/1566))
+- Fix: Include storage paths in document exporter [@shamoon](https://github.com/shamoon) ([#1557](https://github.com/paperless-ngx/paperless-ngx/pull/1557))
+- Chore: Cleanup and validate settings [@stumpylog](https://github.com/stumpylog) ([#1551](https://github.com/paperless-ngx/paperless-ngx/pull/1551))
+- Bugfix: Better gunicorn settings for workers [@stumpylog](https://github.com/stumpylog) ([#1500](https://github.com/paperless-ngx/paperless-ngx/pull/1500))
+- Fix actions button in tasks table [@shamoon](https://github.com/shamoon) ([#1488](https://github.com/paperless-ngx/paperless-ngx/pull/1488))
+- Fix: Add missing filter rule types to SavedViewFilterRule model \& fix migrations [@shamoon](https://github.com/shamoon) ([#1463](https://github.com/paperless-ngx/paperless-ngx/pull/1463))
+- Fix paperless.conf.example typo [@qcasey](https://github.com/qcasey) ([#1460](https://github.com/paperless-ngx/paperless-ngx/pull/1460))
+- Bugfix: Fixes the creation of an archive file, even if noarchive was specified [@stumpylog](https://github.com/stumpylog) ([#1442](https://github.com/paperless-ngx/paperless-ngx/pull/1442))
+- Fix: created_date should not be required [@shamoon](https://github.com/shamoon) ([#1412](https://github.com/paperless-ngx/paperless-ngx/pull/1412))
+- Fix: dev backend testing [@stumpylog](https://github.com/stumpylog) ([#1420](https://github.com/paperless-ngx/paperless-ngx/pull/1420))
+- Bugfix: Catch all exceptions during the task signals [@stumpylog](https://github.com/stumpylog) ([#1387](https://github.com/paperless-ngx/paperless-ngx/pull/1387))
+- Fix: saved view page parameter [@shamoon](https://github.com/shamoon) ([#1376](https://github.com/paperless-ngx/paperless-ngx/pull/1376))
+- Fix: Correct browser unsaved changes warning [@shamoon](https://github.com/shamoon) ([#1369](https://github.com/paperless-ngx/paperless-ngx/pull/1369))
+- Fix: correct date pasting with other formats [@shamoon](https://github.com/shamoon) ([#1370](https://github.com/paperless-ngx/paperless-ngx/pull/1370))
+- Bugfix: Allow webserver bind address to be configured [@stumpylog](https://github.com/stumpylog) ([#1358](https://github.com/paperless-ngx/paperless-ngx/pull/1358))
+- Bugfix: Chain exceptions during exception handling [@stumpylog](https://github.com/stumpylog) ([#1354](https://github.com/paperless-ngx/paperless-ngx/pull/1354))
+- Fix: missing tooltip translation \& filter editor wrapping [@shamoon](https://github.com/shamoon) ([#1305](https://github.com/paperless-ngx/paperless-ngx/pull/1305))
+- Bugfix: Interaction between barcode and directories as tags [@stumpylog](https://github.com/stumpylog) ([#1303](https://github.com/paperless-ngx/paperless-ngx/pull/1303))
+
+### Documentation
+
+- [Beta] Paperless-ngx v1.9.0 Release Candidate [@stumpylog](https://github.com/stumpylog) ([#1560](https://github.com/paperless-ngx/paperless-ngx/pull/1560))
+- docs/configuration: Fix binary variable defaults [@erikarvstedt](https://github.com/erikarvstedt) ([#1528](https://github.com/paperless-ngx/paperless-ngx/pull/1528))
+- Info about installing on subpath [@viktor-c](https://github.com/viktor-c) ([#1350](https://github.com/paperless-ngx/paperless-ngx/pull/1350))
+- Docs: move scanner \& software recs to GH wiki [@shamoon](https://github.com/shamoon) ([#1482](https://github.com/paperless-ngx/paperless-ngx/pull/1482))
+- Docs: Update mobile scanner section [@tooomm](https://github.com/tooomm) ([#1467](https://github.com/paperless-ngx/paperless-ngx/pull/1467))
+- Adding date suggestions to the documents details view [@Eckii24](https://github.com/Eckii24) ([#1367](https://github.com/paperless-ngx/paperless-ngx/pull/1367))
+- docs: scanners: add Brother ads4700w [@ocelotsloth](https://github.com/ocelotsloth) ([#1450](https://github.com/paperless-ngx/paperless-ngx/pull/1450))
+- Feature: Adds storage paths to re-tagger command [@stumpylog](https://github.com/stumpylog) ([#1446](https://github.com/paperless-ngx/paperless-ngx/pull/1446))
+- Changes to Redis documentation [@Zerteax](https://github.com/Zerteax) ([#1441](https://github.com/paperless-ngx/paperless-ngx/pull/1441))
+- Update scanners.rst [@glassbox-sco](https://github.com/glassbox-sco) ([#1430](https://github.com/paperless-ngx/paperless-ngx/pull/1430))
+- Update scanners.rst [@derlucas](https://github.com/derlucas) ([#1415](https://github.com/paperless-ngx/paperless-ngx/pull/1415))
+- Bugfix: Allow webserver bind address to be configured [@stumpylog](https://github.com/stumpylog) ([#1358](https://github.com/paperless-ngx/paperless-ngx/pull/1358))
+- docs: fix small typo [@tooomm](https://github.com/tooomm) ([#1352](https://github.com/paperless-ngx/paperless-ngx/pull/1352))
+- [Documentation] Add v1.8.0 changelog [@github-actions](https://github.com/github-actions) ([#1298](https://github.com/paperless-ngx/paperless-ngx/pull/1298))
+
+### Maintenance
+
+- [Beta] Paperless-ngx v1.9.0 Release Candidate [@stumpylog](https://github.com/stumpylog) ([#1560](https://github.com/paperless-ngx/paperless-ngx/pull/1560))
+- paperless_cmd.sh: use exec to run supervisord [@lemmi](https://github.com/lemmi) ([#1617](https://github.com/paperless-ngx/paperless-ngx/pull/1617))
+- Chore: Extended container image cleanup [@stumpylog](https://github.com/stumpylog) ([#1556](https://github.com/paperless-ngx/paperless-ngx/pull/1556))
+- Chore: Smaller library images [@stumpylog](https://github.com/stumpylog) ([#1546](https://github.com/paperless-ngx/paperless-ngx/pull/1546))
+- Bump tj-actions/changed-files from 24 to 29.0.2 [@dependabot](https://github.com/dependabot) ([#1493](https://github.com/paperless-ngx/paperless-ngx/pull/1493))
+- Bugfix: Better gunicorn settings for workers [@stumpylog](https://github.com/stumpylog) ([#1500](https://github.com/paperless-ngx/paperless-ngx/pull/1500))
+- [CI] Fix release drafter issues [@qcasey](https://github.com/qcasey) ([#1301](https://github.com/paperless-ngx/paperless-ngx/pull/1301))
+- Fix: dev backend testing [@stumpylog](https://github.com/stumpylog) ([#1420](https://github.com/paperless-ngx/paperless-ngx/pull/1420))
+- Chore: Exclude dependabot PRs from Project, set status to Needs Review [@qcasey](https://github.com/qcasey) ([#1397](https://github.com/paperless-ngx/paperless-ngx/pull/1397))
+- Chore: Add to label PRs based on and title [@qcasey](https://github.com/qcasey) ([#1396](https://github.com/paperless-ngx/paperless-ngx/pull/1396))
+- Chore: use pre-commit in the Ci workflow [@stumpylog](https://github.com/stumpylog) ([#1362](https://github.com/paperless-ngx/paperless-ngx/pull/1362))
+- Chore: Fixes permissions for image tag cleanup [@stumpylog](https://github.com/stumpylog) ([#1315](https://github.com/paperless-ngx/paperless-ngx/pull/1315))
+- Bump leonsteinhaeuser/project-beta-automations from 1.2.1 to 1.3.0 [@dependabot](https://github.com/dependabot) ([#1328](https://github.com/paperless-ngx/paperless-ngx/pull/1328))
+- Bump tj-actions/changed-files from 23.1 to 24 [@dependabot](https://github.com/dependabot) ([#1329](https://github.com/paperless-ngx/paperless-ngx/pull/1329))
+- Feature: Remove requirements.txt and use pipenv everywhere [@stumpylog](https://github.com/stumpylog) ([#1316](https://github.com/paperless-ngx/paperless-ngx/pull/1316))
+
+### Dependencies
+
+<details>
+<summary>34 changes</summary>
+
+- Bump pikepdf from 5.5.0 to 5.6.1 [@dependabot](https://github.com/dependabot) ([#1537](https://github.com/paperless-ngx/paperless-ngx/pull/1537))
+- Bump black from 22.6.0 to 22.8.0 [@dependabot](https://github.com/dependabot) ([#1539](https://github.com/paperless-ngx/paperless-ngx/pull/1539))
+- Bump tqdm from 4.64.0 to 4.64.1 [@dependabot](https://github.com/dependabot) ([#1540](https://github.com/paperless-ngx/paperless-ngx/pull/1540))
+- Bump pytest from 7.1.2 to 7.1.3 [@dependabot](https://github.com/dependabot) ([#1538](https://github.com/paperless-ngx/paperless-ngx/pull/1538))
+- Bump tj-actions/changed-files from 24 to 29.0.2 [@dependabot](https://github.com/dependabot) ([#1493](https://github.com/paperless-ngx/paperless-ngx/pull/1493))
+- Bump angular packages, jest-preset-angular in src-ui [@dependabot](https://github.com/dependabot) ([#1502](https://github.com/paperless-ngx/paperless-ngx/pull/1502))
+- Bump jest-environment-jsdom from 28.1.3 to 29.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1507](https://github.com/paperless-ngx/paperless-ngx/pull/1507))
+- Bump [@<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot) ([#1506](https://github.com/paperless-ngx/paperless-ngx/pull/1506))
+- Bump [@<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot](https://github.com/<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot) ([#1505](https://github.com/paperless-ngx/paperless-ngx/pull/1505))
+- Bump zone.js from 0.11.7 to 0.11.8 in /src-ui [@dependabot](https://github.com/dependabot) ([#1504](https://github.com/paperless-ngx/paperless-ngx/pull/1504))
+- Bump ngx-color from 8.0.1 to 8.0.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1494](https://github.com/paperless-ngx/paperless-ngx/pull/1494))
+- Bump cypress from 10.3.1 to 10.7.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1496](https://github.com/paperless-ngx/paperless-ngx/pull/1496))
+- Bump [@<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot](https://github.com/<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot) ([#1495](https://github.com/paperless-ngx/paperless-ngx/pull/1495))
+- Bump [@<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot](https://github.com/<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot) ([#1498](https://github.com/paperless-ngx/paperless-ngx/pull/1498))
+- Bump sphinx from 5.0.2 to 5.1.1 [@dependabot](https://github.com/dependabot) ([#1297](https://github.com/paperless-ngx/paperless-ngx/pull/1297))
+- Chore: Bump Python dependencies [@stumpylog](https://github.com/stumpylog) ([#1445](https://github.com/paperless-ngx/paperless-ngx/pull/1445))
+- Chore: Update Python deps [@stumpylog](https://github.com/stumpylog) ([#1391](https://github.com/paperless-ngx/paperless-ngx/pull/1391))
+- Bump watchfiles from 0.15.0 to 0.16.1 [@dependabot](https://github.com/dependabot) ([#1285](https://github.com/paperless-ngx/paperless-ngx/pull/1285))
+- Bump leonsteinhaeuser/project-beta-automations from 1.2.1 to 1.3.0 [@dependabot](https://github.com/dependabot) ([#1328](https://github.com/paperless-ngx/paperless-ngx/pull/1328))
+- Bump tj-actions/changed-files from 23.1 to 24 [@dependabot](https://github.com/dependabot) ([#1329](https://github.com/paperless-ngx/paperless-ngx/pull/1329))
+- Bump cypress from 10.3.0 to 10.3.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1342](https://github.com/paperless-ngx/paperless-ngx/pull/1342))
+- Bump ngx-color from 7.3.3 to 8.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1343](https://github.com/paperless-ngx/paperless-ngx/pull/1343))
+- Bump [@<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot](https://github.com/<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot) ([#1330](https://github.com/paperless-ngx/paperless-ngx/pull/1330))
+- Bump [@<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot) ([#1341](https://github.com/paperless-ngx/paperless-ngx/pull/1341))
+- Bump jest-preset-angular from 12.1.0 to 12.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1340](https://github.com/paperless-ngx/paperless-ngx/pull/1340))
+- Bump concurrently from 7.2.2 to 7.3.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1326](https://github.com/paperless-ngx/paperless-ngx/pull/1326))
+- Bump ng2-pdf-viewer from 9.0.0 to 9.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1337](https://github.com/paperless-ngx/paperless-ngx/pull/1337))
+- Bump jest-environment-jsdom from 28.1.2 to 28.1.3 in /src-ui [@dependabot](https://github.com/dependabot) ([#1336](https://github.com/paperless-ngx/paperless-ngx/pull/1336))
+- Bump ngx-file-drop from 13.0.0 to 14.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1331](https://github.com/paperless-ngx/paperless-ngx/pull/1331))
+- Bump jest and [@<!---->types/jest in /src-ui @dependabot](https://github.com/<!---->types/jest in /src-ui @dependabot) ([#1333](https://github.com/paperless-ngx/paperless-ngx/pull/1333))
+- Bump bootstrap from 5.1.3 to 5.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1327](https://github.com/paperless-ngx/paperless-ngx/pull/1327))
+- Bump typescript from 4.6.4 to 4.7.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1324](https://github.com/paperless-ngx/paperless-ngx/pull/1324))
+- Bump ts-node from 10.8.1 to 10.9.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1325](https://github.com/paperless-ngx/paperless-ngx/pull/1325))
+- Bump rxjs from 7.5.5 to 7.5.6 in /src-ui [@dependabot](https://github.com/dependabot) ([#1323](https://github.com/paperless-ngx/paperless-ngx/pull/1323))
+</details>
+
+### All App Changes
+
+- [Beta] Paperless-ngx v1.9.0 Release Candidate [@stumpylog](https://github.com/stumpylog) ([#1560](https://github.com/paperless-ngx/paperless-ngx/pull/1560))
+- Feature: Faster, less memory barcode handling [@stumpylog](https://github.com/stumpylog) ([#1594](https://github.com/paperless-ngx/paperless-ngx/pull/1594))
+- Fix: Consume directory permissions were not updated [@stumpylog](https://github.com/stumpylog) ([#1605](https://github.com/paperless-ngx/paperless-ngx/pull/1605))
+- Fix: Double barcode separation creates empty file [@stumpylog](https://github.com/stumpylog) ([#1596](https://github.com/paperless-ngx/paperless-ngx/pull/1596))
+- Fix: Parsing Tika documents fails with AttributeError [@stumpylog](https://github.com/stumpylog) ([#1591](https://github.com/paperless-ngx/paperless-ngx/pull/1591))
+- Fix: Resolve issue with slow classifier [@stumpylog](https://github.com/stumpylog) ([#1576](https://github.com/paperless-ngx/paperless-ngx/pull/1576))
+- Feature: Display django-q process names [@stumpylog](https://github.com/stumpylog) ([#1567](https://github.com/paperless-ngx/paperless-ngx/pull/1567))
+- Fix document comments not updating on document navigation [@shamoon](https://github.com/shamoon) ([#1566](https://github.com/paperless-ngx/paperless-ngx/pull/1566))
+- Feature: Add MariaDB support [@bckelly1](https://github.com/bckelly1) ([#543](https://github.com/paperless-ngx/paperless-ngx/pull/543))
+- Fix: Include storage paths in document exporter [@shamoon](https://github.com/shamoon) ([#1557](https://github.com/paperless-ngx/paperless-ngx/pull/1557))
+- Chore: Cleanup and validate settings [@stumpylog](https://github.com/stumpylog) ([#1551](https://github.com/paperless-ngx/paperless-ngx/pull/1551))
+- Bump pikepdf from 5.5.0 to 5.6.1 [@dependabot](https://github.com/dependabot) ([#1537](https://github.com/paperless-ngx/paperless-ngx/pull/1537))
+- Bump black from 22.6.0 to 22.8.0 [@dependabot](https://github.com/dependabot) ([#1539](https://github.com/paperless-ngx/paperless-ngx/pull/1539))
+- Bump tqdm from 4.64.0 to 4.64.1 [@dependabot](https://github.com/dependabot) ([#1540](https://github.com/paperless-ngx/paperless-ngx/pull/1540))
+- Bump pytest from 7.1.2 to 7.1.3 [@dependabot](https://github.com/dependabot) ([#1538](https://github.com/paperless-ngx/paperless-ngx/pull/1538))
+- Bump angular packages, jest-preset-angular in src-ui [@dependabot](https://github.com/dependabot) ([#1502](https://github.com/paperless-ngx/paperless-ngx/pull/1502))
+- Bump jest-environment-jsdom from 28.1.3 to 29.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1507](https://github.com/paperless-ngx/paperless-ngx/pull/1507))
+- Bump [@<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.6.3 to 18.7.14 in /src-ui @dependabot) ([#1506](https://github.com/paperless-ngx/paperless-ngx/pull/1506))
+- Bump [@<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot](https://github.com/<!---->angular-builders/jest from 14.0.0 to 14.0.1 in /src-ui @dependabot) ([#1505](https://github.com/paperless-ngx/paperless-ngx/pull/1505))
+- Bump zone.js from 0.11.7 to 0.11.8 in /src-ui [@dependabot](https://github.com/dependabot) ([#1504](https://github.com/paperless-ngx/paperless-ngx/pull/1504))
+- Bump ngx-color from 8.0.1 to 8.0.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1494](https://github.com/paperless-ngx/paperless-ngx/pull/1494))
+- Bump cypress from 10.3.1 to 10.7.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1496](https://github.com/paperless-ngx/paperless-ngx/pull/1496))
+- Bump [@<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot](https://github.com/<!---->cypress/schematic from 2.0.0 to 2.1.1 in /src-ui @dependabot) ([#1495](https://github.com/paperless-ngx/paperless-ngx/pull/1495))
+- Bump [@<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot](https://github.com/<!---->popperjs/core from 2.11.5 to 2.11.6 in /src-ui @dependabot) ([#1498](https://github.com/paperless-ngx/paperless-ngx/pull/1498))
+- Feature: Simplify IMAP login for UTF-8 [@stumpylog](https://github.com/stumpylog) ([#1492](https://github.com/paperless-ngx/paperless-ngx/pull/1492))
+- Fix actions button in tasks table [@shamoon](https://github.com/shamoon) ([#1488](https://github.com/paperless-ngx/paperless-ngx/pull/1488))
+- Fix: Add missing filter rule types to SavedViewFilterRule model \& fix migrations [@shamoon](https://github.com/shamoon) ([#1463](https://github.com/paperless-ngx/paperless-ngx/pull/1463))
+- Feature: Even better re-do of OCR [@stumpylog](https://github.com/stumpylog) ([#1451](https://github.com/paperless-ngx/paperless-ngx/pull/1451))
+- Feature: document comments [@tim-vogel](https://github.com/tim-vogel) ([#1375](https://github.com/paperless-ngx/paperless-ngx/pull/1375))
+- Adding date suggestions to the documents details view [@Eckii24](https://github.com/Eckii24) ([#1367](https://github.com/paperless-ngx/paperless-ngx/pull/1367))
+- Bump sphinx from 5.0.2 to 5.1.1 [@dependabot](https://github.com/dependabot) ([#1297](https://github.com/paperless-ngx/paperless-ngx/pull/1297))
+- Feature: Event driven consumer [@stumpylog](https://github.com/stumpylog) ([#1421](https://github.com/paperless-ngx/paperless-ngx/pull/1421))
+- Bugfix: Fixes the creation of an archive file, even if noarchive was specified [@stumpylog](https://github.com/stumpylog) ([#1442](https://github.com/paperless-ngx/paperless-ngx/pull/1442))
+- Feature: Adds storage paths to re-tagger command [@stumpylog](https://github.com/stumpylog) ([#1446](https://github.com/paperless-ngx/paperless-ngx/pull/1446))
+- Feature: Preserve original filename in metadata [@GwynHannay](https://github.com/GwynHannay) ([#1440](https://github.com/paperless-ngx/paperless-ngx/pull/1440))
+- Handle tags for gmail email accounts [@sisao](https://github.com/sisao) ([#1433](https://github.com/paperless-ngx/paperless-ngx/pull/1433))
+- Fix: should not be required [@shamoon](https://github.com/shamoon) ([#1412](https://github.com/paperless-ngx/paperless-ngx/pull/1412))
+- Bugfix: Catch all exceptions during the task signals [@stumpylog](https://github.com/stumpylog) ([#1387](https://github.com/paperless-ngx/paperless-ngx/pull/1387))
+- Fix: saved view page parameter [@shamoon](https://github.com/shamoon) ([#1376](https://github.com/paperless-ngx/paperless-ngx/pull/1376))
+- Fix: Correct browser unsaved changes warning [@shamoon](https://github.com/shamoon) ([#1369](https://github.com/paperless-ngx/paperless-ngx/pull/1369))
+- Fix: correct date pasting with other formats [@shamoon](https://github.com/shamoon) ([#1370](https://github.com/paperless-ngx/paperless-ngx/pull/1370))
+- Chore: use pre-commit in the Ci workflow [@stumpylog](https://github.com/stumpylog) ([#1362](https://github.com/paperless-ngx/paperless-ngx/pull/1362))
+- Bugfix: Chain exceptions during exception handling [@stumpylog](https://github.com/stumpylog) ([#1354](https://github.com/paperless-ngx/paperless-ngx/pull/1354))
+- Bump watchfiles from 0.15.0 to 0.16.1 [@dependabot](https://github.com/dependabot) ([#1285](https://github.com/paperless-ngx/paperless-ngx/pull/1285))
+- Bump cypress from 10.3.0 to 10.3.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1342](https://github.com/paperless-ngx/paperless-ngx/pull/1342))
+- Bump ngx-color from 7.3.3 to 8.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1343](https://github.com/paperless-ngx/paperless-ngx/pull/1343))
+- Bump [@<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot](https://github.com/<!---->angular/cli from 14.0.4 to 14.1.0 in /src-ui @dependabot) ([#1330](https://github.com/paperless-ngx/paperless-ngx/pull/1330))
+- Bump [@<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot](https://github.com/<!---->types/node from 18.0.0 to 18.6.3 in /src-ui @dependabot) ([#1341](https://github.com/paperless-ngx/paperless-ngx/pull/1341))
+- Bump jest-preset-angular from 12.1.0 to 12.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1340](https://github.com/paperless-ngx/paperless-ngx/pull/1340))
+- Bump concurrently from 7.2.2 to 7.3.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1326](https://github.com/paperless-ngx/paperless-ngx/pull/1326))
+- Bump ng2-pdf-viewer from 9.0.0 to 9.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1337](https://github.com/paperless-ngx/paperless-ngx/pull/1337))
+- Bump jest-environment-jsdom from 28.1.2 to 28.1.3 in /src-ui [@dependabot](https://github.com/dependabot) ([#1336](https://github.com/paperless-ngx/paperless-ngx/pull/1336))
+- Bump ngx-file-drop from 13.0.0 to 14.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1331](https://github.com/paperless-ngx/paperless-ngx/pull/1331))
+- Bump jest and [@<!---->types/jest in /src-ui @dependabot](https://github.com/<!---->types/jest in /src-ui @dependabot) ([#1333](https://github.com/paperless-ngx/paperless-ngx/pull/1333))
+- Bump bootstrap from 5.1.3 to 5.2.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1327](https://github.com/paperless-ngx/paperless-ngx/pull/1327))
+- Bump typescript from 4.6.4 to 4.7.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1324](https://github.com/paperless-ngx/paperless-ngx/pull/1324))
+- Bump ts-node from 10.8.1 to 10.9.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1325](https://github.com/paperless-ngx/paperless-ngx/pull/1325))
+- Bump rxjs from 7.5.5 to 7.5.6 in /src-ui [@dependabot](https://github.com/dependabot) ([#1323](https://github.com/paperless-ngx/paperless-ngx/pull/1323))
+- Fix: missing tooltip translation \& filter editor wrapping [@shamoon](https://github.com/shamoon) ([#1305](https://github.com/paperless-ngx/paperless-ngx/pull/1305))
+- Feature: Remove requirements.txt and use pipenv everywhere [@stumpylog](https://github.com/stumpylog) ([#1316](https://github.com/paperless-ngx/paperless-ngx/pull/1316))
+- Bugfix: Interaction between barcode and directories as tags [@stumpylog](https://github.com/stumpylog) ([#1303](https://github.com/paperless-ngx/paperless-ngx/pull/1303))
+
+## paperless-ngx 1.8.0
+
+### Features
+
+- Feature use env vars in pre post scripts [@ziprandom](https://github.com/ziprandom) ([#1154](https://github.com/paperless-ngx/paperless-ngx/pull/1154))
+- frontend task queue [@shamoon](https://github.com/shamoon) ([#1020](https://github.com/paperless-ngx/paperless-ngx/pull/1020))
+- Fearless scikit-learn updates [@stumpylog](https://github.com/stumpylog) ([#1082](https://github.com/paperless-ngx/paperless-ngx/pull/1082))
+- Adds support for Docker secrets [@stumpylog](https://github.com/stumpylog) ([#1034](https://github.com/paperless-ngx/paperless-ngx/pull/1034))
+- make frontend timezone un-aware [@shamoon](https://github.com/shamoon) ([#957](https://github.com/paperless-ngx/paperless-ngx/pull/957))
+- Change document thumbnails to WebP [@stumpylog](https://github.com/stumpylog) ([#1127](https://github.com/paperless-ngx/paperless-ngx/pull/1127))
+- Fork django-q to update dependencies [@stumpylog](https://github.com/stumpylog) ([#1014](https://github.com/paperless-ngx/paperless-ngx/pull/1014))
+- Fix: Rework query params logic [@shamoon](https://github.com/shamoon) ([#1000](https://github.com/paperless-ngx/paperless-ngx/pull/1000))
+- Enhancement: show note on language change and offer reload [@shamoon](https://github.com/shamoon) ([#1030](https://github.com/paperless-ngx/paperless-ngx/pull/1030))
+- Include error information when Redis connection fails [@stumpylog](https://github.com/stumpylog) ([#1016](https://github.com/paperless-ngx/paperless-ngx/pull/1016))
+- frontend settings saved to database [@shamoon](https://github.com/shamoon) ([#919](https://github.com/paperless-ngx/paperless-ngx/pull/919))
+- Add "Created" as additional (optional) parameter for post_documents [@eingemaischt](https://github.com/eingemaischt) ([#965](https://github.com/paperless-ngx/paperless-ngx/pull/965))
+- Convert Changelog to markdown, auto-commit future changelogs [@qcasey](https://github.com/qcasey) ([#935](https://github.com/paperless-ngx/paperless-ngx/pull/935))
+- allow all ASN filtering functions [@shamoon](https://github.com/shamoon) ([#920](https://github.com/paperless-ngx/paperless-ngx/pull/920))
+- gunicorn: Allow IPv6 sockets [@vlcty](https://github.com/vlcty) ([#924](https://github.com/paperless-ngx/paperless-ngx/pull/924))
+- initial app loading indicators [@shamoon](https://github.com/shamoon) ([#899](https://github.com/paperless-ngx/paperless-ngx/pull/899))
+
+### Bug Fixes
+
+- Fix: dropdown selected items not visible again [@shamoon](https://github.com/shamoon) ([#1261](https://github.com/paperless-ngx/paperless-ngx/pull/1261))
+- [CI] Fix automatic changelog generation on release [@qcasey](https://github.com/qcasey) ([#1249](https://github.com/paperless-ngx/paperless-ngx/pull/1249))
+- Fix: Prevent duplicate api calls on text filtering [@shamoon](https://github.com/shamoon) ([#1133](https://github.com/paperless-ngx/paperless-ngx/pull/1133))
+- make frontend timezone un-aware [@shamoon](https://github.com/shamoon) ([#957](https://github.com/paperless-ngx/paperless-ngx/pull/957))
+- Feature / fix quick toggleable filters [@shamoon](https://github.com/shamoon) ([#1122](https://github.com/paperless-ngx/paperless-ngx/pull/1122))
+- Chore: Manually downgrade reportlab (and update everything else) [@stumpylog](https://github.com/stumpylog) ([#1116](https://github.com/paperless-ngx/paperless-ngx/pull/1116))
+- Bugfix: Don't assume default Docker folders [@stumpylog](https://github.com/stumpylog) ([#1088](https://github.com/paperless-ngx/paperless-ngx/pull/1088))
+- Bugfix: Better sanity check messages [@stumpylog](https://github.com/stumpylog) ([#1049](https://github.com/paperless-ngx/paperless-ngx/pull/1049))
+- Fix vertical margins between pages of pdf viewer [@shamoon](https://github.com/shamoon) ([#1081](https://github.com/paperless-ngx/paperless-ngx/pull/1081))
+- Bugfix: Pass debug setting on to django-q [@stumpylog](https://github.com/stumpylog) ([#1058](https://github.com/paperless-ngx/paperless-ngx/pull/1058))
+- Bugfix: Don't assume the document has a title set [@stumpylog](https://github.com/stumpylog) ([#1057](https://github.com/paperless-ngx/paperless-ngx/pull/1057))
+- Bugfix: Corrects the setting of max pixel size for OCR [@stumpylog](https://github.com/stumpylog) ([#1008](https://github.com/paperless-ngx/paperless-ngx/pull/1008))
+- better date pasting [@shamoon](https://github.com/shamoon) ([#1007](https://github.com/paperless-ngx/paperless-ngx/pull/1007))
+- Enhancement: Alphabetize tags by default [@shamoon](https://github.com/shamoon) ([#1017](https://github.com/paperless-ngx/paperless-ngx/pull/1017))
+- Fix: Rework query params logic [@shamoon](https://github.com/shamoon) ([#1000](https://github.com/paperless-ngx/paperless-ngx/pull/1000))
+- Fix: add translation for some un-translated tooltips [@shamoon](https://github.com/shamoon) ([#995](https://github.com/paperless-ngx/paperless-ngx/pull/995))
+- Change npm --no-optional to --omit=optional [@shamoon](https://github.com/shamoon) ([#986](https://github.com/paperless-ngx/paperless-ngx/pull/986))
+- Add `myst-parser` to fix readthedocs [@qcasey](https://github.com/qcasey) ([#982](https://github.com/paperless-ngx/paperless-ngx/pull/982))
+- Fix: Title is changed after switching doc quickly [@shamoon](https://github.com/shamoon) ([#979](https://github.com/paperless-ngx/paperless-ngx/pull/979))
+- Fix: warn when closing a document with unsaved changes due to max open docs [@shamoon](https://github.com/shamoon) ([#956](https://github.com/paperless-ngx/paperless-ngx/pull/956))
+- Bugfix: Adds configurable intoify debounce time [@stumpylog](https://github.com/stumpylog) ([#953](https://github.com/paperless-ngx/paperless-ngx/pull/953))
+- Bugfix: Fixes document filename date off by 1 issue [@stumpylog](https://github.com/stumpylog) ([#942](https://github.com/paperless-ngx/paperless-ngx/pull/942))
+- fixes #<!---->949: change to MIME detection for files [@gador](https://github.com/gador) ([#962](https://github.com/paperless-ngx/paperless-ngx/pull/962))
+- docs: fix some typos [@Berjou](https://github.com/Berjou) ([#948](https://github.com/paperless-ngx/paperless-ngx/pull/948))
+- [Docs] Fix 2 small typos [@tooomm](https://github.com/tooomm) ([#946](https://github.com/paperless-ngx/paperless-ngx/pull/946))
+- [Readme] Fix typo [@tooomm](https://github.com/tooomm) ([#941](https://github.com/paperless-ngx/paperless-ngx/pull/941))
+- Fix: management pages plurals incorrect in other languages [@shamoon](https://github.com/shamoon) ([#939](https://github.com/paperless-ngx/paperless-ngx/pull/939))
+- Fix: v1.7.1 frontend visual fixes [@shamoon](https://github.com/shamoon) ([#933](https://github.com/paperless-ngx/paperless-ngx/pull/933))
+- Fix: unassigned query params ignored [@shamoon](https://github.com/shamoon) ([#930](https://github.com/paperless-ngx/paperless-ngx/pull/930))
+- Fix: allow commas in non-multi rules query params [@shamoon](https://github.com/shamoon) ([#923](https://github.com/paperless-ngx/paperless-ngx/pull/923))
+- Fix: Include version in export for better error messages [@stumpylog](https://github.com/stumpylog) ([#883](https://github.com/paperless-ngx/paperless-ngx/pull/883))
+- Bugfix: Superuser Management Won't Reset Password [@stumpylog](https://github.com/stumpylog) ([#903](https://github.com/paperless-ngx/paperless-ngx/pull/903))
+- Fix Ignore Date Parsing [@stumpylog](https://github.com/stumpylog) ([#721](https://github.com/paperless-ngx/paperless-ngx/pull/721))
+
+### Documentation
+
+- Feature use env vars in pre post scripts [@ziprandom](https://github.com/ziprandom) ([#1154](https://github.com/paperless-ngx/paperless-ngx/pull/1154))
+- Add `myst-parser` to fix readthedocs [@qcasey](https://github.com/qcasey) ([#982](https://github.com/paperless-ngx/paperless-ngx/pull/982))
+- Add "Created" as additional (optional) parameter for post_documents [@eingemaischt](https://github.com/eingemaischt) ([#965](https://github.com/paperless-ngx/paperless-ngx/pull/965))
+- Bugfix: Adds configurable intoify debounce time [@stumpylog](https://github.com/stumpylog) ([#953](https://github.com/paperless-ngx/paperless-ngx/pull/953))
+- docs: fix some typos [@Berjou](https://github.com/Berjou) ([#948](https://github.com/paperless-ngx/paperless-ngx/pull/948))
+- [Docs] Fix 2 small typos [@tooomm](https://github.com/tooomm) ([#946](https://github.com/paperless-ngx/paperless-ngx/pull/946))
+- Convert Changelog to markdown, auto-commit future changelogs [@qcasey](https://github.com/qcasey) ([#935](https://github.com/paperless-ngx/paperless-ngx/pull/935))
+- [Readme] Fix typo [@tooomm](https://github.com/tooomm) ([#941](https://github.com/paperless-ngx/paperless-ngx/pull/941))
+
+### Maintenance
+
+- Adds support for Docker secrets [@stumpylog](https://github.com/stumpylog) ([#1034](https://github.com/paperless-ngx/paperless-ngx/pull/1034))
+- Bugfix: Don't assume default Docker folders [@stumpylog](https://github.com/stumpylog) ([#1088](https://github.com/paperless-ngx/paperless-ngx/pull/1088))
+- Include error information when Redis connection fails [@stumpylog](https://github.com/stumpylog) ([#1016](https://github.com/paperless-ngx/paperless-ngx/pull/1016))
+- Fix: add translation for some un-translated tooltips [@shamoon](https://github.com/shamoon) ([#995](https://github.com/paperless-ngx/paperless-ngx/pull/995))
+- gunicorn: Allow IPv6 sockets [@vlcty](https://github.com/vlcty) ([#924](https://github.com/paperless-ngx/paperless-ngx/pull/924))
+
+### Dependencies
+
+<details>
+<summary>34 changes</summary>
+
+- Fearless scikit-learn updates [@stumpylog](https://github.com/stumpylog) ([#1082](https://github.com/paperless-ngx/paperless-ngx/pull/1082))
+- Bump pillow from 9.1.1 to 9.2.0 [@dependabot](https://github.com/dependabot) ([#1193](https://github.com/paperless-ngx/paperless-ngx/pull/1193))
+- Bump watchdog from 2.1.8 to 2.1.9 [@dependabot](https://github.com/dependabot) ([#1132](https://github.com/paperless-ngx/paperless-ngx/pull/1132))
+- Bump scikit-learn from 1.0.2 to 1.1.1 [@dependabot](https://github.com/dependabot) ([#992](https://github.com/paperless-ngx/paperless-ngx/pull/992))
+- Bump setuptools from 62.3.3 to 62.6.0 [@dependabot](https://github.com/dependabot) ([#1150](https://github.com/paperless-ngx/paperless-ngx/pull/1150))
+- Bump django-filter from 21.1 to 22.1 [@dependabot](https://github.com/dependabot) ([#1191](https://github.com/paperless-ngx/paperless-ngx/pull/1191))
+- Bump actions/setup-python from 3 to 4 [@dependabot](https://github.com/dependabot) ([#1176](https://github.com/paperless-ngx/paperless-ngx/pull/1176))
+- Bump sphinx from 4.5.0 to 5.0.2 [@dependabot](https://github.com/dependabot) ([#1151](https://github.com/paperless-ngx/paperless-ngx/pull/1151))
+- Bump docker/metadata-action from 3 to 4 [@dependabot](https://github.com/dependabot) ([#1178](https://github.com/paperless-ngx/paperless-ngx/pull/1178))
+- Bump tj-actions/changed-files from 22.1 to 23.1 [@dependabot](https://github.com/dependabot) ([#1179](https://github.com/paperless-ngx/paperless-ngx/pull/1179))
+- Bump @<!---->angular/cli from 13.3.7 to 14.0.4 in /src-ui [@dependabot](https://github.com/dependabot) ([#1177](https://github.com/paperless-ngx/paperless-ngx/pull/1177))
+- Bump cypress from 10.0.1 to 10.3.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1187](https://github.com/paperless-ngx/paperless-ngx/pull/1187))
+- Bump zone.js from 0.11.5 to 0.11.6 in /src-ui [@dependabot](https://github.com/dependabot) ([#1185](https://github.com/paperless-ngx/paperless-ngx/pull/1185))
+- Bump ts-node from 10.8.0 to 10.8.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1184](https://github.com/paperless-ngx/paperless-ngx/pull/1184))
+- Bump jest-environment-jsdom from 28.1.0 to 28.1.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1175](https://github.com/paperless-ngx/paperless-ngx/pull/1175))
+- Bump @<!---->types/node from 17.0.38 to 18.0.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1183](https://github.com/paperless-ngx/paperless-ngx/pull/1183))
+- Bump concurrently from 7.2.1 to 7.2.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1181](https://github.com/paperless-ngx/paperless-ngx/pull/1181))
+- Bump jest-preset-angular from 12.0.1 to 12.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1182](https://github.com/paperless-ngx/paperless-ngx/pull/1182))
+- Bump jest and @<!---->types/jest in /src-ui [@dependabot](https://github.com/dependabot) ([#1180](https://github.com/paperless-ngx/paperless-ngx/pull/1180))
+- Bump whitenoise from 6.1.0 to 6.2.0 [@dependabot](https://github.com/dependabot) ([#1103](https://github.com/paperless-ngx/paperless-ngx/pull/1103))
+- Bump cypress from 9.6.1 to 10.0.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1083](https://github.com/paperless-ngx/paperless-ngx/pull/1083))
+- Bump docker/setup-qemu-action from 1 to 2 [@dependabot](https://github.com/dependabot) ([#1065](https://github.com/paperless-ngx/paperless-ngx/pull/1065))
+- Bump docker/setup-buildx-action from 1 to 2 [@dependabot](https://github.com/dependabot) ([#1064](https://github.com/paperless-ngx/paperless-ngx/pull/1064))
+- Bump docker/build-push-action from 2 to 3 [@dependabot](https://github.com/dependabot) ([#1063](https://github.com/paperless-ngx/paperless-ngx/pull/1063))
+- Bump @<!---->cypress/schematic from 1.7.0 to 2.0.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1075](https://github.com/paperless-ngx/paperless-ngx/pull/1075))
+- Bump tj-actions/changed-files from 19 to 22.1 [@dependabot](https://github.com/dependabot) ([#1062](https://github.com/paperless-ngx/paperless-ngx/pull/1062))
+- Bump concurrently from 7.1.0 to 7.2.1 in /src-ui [@dependabot](https://github.com/dependabot) ([#1073](https://github.com/paperless-ngx/paperless-ngx/pull/1073))
+- Bump @<!---->types/jest from 27.4.1 to 27.5.2 in /src-ui [@dependabot](https://github.com/dependabot) ([#1074](https://github.com/paperless-ngx/paperless-ngx/pull/1074))
+- Bump ts-node from 10.7.0 to 10.8.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1070](https://github.com/paperless-ngx/paperless-ngx/pull/1070))
+- Bump jest from 28.0.3 to 28.1.0 in /src-ui [@dependabot](https://github.com/dependabot) ([#1071](https://github.com/paperless-ngx/paperless-ngx/pull/1071))
+- Chore: npm package updates 22-06-01 [@shamoon](https://github.com/shamoon) ([#1069](https://github.com/paperless-ngx/paperless-ngx/pull/1069))
+- Bump docker/login-action from 1 to 2 [@dependabot](https://github.com/dependabot) ([#1061](https://github.com/paperless-ngx/paperless-ngx/pull/1061))
+- Chore: Manually update dependencies [@stumpylog](https://github.com/stumpylog) ([#1013](https://github.com/paperless-ngx/paperless-ngx/pull/1013))
+- Chore: Manually update all Python dependencies [@stumpylog](https://github.com/stumpylog) ([#973](https://github.com/paperless-ngx/paperless-ngx/pull/973))
+</details>
+
 ## paperless-ngx 1.7.1
 
 ### Features
@@ -17,7 +347,7 @@
 ### Bug Fixes
 
 - Feature / fix saved view \& sort field query params [\@shamoon](https://github.com/shamoon) ([\#881](https://github.com/paperless-ngx/paperless-ngx/pull/881))
-- mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873))
+- Mobile friendlier manage pages [\@shamoon](https://github.com/shamoon) ([\#873](https://github.com/paperless-ngx/paperless-ngx/pull/873))
 - Add timeout to healthcheck [\@shamoon](https://github.com/shamoon) ([\#880](https://github.com/paperless-ngx/paperless-ngx/pull/880))
 - Always accept yyyy-mm-dd date inputs [\@shamoon](https://github.com/shamoon) ([\#864](https://github.com/paperless-ngx/paperless-ngx/pull/864))
 - Fix local Docker image building [\@stumpylog](https://github.com/stumpylog) ([\#849](https://github.com/paperless-ngx/paperless-ngx/pull/849))
@@ -70,137 +400,137 @@
 
 ## paperless-ngx 1.7.0
 
-Breaking Changes
+### Breaking Changes
 
 - `PAPERLESS_URL` is now required when using a reverse proxy. See
   [\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674).
 
-Features
+### Features
 
 - Allow setting more than one tag in mail rules
-  [\@jonasc](https://github.com/jonasc) (\#270)
-- global drag\'n\'drop [\@shamoon](https://github.com/shamoon)
-  (\#283).
+  [\@jonasc](https://github.com/jonasc) ([\#270](https://github.com/paperless-ngx/paperless-ngx/pull/270))
+- Global drag\'n\'drop [\@shamoon](https://github.com/shamoon)
+  ([\#283](https://github.com/paperless-ngx/paperless-ngx/pull/283))
 - Fix: download buttons should disable while waiting
-  [\@shamoon](https://github.com/shamoon) (\#630).
-- Update checker [\@shamoon](https://github.com/shamoon) (\#591).
+  [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630))
+- Update checker [\@shamoon](https://github.com/shamoon) ([\#591](https://github.com/paperless-ngx/paperless-ngx/pull/591))
 - Show prompt on password-protected pdfs
-  [\@shamoon](https://github.com/shamoon) (\#564).
+  [\@shamoon](https://github.com/shamoon) ([\#564](https://github.com/paperless-ngx/paperless-ngx/pull/564))
 - Filtering query params aka browser navigation for filtering
-  [\@shamoon](https://github.com/shamoon) (\#540).
+  [\@shamoon](https://github.com/shamoon) ([\#540](https://github.com/paperless-ngx/paperless-ngx/pull/540))
 - Clickable tags in dashboard widgets
-  [\@shamoon](https://github.com/shamoon) (\#515).
+  [\@shamoon](https://github.com/shamoon) ([\#515](https://github.com/paperless-ngx/paperless-ngx/pull/515))
 - Add bottom pagination [\@shamoon](https://github.com/shamoon)
-  (\#372).
+  ([\#372](https://github.com/paperless-ngx/paperless-ngx/pull/372))
 - Feature barcode splitter [\@gador](https://github.com/gador)
-  (\#532).
-- App loading screen [\@shamoon](https://github.com/shamoon) (\#298).
+  ([\#532](https://github.com/paperless-ngx/paperless-ngx/pull/532))
+- App loading screen [\@shamoon](https://github.com/shamoon) ([\#298](https://github.com/paperless-ngx/paperless-ngx/pull/298))
 - Use progress bar for delayed buttons
-  [\@shamoon](https://github.com/shamoon) (\#415).
+  [\@shamoon](https://github.com/shamoon) ([\#415](https://github.com/paperless-ngx/paperless-ngx/pull/415))
 - Add minimum length for documents text filter
-  [\@shamoon](https://github.com/shamoon) (\#401).
+  [\@shamoon](https://github.com/shamoon) ([\#401](https://github.com/paperless-ngx/paperless-ngx/pull/401))
 - Added nav buttons in the document detail view
-  [\@GruberViktor](https://github.com/gruberviktor) (\#273).
+  [\@GruberViktor](https://github.com/gruberviktor) ([\#273](https://github.com/paperless-ngx/paperless-ngx/pull/273))
 - Improve date keyboard input [\@shamoon](https://github.com/shamoon)
-  (\#253).
-- Color theming [\@shamoon](https://github.com/shamoon) (\#243).
+  ([\#253](https://github.com/paperless-ngx/paperless-ngx/pull/253))
+- Color theming [\@shamoon](https://github.com/shamoon) ([\#243](https://github.com/paperless-ngx/paperless-ngx/pull/243))
 - Parse dates when entered without separators
-  [\@GruberViktor](https://github.com/gruberviktor) (\#250).
+  [\@GruberViktor](https://github.com/gruberviktor) ([\#250](https://github.com/paperless-ngx/paperless-ngx/pull/250))
 
-Bug Fixes
+### Bug Fixes
 
-- add \"localhost\" to ALLOWED_HOSTS
-  [\@gador](https://github.com/gador) (\#700).
-- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690).
+- Add \"localhost\" to ALLOWED_HOSTS
+  [\@gador](https://github.com/gador) ([\#700](https://github.com/paperless-ngx/paperless-ngx/pull/700))
+- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690))
 - Adds wait for file before consuming
-  [\@stumpylog](https://github.com/stumpylog) (\#483).
+  [\@stumpylog](https://github.com/stumpylog) ([\#483](https://github.com/paperless-ngx/paperless-ngx/pull/483))
 - Fix: frontend document editing erases time data
-  [\@shamoon](https://github.com/shamoon) (\#654).
+  [\@shamoon](https://github.com/shamoon) ([\#654](https://github.com/paperless-ngx/paperless-ngx/pull/654))
 - Increase length of SavedViewFilterRule
-  [\@stumpylog](https://github.com/stumpylog) (\#612).
+  [\@stumpylog](https://github.com/stumpylog) ([\#612](https://github.com/paperless-ngx/paperless-ngx/pull/612))
 - Fixes attachment filename matching during mail fetching
-  [\@stumpylog](https://github.com/stumpylog) (\#680).
+  [\@stumpylog](https://github.com/stumpylog) ([\#680](https://github.com/paperless-ngx/paperless-ngx/pull/680))
 - Add `PAPERLESS_URL` env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/discussions/674))
 - Fix: download buttons should disable while waiting
-  [\@shamoon](https://github.com/shamoon) (\#630).
+  [\@shamoon](https://github.com/shamoon) ([\#630](https://github.com/paperless-ngx/paperless-ngx/pull/630))
 - Fixes downloaded filename, add more consumer ignore settings
-  [\@stumpylog](https://github.com/stumpylog) (\#599).
+  [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599))
 - FIX BUG: case-sensitive matching was not possible
-  [\@danielBreitlauch](https://github.com/danielbreitlauch) (\#594).
-- uses shutil.move instead of rename
-  [\@gador](https://github.com/gador) (\#617).
+  [\@danielBreitlauch](https://github.com/danielbreitlauch) ([\#594](https://github.com/paperless-ngx/paperless-ngx/pull/594))
+- Uses shutil.move instead of rename
+  [\@gador](https://github.com/gador) ([\#617](https://github.com/paperless-ngx/paperless-ngx/pull/617))
 - Fix npm deps 01.02.22 2 [\@shamoon](https://github.com/shamoon)
-  (\#610).
+  ([\#610](https://github.com/paperless-ngx/paperless-ngx/discussions/610))
 - Fix npm dependencies 01.02.22
-  [\@shamoon](https://github.com/shamoon) (\#600).
-- fix issue 416: implement PAPERLESS_OCR_MAX_IMAGE_PIXELS
-  [\@hacker-h](https://github.com/hacker-h) (\#441).
-- fix: exclude cypress from build in Dockerfile
-  [\@FrankStrieter](https://github.com/FrankStrieter) (\#526).
+  [\@shamoon](https://github.com/shamoon) ([\#600](https://github.com/paperless-ngx/paperless-ngx/pull/600))
+- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
+  [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441))
+- Fix: exclude cypress from build in Dockerfile
+  [\@FrankStrieter](https://github.com/FrankStrieter) ([\#526](https://github.com/paperless-ngx/paperless-ngx/pull/526))
 - Corrections to pass pre-commit hooks
-  [\@schnuffle](https://github.com/schnuffle) (\#454).
+  [\@schnuffle](https://github.com/schnuffle) ([\#454](https://github.com/paperless-ngx/paperless-ngx/pull/454))
 - Fix 311 unable to click checkboxes in document list
-  [\@shamoon](https://github.com/shamoon) (\#313).
+  [\@shamoon](https://github.com/shamoon) ([\#313](https://github.com/paperless-ngx/paperless-ngx/pull/313))
 - Fix imap tools bug [\@stumpylog](https://github.com/stumpylog)
-  (\#393).
+  ([\#393](https://github.com/paperless-ngx/paperless-ngx/pull/393))
 - Fix filterable dropdown buttons arent translated
-  [\@shamoon](https://github.com/shamoon) (\#366).
+  [\@shamoon](https://github.com/shamoon) ([\#366](https://github.com/paperless-ngx/paperless-ngx/pull/366))
 - Fix 224: \"Auto-detected date is day before receipt date\"
-  [\@a17t](https://github.com/a17t) (\#246).
+  [\@a17t](https://github.com/a17t) ([\#246](https://github.com/paperless-ngx/paperless-ngx/pull/246))
 - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon)
-  (\#322).
+  ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322))
 - Fix page links hidden [\@shamoon](https://github.com/shamoon)
-  (\#314).
+  ([\#314](https://github.com/paperless-ngx/paperless-ngx/pull/314))
 - Fix: Include excluded items in dropdown count
-  [\@shamoon](https://github.com/shamoon) (\#263).
+  [\@shamoon](https://github.com/shamoon) ([\#263](https://github.com/paperless-ngx/paperless-ngx/pull/263))
 
-Translation
+### Translation
 
 - [\@miku323](https://github.com/miku323) contributed to Slovenian
-  translation.
+  translation
 - [\@FaintGhost](https://github.com/FaintGhost) contributed to Chinese
-  Simplified translation.
+  Simplified translation
 - [\@DarkoBG79](https://github.com/DarkoBG79) contributed to Serbian
-  translation.
+  translation
 - [Kemal Secer](https://crowdin.com/profile/kemal.secer) contributed
-  to Turkish translation.
+  to Turkish translation
 - [\@Prominence](https://github.com/Prominence) contributed to
-  Belarusian translation.
+  Belarusian translation
 
-Documentation
+### Documentation
 
-- Fix: scanners table [\@qcasey](https://github.com/qcasey) (\#690).
-- Add [PAPERLESS\_URL]{.title-ref} env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+- Fix: scanners table [\@qcasey](https://github.com/qcasey) ([\#690](https://github.com/paperless-ngx/paperless-ngx/pull/690))
+- Add `PAPERLESS_URL` env variable & CSRF var
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674))
 - Fixes downloaded filename, add more consumer ignore settings
-  [\@stumpylog](https://github.com/stumpylog) (\#599).
-- fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
-  [\@hacker-h](https://github.com/hacker-h) (\#441).
+  [\@stumpylog](https://github.com/stumpylog) ([\#599](https://github.com/paperless-ngx/paperless-ngx/pull/599))
+- Fix issue 416: implement `PAPERLESS_OCR_MAX_IMAGE_PIXELS`
+  [\@hacker-h](https://github.com/hacker-h) ([\#441](https://github.com/paperless-ngx/paperless-ngx/pull/441))
 - Fix minor sphinx errors [\@shamoon](https://github.com/shamoon)
-  (\#322).
+  ([\#322](https://github.com/paperless-ngx/paperless-ngx/pull/322))
 
-Maintenance
+### Maintenance
 
 - Add `PAPERLESS_URL` env variable & CSRF var
-  [\@shamoon](https://github.com/shamoon) (\#674).
+  [\@shamoon](https://github.com/shamoon) ([\#674](https://github.com/paperless-ngx/paperless-ngx/pull/674))
 - Chore: Implement release-drafter action for Changelogs
-  [\@qcasey](https://github.com/qcasey) (\#669).
-- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) (\#667).
+  [\@qcasey](https://github.com/qcasey) ([\#669](https://github.com/paperless-ngx/paperless-ngx/pull/669))
+- Chore: Add CODEOWNERS [\@qcasey](https://github.com/qcasey) ([\#667](https://github.com/paperless-ngx/paperless-ngx/pull/667))
 - Support docker-compose v2 in install
-  [\@stumpylog](https://github.com/stumpylog) (\#611).
+  [\@stumpylog](https://github.com/stumpylog) ([\#611](https://github.com/paperless-ngx/paperless-ngx/pull/611))
 - Add Belarusian localization [\@shamoon](https://github.com/shamoon)
-  (\#588).
+  ([\#588](https://github.com/paperless-ngx/paperless-ngx/pull/588))
 - Add Turkish localization [\@shamoon](https://github.com/shamoon)
-  (\#536).
+  ([\#536](https://github.com/paperless-ngx/paperless-ngx/pull/536))
 - Add Serbian localization [\@shamoon](https://github.com/shamoon)
-  (\#504).
+  ([\#504](https://github.com/paperless-ngx/paperless-ngx/pull/504))
 - Create PULL_REQUEST_TEMPLATE.md
-  [\@shamoon](https://github.com/shamoon) (\#304).
+  [\@shamoon](https://github.com/shamoon) ([\#304](https://github.com/paperless-ngx/paperless-ngx/pull/304))
 - Add Chinese localization [\@shamoon](https://github.com/shamoon)
-  (\#247).
+  ([\#247](https://github.com/paperless-ngx/paperless-ngx/pull/247))
 - Add Slovenian language for frontend
-  [\@shamoon](https://github.com/shamoon) (\#315).
+  [\@shamoon](https://github.com/shamoon) ([\#315](https://github.com/paperless-ngx/paperless-ngx/pull/315))
 
 ## paperless-ngx 1.6.0
 
@@ -216,46 +546,46 @@ include:
 - Updated Python and Angular dependencies.
 - Dropped support for Python 3.7.
 - Dropped support for Ansible playbooks (thanks
-  [\@slankes](https://github.com/slankes) \#109). If someone would
-  like to continue supporting them, please see the [ansible
+  [\@slankes](https://github.com/slankes) [\#109](https://github.com/paperless-ngx/paperless-ngx/pull/109)). If someone would
+  like to continue supporting them, please see our [ansible
   repo](https://github.com/paperless-ngx/paperless-ngx-ansible).
 - Python code is now required to use Black formatting (thanks
-  [\@kpj](https://github.com/kpj) \#168).
+  [\@kpj](https://github.com/kpj) [\#168](https://github.com/paperless-ngx/paperless-ngx/pull/168)).
 - [\@tribut](https://github.com/tribut) added support for a custom SSO
-  logout redirect (jonaswinkler\#1258). See
+  logout redirect ([jonaswinkler\#1258](https://github.com/jonaswinkler/paperless-ng/pull/1258)). See
   `PAPERLESS_LOGOUT_REDIRECT_URL`.
 - [\@shamoon](https://github.com/shamoon) added a loading indicator
-  when document list is reloading (jonaswinkler\#1297).
+  when document list is reloading ([jonaswinkler\#1297](https://github.com/jonaswinkler/paperless-ng/pull/1297)).
 - [\@shamoon](https://github.com/shamoon) improved the PDF viewer on
-  mobile (\#2).
+  mobile ([\#2](https://github.com/paperless-ngx/paperless-ngx/pull/2)).
 - [\@shamoon](https://github.com/shamoon) added \'any\' / \'all\' and
-  \'not\' filtering with tags (\#10).
+  \'not\' filtering with tags ([\#10](https://github.com/paperless-ngx/paperless-ngx/pull/10)).
 - [\@shamoon](https://github.com/shamoon) added warnings for unsaved
-  changes, with smart edit buttons (\#13).
+  changes, with smart edit buttons ([\#13](https://github.com/paperless-ngx/paperless-ngx/pull/13)).
 - [\@benjaminfrank](https://github.com/benjaminfrank) enabled a
-  non-root access to port 80 via systemd (\#18).
+  non-root access to port 80 via systemd ([\#18](https://github.com/paperless-ngx/paperless-ngx/pull/18)).
 - [\@tribut](https://github.com/tribut) added simple \"delete to
-  trash\" functionality (\#24). See `PAPERLESS_TRASH_DIR`.
+  trash\" functionality ([\#24](https://github.com/paperless-ngx/paperless-ngx/pull/24)). See `PAPERLESS_TRASH_DIR`.
 - [\@amenk](https://github.com/amenk) fixed the search box overlay
-  menu on mobile (\#32).
+  menu on mobile ([\#32](https://github.com/paperless-ngx/paperless-ngx/pull/32)).
 - [\@dblitt](https://github.com/dblitt) updated the login form to not
-  auto-capitalize usernames (\#36).
+  auto-capitalize usernames ([\#36](https://github.com/paperless-ngx/paperless-ngx/pull/36)).
 - [\@evilsidekick293](https://github.com/evilsidekick293) made the
-  worker timeout configurable (\#37). See `PAPERLESS_WORKER_TIMEOUT`.
+  worker timeout configurable ([\#37](https://github.com/paperless-ngx/paperless-ngx/pull/37)). See `PAPERLESS_WORKER_TIMEOUT`.
 - [\@Nicarim](https://github.com/Nicarim) fixed downloads of UTF-8
-  formatted documents in Firefox (\#56).
+  formatted documents in Firefox ([\#56](https://github.com/paperless-ngx/paperless-ngx/pull/56)).
 - [\@mweimerskirch](https://github.com/mweimerskirch) sorted the
-  language dropdown by locale (\#78).
+  language dropdown by locale ([\#78](https://github.com/paperless-ngx/paperless-ngx/issues/78)).
 - [\@mweimerskirch](https://github.com/mweimerskirch) enabled the
-  Czech (\#83) and Danish (\#84) translations.
+  Czech ([\#83](https://github.com/paperless-ngx/paperless-ngx/pull/83)) and Danish ([\#84](https://github.com/paperless-ngx/paperless-ngx/pull/84)) translations.
 - [\@cschmatzler](https://github.com/cschmatzler) enabled specifying
-  the webserver port (\#124). See `PAPERLESS_PORT`.
+  the webserver port ([\#124](https://github.com/paperless-ngx/paperless-ngx/pull/124)). See `PAPERLESS_PORT`.
 - [\@muellermartin](https://github.com/muellermartin) fixed an error
-  when uploading transparent PNGs (\#133).
+  when uploading transparent PNGs ([\#133](https://github.com/paperless-ngx/paperless-ngx/pull/133)).
 - [\@shamoon](https://github.com/shamoon) created a slick new logo
-  (\#165).
+  ([\#165](https://github.com/paperless-ngx/paperless-ngx/pull/165)).
 - [\@tim-vogel](https://github.com/tim-vogel) fixed exports missing
-  groups (\#193).
+  groups ([\#193](https://github.com/paperless-ngx/paperless-ngx/pull/193)).
 
 Known issues:
 

docs/configuration.rst

@@ -27,11 +27,29 @@ PAPERLESS_REDIS=<url>
     This is required for processing scheduled tasks such as email fetching, index
     optimization and for training the automatic document matcher.
 
+    * If your Redis server needs login credentials PAPERLESS_REDIS = ``redis://<username>:<password>@<host>:<port>``
+
+    * With the requirepass option PAPERLESS_REDIS = ``redis://:<password>@<host>:<port>``
+
+    `More information on securing your Redis Instance <https://redis.io/docs/getting-started/#securing-redis>`_.
+
     Defaults to redis://localhost:6379.
 
+PAPERLESS_DBENGINE=<engine_name>
+    Optional, gives the ability to choose Postgres or MariaDB for database engine.
+    Available options are `postgresql` and `mariadb`.
+
+    Default is `postgresql`.
+
+    .. warning::
+
+      Using MariaDB comes with some caveats.  See :ref:`advanced-mysql-caveats` for details.
+
+
 PAPERLESS_DBHOST=<hostname>
     By default, sqlite is used as the database backend. This can be changed here.
-    Set PAPERLESS_DBHOST and PostgreSQL will be used instead of mysql.
+
+    Set PAPERLESS_DBHOST and another database will be used instead of sqlite.
 
 PAPERLESS_DBPORT=<port>
     Adjust port if necessary.
@@ -39,17 +57,17 @@ PAPERLESS_DBPORT=<port>
     Default is 5432.
 
 PAPERLESS_DBNAME=<name>
-    Database name in PostgreSQL.
+    Database name in PostgreSQL or MariaDB.
 
     Defaults to "paperless".
 
 PAPERLESS_DBUSER=<name>
-    Database user in PostgreSQL.
+    Database user in PostgreSQL or MariaDB.
 
     Defaults to "paperless".
 
 PAPERLESS_DBPASS=<password>
-    Database password for PostgreSQL.
+    Database password for PostgreSQL or MariaDB.
 
     Defaults to "paperless".
 
@@ -60,6 +78,13 @@ PAPERLESS_DBSSLMODE=<mode>
 
     Default is ``prefer``.
 
+PAPERLESS_DB_TIMEOUT=<float>
+    Amount of time for a database connection to wait for the database to unlock.
+    Mostly applicable for an sqlite based installation, consider changing to postgresql
+    if you need to increase this.
+
+    Defaults to unset, keeping the Django defaults.
+
 Paths and folders
 #################
 
@@ -202,9 +227,16 @@ PAPERLESS_FORCE_SCRIPT_NAME=<path>
 PAPERLESS_STATIC_URL=<path>
     Override the STATIC_URL here.  Unless you're hosting Paperless off a
     subdomain like /paperless/, you probably don't need to change this.
+    If you do change it, be sure to include the trailing slash.
 
     Defaults to "/static/".
 
+    .. note::
+
+        When hosting paperless behind a reverse proxy like Traefik or Nginx at a subpath e.g.
+        example.com/paperlessngx you will also need to set ``PAPERLESS_FORCE_SCRIPT_NAME``
+        (see above).
+
 PAPERLESS_AUTO_LOGIN_USERNAME=<username>
     Specify a username here so that paperless will automatically perform login
     with the selected user.
@@ -512,7 +544,7 @@ requires are as follows:
         # ...
 
         gotenberg:
-            image: gotenberg/gotenberg:7.4
+            image: gotenberg/gotenberg:7.6
             restart: unless-stopped
             command:
                 - "gotenberg"
@@ -675,6 +707,7 @@ PAPERLESS_CONSUMER_ENABLE_BARCODES=<bool>
 
     Defaults to false.
 
+
 PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT=<bool>
     Whether TIFF image files should be scanned for barcodes.
     This will automatically convert any TIFF image(s) to pdfs for later
@@ -734,6 +767,19 @@ PAPERLESS_FILENAME_DATE_ORDER=<format>
 
     Defaults to none, which disables this feature.
 
+PAPERLESS_NUMBER_OF_SUGGESTED_DATES=<num>
+    Paperless searches an entire document for dates. The first date found will
+    be used as the initial value for the created date. When this variable is
+    greater than 0 (or left to it's default value), paperless will also suggest
+    other dates found in the document, up to a maximum of this setting. Note that
+    duplicates will be removed, which can result in fewer dates displayed in the
+    frontend than this setting value.
+
+    The task to find all dates can be time-consuming and increases with a higher
+    (maximum) number of suggested dates and slower hardware.
+
+    Defaults to 3. Set to 0 to disable this feature.
+
 PAPERLESS_THUMBNAIL_FONT_NAME=<filename>
     Paperless creates thumbnails for plain text files by rendering the content
     of the file on an image and uses a predefined font for that. This
@@ -779,10 +825,10 @@ the program doesn't automatically execute it (ie. the program isn't in your
 $PATH), then you'll need to specify the literal path for that program.
 
 PAPERLESS_CONVERT_BINARY=<path>
-    Defaults to "/usr/bin/convert".
+    Defaults to "convert".
 
 PAPERLESS_GS_BINARY=<path>
-    Defaults to "/usr/bin/gs".
+    Defaults to "gs".
 
 
 .. _configuration-docker:
@@ -801,6 +847,13 @@ PAPERLESS_WEBSERVER_WORKERS=<num>
 
     Defaults to 1.
 
+PAPERLESS_BIND_ADDR=<ip address>
+    The IP address the webserver will listen on inside the container. There are
+    special setups where you may need to configure this value to restrict the
+    Ip address or interface the webserver listens on.
+
+    Defaults to [::], meaning all interfaces, including IPv6.
+
 PAPERLESS_PORT=<port>
     The port number the webserver will listen on inside the container. There are
     special setups where you may need this to avoid collisions with other
@@ -855,6 +908,14 @@ PAPERLESS_OCR_LANGUAGES=<list>
 
     Defaults to none, which does not install any additional languages.
 
+PAPERLESS_ENABLE_FLOWER=<defined>
+    If this environment variable is defined, the Celery monitoring tool
+    `Flower <https://flower.readthedocs.io/en/latest/index.html>`_ will
+    be started by the container.
+
+    You can read more about this in the :ref:`advanced setup <advanced-celery-monitoring>`
+    documentation.
+
 
 .. _configuration-update-checking:
 
@@ -862,18 +923,9 @@ Update Checking
 ###############
 
 PAPERLESS_ENABLE_UPDATE_CHECK=<bool>
-    Enable (or disable) the automatic check for available updates. This feature is disabled
-    by default but if it is not explicitly set Paperless-ngx will show a message about this.
 
-    If enabled, the feature works by pinging the the Github API for the latest release e.g.
-    https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest
-    to determine whether a new version is available.
+    .. note::
 
-    Actual updating of the app must still be performed manually.
-
-    Note that for users of thirdy-party containers e.g. linuxserver.io this notification
-    may be 'ahead' of a new release from the third-party maintainers.
-
-    In either case, no tracking data is collected by the app in any way.
-
-    Defaults to none, which disables the feature.
+            This setting was deprecated in favor of a frontend setting after v1.9.2. A one-time
+            migration is performed for users who have this setting set. This setting is always
+            ignored if the corresponding frontend setting has been set.

docs/extending.rst

@@ -79,7 +79,7 @@ To do the setup you need to perform the steps from the following chapters in a c
 6.  You can now either ...
 
     *  install redis or
-    *  use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a postgresql server) or
+    *  use the included scripts/start-services.sh to use docker to fire up a redis instance (and some other services such as tika, gotenberg and a database server) or
     *  spin up a bare redis container
 
         .. code:: shell-session
@@ -112,7 +112,7 @@ To do the setup you need to perform the steps from the following chapters in a c
 
     .. code:: shell-session
 
-        python3 manage.py runserver & python3 manage.py document_consumer & python3 manage.py qcluster
+        python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
 
 11. Login with the superuser credentials provided in step 8 at ``http://localhost:8000`` to create a session that enables you to use the backend.
 
@@ -128,14 +128,14 @@ Configure the IDE to use the src/ folder as the base source folder. Configure th
 launch configurations in your IDE:
 
 *   python3 manage.py runserver
-*   python3 manage.py qcluster
+*   celery --app paperless worker
 *   python3 manage.py document_consumer
 
 To start them all:
 
 .. code:: shell-session
 
-    python3 manage.py runserver & python3 manage.py document_consumer & python3 manage.py qcluster
+    python3 manage.py runserver & python3 manage.py document_consumer & celery --app paperless worker
 
 Testing and code style:
 

docs/index.rst

@@ -44,7 +44,7 @@ resources in the documentation:
     learn about how paperless automates all tagging using machine learning.
 *   Paperless now comes with a :ref:`proper email consumer <usage-email>`
     that's fully tested and production ready.
-*   Paperless creates searchable PDF/A documents from whatever you you put into
+*   Paperless creates searchable PDF/A documents from whatever you put into
     the consumption directory. This means that you can select text in
     image-only documents coming from your scanner.
 *   See :ref:`this note <utilities-encyption>` about GnuPG encryption in

docs/requirements.txt

@@ -1 +1 @@
-myst-parser==0.17.2
+myst-parser==0.18.1

docs/scanners.rst

@@ -1,142 +1,8 @@
 
 .. _scanners:
 
-***********************
-Scanner recommendations
-***********************
+*******************
+Scanners & Software
+*******************
 
-As Paperless operates by watching a folder for new files, doesn't care what
-scanner you use, but sometimes finding a scanner that will write to an FTP,
-NFS, or SMB server can be difficult.  This page is here to help you find one
-that works right for you based on recommendations from other Paperless users.
-
-Physical scanners
-=================
-
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brand   | Model             | Supports                                      | Recommended By |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-|         |                   | FTP | SFTP | NFS | SMB      | SMTP | API [1]_ |                |
-+=========+===================+=====+======+=====+==========+======+==========+================+
-| Brother | `ADS-1700W`_      | yes |      |     | yes      | yes  |          |`holzhannes`_   |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `ADS-1600W`_      | yes |      |     | yes      | yes  |          |`holzhannes`_   |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `ADS-1500W`_      | yes |      |     | yes      | yes  |          |`danielquinn`_  |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `ADS-1100W`_      | yes |      |     |          |      |          |`ytzelf`_       |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `ADS-2800W`_      | yes | yes  |     | yes      | yes  |          |`philpagel`_    |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `MFC-J6930DW`_    | yes |      |     |          |      |          |`ayounggun`_    |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `MFC-L5850DW`_    | yes |      |     |          | yes  |          |`holzhannes`_   |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `MFC-L2750DW`_    | yes |      |     | yes      | yes  |          |`muued`_        |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `MFC-J5910DW`_    | yes |      |     |          |      |          |`bmsleight`_    |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `MFC-8950DW`_     | yes |      |     | yes      | yes  |          |`philpagel`_    |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Brother | `MFC-9142CDN`_    | yes |      |     | yes      |      |          |`REOLDEV`_      |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Canon   | `Maxify MB 5350`_ |     |      |     | yes [2]_ | yes  |          |`eingemaischt`_ |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Fujitsu | `ix500`_          | yes |      |     | yes      |      |          |`eonist`_       |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Epson   | `ES-580W`_        | yes |      |     | yes      | yes  |          |`fignew`_       |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Epson   | `WF-7710DWF`_     | yes |      |     | yes      |      |          |`Skylinar`_     |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Fujitsu | `S1300i`_         | yes |      |     | yes      |      |          |`jonaswinkler`_ |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-| Doxie   | `Q2`_             |     |      |     |          |      | yes      |`Unkn0wnCat`_   |
-+---------+-------------------+-----+------+-----+----------+------+----------+----------------+
-
-.. _MFC-L5850DW: https://www.brother-usa.com/products/mfcl5850dw
-.. _MFC-L2750DW: https://www.brother.de/drucker/laserdrucker/mfc-l2750dw
-.. _ADS-1700W: https://www.brother-usa.com/products/ads1700w
-.. _ADS-1600W: https://www.brother-usa.com/products/ads1600w
-.. _ADS-1500W: https://www.brother.ca/en/p/ads1500w
-.. _ADS-1100W: https://support.brother.com/g/b/downloadtop.aspx?c=fr&lang=fr&prod=ads1100w_eu_as_cn
-.. _ADS-2800W: https://www.brother-usa.com/products/ads2800w
-.. _Maxify MB 5350: https://www.canon.de/printers/inkjet/maxify/maxify_mb5350/specification.html
-.. _MFC-J6930DW: https://www.brother.ca/en/p/MFCJ6930DW
-.. _MFC-J5910DW: https://www.brother.co.uk/printers/inkjet-printers/mfcj5910dw
-.. _MFC-8950DW: https://www.brother-usa.com/products/mfc8950dw
-.. _MFC-9142CDN: https://www.brother.co.uk/printers/laser-printers/mfc9140cdn
-.. _ES-580W: https://epson.com/Support/Scanners/ES-Series/Epson-WorkForce-ES-580W/s/SPT_B11B258201
-.. _WF-7710DWF: https://www.epson.de/en/products/printers/inkjet-printers/for-home/workforce-wf-7710dwf
-.. _ix500: http://www.fujitsu.com/us/products/computing/peripheral/scanners/scansnap/ix500/
-.. _S1300i: https://www.fujitsu.com/global/products/computing/peripheral/scanners/soho/s1300i/
-.. _Q2: https://www.getdoxie.com/product/doxie-q/
-
-.. _ayounggun: https://github.com/ayounggun
-.. _bmsleight: https://github.com/bmsleight
-.. _danielquinn: https://github.com/danielquinn
-.. _eonist: https://github.com/eonist
-.. _fignew: https://github.com/fignew
-.. _holzhannes: https://github.com/holzhannes
-.. _jonaswinkler: https://github.com/jonaswinkler
-.. _REOLDEV: https://github.com/REOLDEV
-.. _Skylinar: https://github.com/Skylinar
-.. _ytzelf: https://github.com/ytzelf
-.. _Unkn0wnCat: https://github.com/Unkn0wnCat
-.. _muued: https://github.com/muued
-.. _philpagel: https://github.com/philpagel
-.. _eingemaischt: https://github.com/eingemaischt
-
-.. [1] Scanners with API Integration allow to push scanned documents directly to :ref:`Paperless API <api-file_uploads>`, sometimes referred to as Webhook or Document POST.
-.. [2] Canon Multi Function Printers show strange behavior over SMB. They close and reopen the file after every page. It's recommended to tune the
-       :ref:`polling <configuration-polling>` and :ref:`inotify <configuration-inotify>` configuration values for your scanner. The scanner timeout is 3 minutes, so ``180`` is a good starting point.
-
-Mobile phone software
-=====================
-
-You can use your phone to "scan" documents. The regular camera app will work, but may have too low contrast for OCR to work well. Apps specifically for scanning are recommended.
-
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| Name                        | OS             | Supports                         | Recommended By   |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-|                             |                | FTP | NFS | SMB | Email | WebDAV |                  |
-+=============================+================+=====+=====+=====+=======+========+==================+
-| `Office Lens`_              | Android        | ?   | ?   | ?   | ?     | ?      | `jonaswinkler`_  |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| `Genius Scan`_              | Android        | yes | no  | yes | yes   | yes    | `hannahswain`_   |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| `OpenScan`_                 | Android        | no  | no  | no  | no    | no     | `benjaminfrank`_ |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-| `OCR Scanner - QuickScan`_  | iOS            | no  | no  | no  | no    | yes    | `holzhannes`_    |
-+-----------------------------+----------------+-----+-----+-----+-------+--------+------------------+
-
-On Android, you can use these applications in combination with one of the :ref:`Paperless-ngx compatible apps <usage-mobile_upload>` to "Share" the documents produced by these scanner apps with paperless. On iOS, you can share the scanned documents via iOS-Sharing to other mail, WebDav or FTP apps.
-
-There is also an iOS Shortcut that allows you to directly upload text, PDF and image documents available here: https://www.icloud.com/shortcuts/d234abc0885040129d9d75fa45fe1154
-Please note this only works for documents downloaded to iCloud / the device, in other words not directly from a URL.
-
-.. _Office Lens: https://play.google.com/store/apps/details?id=com.microsoft.office.officelens
-.. _Genius Scan: https://play.google.com/store/apps/details?id=com.thegrizzlylabs.geniusscan.free
-.. _OCR Scanner - QuickScan: https://apps.apple.com/us/app/quickscan-scanner-text-ocr/id1513790291
-.. _OpenScan: https://github.com/Ethereal-Developers-Inc/OpenScan
-
-.. _hannahswain: https://github.com/hannahswain
-.. _benjaminfrank: https://github.com/benjaminfrank
-
-API Scanning Setup
-==================
-
-This sections contains information on how to set up scanners to post directly to :ref:`Paperless API <api-file_uploads>`.
-
-Doxie Q2
---------
-
-This part assumes your Doxie is connected to WiFi and you know its IP.
-
-1. Open your Doxie web UI by navigating to its IP address
-2. Navigate to Options -> Webhook
-3. Set the *URL* to ``https://[your-paperless-ngx-instance]/api/documents/post_document/``
-4. Set the *File Parameter Name* to ``document``
-5. Add the username and password to the respective fields (Consider creating a user just for your Doxie)
-6. Click *Submit* at the bottom of the page
-
-Congrats, you can now scan directly from your Doxie to your Paperless-ngx instance!
+Paperless-ngx is compatible with many different scanners and scanning tools. A user-maintained list of scanners and other software is available on `the wiki <https://github.com/paperless-ngx/paperless-ngx/wiki/Scanner-&-Software-Recommendations>`_.

docs/setup.rst

@@ -39,7 +39,7 @@ Paperless consists of the following components:
 
     .. _setup-task_processor:
 
-*   **The task processor:** Paperless relies on `Django Q <https://django-q.readthedocs.io/en/latest/>`_
+*   **The task processor:** Paperless relies on `Celery - Distributed Task Queue <https://docs.celeryq.dev/en/stable/index.html>`_
     for doing most of the heavy lifting. This is a task queue that accepts tasks from
     multiple sources and processes these in parallel. It also comes with a scheduler that executes
     certain commands periodically.
@@ -62,18 +62,11 @@ Paperless consists of the following components:
     tasks fail and inspect the errors (i.e., wrong email credentials, errors during consuming a specific
     file, etc).
 
-    You may start the task processor by executing:
-
-    .. code:: shell-session
-
-        $ cd /path/to/paperless/src/
-        $ python3 manage.py qcluster
-
 *   A `redis <https://redis.io/>`_ message broker: This is a really lightweight service that is responsible
     for getting the tasks from the webserver and the consumer to the task scheduler. These run in a different
     process (maybe even on different machines!), and therefore, this is necessary.
 
-*   Optional: A database server. Paperless supports both PostgreSQL and SQLite for storing its data.
+*   Optional: A database server. Paperless supports PostgreSQL, MariaDB and SQLite for storing its data.
 
 
 Installation
@@ -224,6 +217,7 @@ Install Paperless from Docker Hub
         You can utilize Docker secrets for some configuration settings by
         appending `_FILE` to some configuration values.  This is supported currently
         only by:
+
           * PAPERLESS_DBUSER
           * PAPERLESS_DBPASS
           * PAPERLESS_SECRET_KEY
@@ -290,7 +284,20 @@ Build the Docker image yourself
     .. code:: yaml
 
         webserver:
-            build: .
+            build:
+              context: .
+              args:
+                QPDF_VERSION: x.y.x
+                PIKEPDF_VERSION: x.y.z
+                PSYCOPG2_VERSION: x.y.z
+                JBIG2ENC_VERSION: 0.29
+
+    .. note::
+
+        You should match the build argument versions to the version for the release you have
+        checked out.  These are pre-built images with certain, more updated software.
+        If you want to build these images your self, that is possible, but beyond
+        the scope of these steps.
 
 4.  Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run
     ``docker-compose pull`` to pull the image, do
@@ -316,11 +323,13 @@ writing. Windows is not and will never be supported.
     *   ``python3-pip``
     *   ``python3-dev``
 
+    *   ``default-libmysqlclient-dev`` for MariaDB
     *   ``fonts-liberation`` for generating thumbnails for plain text files
     *   ``imagemagick`` >= 6 for PDF conversion
     *   ``gnupg`` for handling encrypted documents
     *   ``libpq-dev`` for PostgreSQL
     *   ``libmagic-dev`` for mime type detection
+    *   ``mariadb-client`` for MariaDB compile time
     *   ``mime-support`` for mime type detection
     *   ``libzbar0`` for barcode detection
     *   ``poppler-utils`` for barcode detection
@@ -329,7 +338,7 @@ writing. Windows is not and will never be supported.
 
     .. code::
 
-        python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils
+        python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev libmagic-dev mime-support libzbar0 poppler-utils
 
     These dependencies are required for OCRmyPDF, which is used for text recognition.
 
@@ -339,7 +348,7 @@ writing. Windows is not and will never be supported.
     *   ``qpdf``
     *   ``liblept5``
     *   ``libxml2``
-    *   ``pngquant``
+    *   ``pngquant`` (suggested for certain PDF image optimizations)
     *   ``zlib1g``
     *   ``tesseract-ocr`` >= 4.0.0 for OCR
     *   ``tesseract-ocr`` language packs (``tesseract-ocr-eng``, ``tesseract-ocr-deu``, etc)
@@ -358,10 +367,10 @@ writing. Windows is not and will never be supported.
     You will also need ``build-essential``, ``python3-setuptools`` and ``python3-wheel``
     for installing some of the python dependencies.
 
-2.  Install ``redis`` >= 5.0 and configure it to start automatically.
+2.  Install ``redis`` >= 6.0 and configure it to start automatically.
 
 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
-    to use PostgreSQL, SQLite is available as well.
+    to use PostgreSQL, MariaDB and SQLite are available as well.
 
     .. note::
 
@@ -377,6 +386,7 @@ writing. Windows is not and will never be supported.
     settings to your needs. Required settings for getting paperless running are:
 
     *   ``PAPERLESS_REDIS`` should point to your redis server, such as redis://localhost:6379.
+    *   ``PAPERLESS_DBENGINE`` optional, and should be one of `postgres, mariadb, or sqlite`
     *   ``PAPERLESS_DBHOST`` should be the hostname on which your PostgreSQL server is running. Do not configure this
         to use SQLite instead. Also configure port, database name, user and password as necessary.
     *   ``PAPERLESS_CONSUMPTION_DIR`` should point to a folder which paperless should watch for documents. You might
@@ -457,8 +467,9 @@ writing. Windows is not and will never be supported.
     as a starting point.
 
     Paperless needs the ``webserver`` script to run the webserver, the
-    ``consumer`` script to watch the input folder, and the ``scheduler``
-    script to run tasks such as email checking and document consumption.
+    ``consumer`` script to watch the input folder, ``taskqueue`` for the background workers
+    used to handle things like document consumption and the ``scheduler`` script to run tasks such as
+    email checking at certain times .
 
 		The ``socket`` script enables ``gunicorn`` to run on port 80 without
 		root privileges. For this you need to uncomment the ``Require=paperless-webserver.socket``
@@ -509,6 +520,13 @@ writing. Windows is not and will never be supported.
     to compile this by yourself, because this software has been patented until around 2017 and
     binary packages are not available for most distributions.
 
+15. Optional: If using the NLTK machine learning processing (see ``PAPERLESS_ENABLE_NLTK`` in
+    :ref:`configuration` for details), download the NLTK data for the Snowball Stemmer, Stopwords
+    and Punkt tokenizer to your ``PAPERLESS_DATA_DIR/nltk``.  Refer to
+    the `NLTK instructions <https://www.nltk.org/data.html>`_ for details on how to
+    download the data.
+
+
 Migrating to Paperless-ngx
 ##########################
 
@@ -630,13 +648,56 @@ Migration to paperless-ngx is then performed in a few simple steps:
 10.  Optionally, follow the instructions below to migrate your existing data to PostgreSQL.
 
 
+Migrating from LinuxServer.io Docker Image
+==========================================
+
+As with any upgrades and large changes, it is highly recommended to create a backup before
+starting.  This assumes the image was running using Docker Compose, but the instructions
+are translatable to Docker commands as well.
+
+1.  Stop and remove the paperless container
+2.  If using an external database, stop the container
+3.  Update Redis configuration
+
+    a)  If ``REDIS_URL`` is already set, change it to ``PAPERLESS_REDIS`` and continue
+        to step 4.
+    b)  Otherwise, in the ``docker-compose.yml`` add a new service for Redis,
+        following `the example compose files <https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose>`_
+    c)  Set the environment variable ``PAPERLESS_REDIS`` so it points to the new Redis container
+
+4.  Update user mapping
+
+    a)  If set, change the environment variable ``PUID`` to ``USERMAP_UID``
+    b)  If set, change the environment variable ``PGID`` to ``USERMAP_GID``
+
+5.  Update configuration paths
+
+    a) Set the environment variable ``PAPERLESS_DATA_DIR``
+       to ``/config``
+
+6.  Update media paths
+
+    a) Set the environment variable ``PAPERLESS_MEDIA_ROOT``
+       to ``/data/media``
+
+7.  Update timezone
+
+    a) Set the environment variable ``PAPERLESS_TIME_ZONE``
+       to the same value as ``TZ``
+
+8.  Modify the ``image:`` to point to ``ghcr.io/paperless-ngx/paperless-ngx:latest`` or
+    a specific version if preferred.
+
+9.  Start the containers as before, using ``docker-compose``.
+
 .. _setup-sqlite_to_psql:
 
-Moving data from SQLite to PostgreSQL
-=====================================
+Moving data from SQLite to PostgreSQL or MySQL/MariaDB
+======================================================
 
-Moving your data from SQLite to PostgreSQL is done via executing a series of django
-management commands as below.
+Moving your data from SQLite to PostgreSQL or MySQL/MariaDB is done via executing a series of django
+management commands as below.  The commands below use PostgreSQL, but are applicable to MySQL/MariaDB
+with the
 
 .. caution::
 
@@ -653,6 +714,11 @@ management commands as below.
     and filenames (1024 characters). If you have data in these fields that surpasses these
     limits, migration to PostgreSQL is not possible and will fail with an error.
 
+.. warning::
+
+    MySQL is case insensitive by default, treating values like "Name" and "NAME" as identical.
+    See :ref:`advanced-mysql-caveats` for details.
+
 
 1.  Stop paperless, if it is running.
 2.  Tell paperless to use PostgreSQL:
@@ -763,6 +829,8 @@ configuring some options in paperless can help improve performance immensely:
     OCR results.
 *   If using docker, consider setting ``PAPERLESS_WEBSERVER_WORKERS`` to
     1. This will save some memory.
+*   Consider setting ``PAPERLESS_ENABLE_NLTK`` to false, to disable the more
+    advanced language processing, which can take more memory and processing time.
 
 For details, refer to :ref:`configuration`.
 

docs/troubleshooting.rst

@@ -19,7 +19,7 @@ Check for the following issues:
 
     .. code:: shell-session
 
-        $ python3 manage.py qcluster
+        $ celery --app paperless worker
 
 *   Look at the output of paperless and inspect it for any errors.
 *   Go to the admin interface, and check if there are failed tasks. If so, the
@@ -125,7 +125,7 @@ If using docker-compose, this is achieved by the following configuration change
 .. code:: yaml
 
     gotenberg:
-        image: gotenberg/gotenberg:7.4
+        image: gotenberg/gotenberg:7.6
         restart: unless-stopped
         command:
             - "gotenberg"
@@ -301,3 +301,28 @@ try adjusting the :ref:`polling configuration <configuration-polling>`.
 
     The user will need to manually move the file out of the consume folder and
     back in, for the initial failing file to be consumed.
+
+Log reports "Creating PaperlessTask failed".
+#########################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.management.consumer] Creating PaperlessTask failed: db locked
+
+You are likely using an sqlite based installation, with an increased number of workers and are running into sqlite's concurrency limitations.
+Uploading or consuming multiple files at once results in many workers attempting to access the database simultaneously.
+
+Consider changing to the PostgreSQL database if you will be processing many documents at once often.  Otherwise,
+try tweaking the ``PAPERLESS_DB_TIMEOUT`` setting to allow more time for the database to unlock.  This may have
+minor performance implications.
+
+
+gunicorn fails to start with "is not a valid port number"
+#########################################################
+
+You are likely running using Kubernetes, which automatically creates an environment variable named `${serviceName}_PORT`.
+This is the same environment variable which is used by Paperless to optionally change the port gunicorn listens on.
+
+To fix this, set `PAPERLESS_PORT` again to your desired port, or the default of 8000.

gunicorn.conf.py

@@ -1,9 +1,17 @@
 import os
 
-bind = f'[::]:{os.getenv("PAPERLESS_PORT", 8000)}'
+# See https://docs.gunicorn.org/en/stable/settings.html for
+# explanations of settings
+
+bind = f'{os.getenv("PAPERLESS_BIND_ADDR", "[::]")}:{os.getenv("PAPERLESS_PORT", 8000)}'
+
 workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 1))
 worker_class = "paperless.workers.ConfigurableWorker"
 timeout = 120
+preload_app = True
+
+# https://docs.gunicorn.org/en/stable/faq.html#blocking-os-fchmod
+worker_tmp_dir = "/dev/shm"
 
 
 def pre_fork(server, worker):

install-paperless-ngx.sh

@@ -118,12 +118,12 @@ ask "Current time zone" "$default_time_zone"
 TIME_ZONE=$ask_result
 
 echo ""
-echo "Database backend: PostgreSQL and SQLite are available. Use PostgreSQL"
+echo "Database backend: PostgreSQL, MariaDB, and SQLite are available. Use PostgreSQL"
 echo "if unsure. If you're running on a low-power device such as Raspberry"
 echo "Pi, use SQLite to save resources."
 echo ""
 
-ask "Database backend" "postgres" "postgres sqlite"
+ask "Database backend" "postgres" "postgres sqlite mariadb"
 DATABASE_BACKEND=$ask_result
 
 echo ""
@@ -214,9 +214,9 @@ echo ""
 ask_docker_folder "Data folder" ""
 DATA_FOLDER=$ask_result
 
-if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
+if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then
 	echo ""
-	echo "The database folder, where postgres stores its data."
+	echo "The database folder, where your database stores its data."
 	echo "Leave empty to have this managed by docker."
 	echo ""
 	echo "CAUTION: If specified, you must specify an absolute path starting with /"
@@ -224,7 +224,7 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
 	echo ""
 
 	ask_docker_folder "Database folder" ""
-	POSTGRES_FOLDER=$ask_result
+	DATABASE_FOLDER=$ask_result
 fi
 
 echo ""
@@ -278,13 +278,14 @@ if [[ -z $DATA_FOLDER ]] ; then
 else
 	echo "Data folder: $DATA_FOLDER"
 fi
-if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
-	if [[ -z $POSTGRES_FOLDER ]] ; then
-		echo "Database (postgres) folder: Managed by docker"
+if [[ "$DATABASE_BACKEND" == "postgres" || "$DATABASE_BACKEND" == "mariadb" ]] ; then
+	if [[ -z $DATABASE_FOLDER ]] ; then
+		echo "Database folder: Managed by docker"
 	else
-		echo "Database (postgres) folder: $POSTGRES_FOLDER"
+		echo "Database folder: $DATABASE_FOLDER"
 	fi
 fi
+
 echo ""
 echo "URL: $URL"
 echo "Port: $PORT"
@@ -356,9 +357,16 @@ if [[ -n $DATA_FOLDER ]] ; then
 	sed -i "/^\s*data:/d" docker-compose.yml
 fi
 
-if [[ -n $POSTGRES_FOLDER ]] ; then
-	sed -i "s#- pgdata:/var/lib/postgresql/data#- $POSTGRES_FOLDER:/var/lib/postgresql/data#g" docker-compose.yml
-	sed -i "/^\s*pgdata:/d" docker-compose.yml
+# If the database folder was provided (not blank), replace the pgdata/dbdata volume with a bind mount
+# of the provided folder
+if [[ -n $DATABASE_FOLDER ]] ; then
+	if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
+		sed -i "s#- pgdata:/var/lib/postgresql/data#- $DATABASE_FOLDER:/var/lib/postgresql/data#g" docker-compose.yml
+		sed -i "/^\s*pgdata:/d" docker-compose.yml
+	elif [[ "$DATABASE_BACKEND" == "mariadb" ]]; then
+		sed -i "s#- dbdata:/var/lib/mysql#- $DATABASE_FOLDER:/var/lib/mysql#g" docker-compose.yml
+		sed -i "/^\s*dbdata:/d" docker-compose.yml
+	fi
 fi
 
 # remove trailing blank lines from end of file
@@ -375,4 +383,4 @@ ${DOCKER_COMPOSE_CMD} pull
 
 ${DOCKER_COMPOSE_CMD} run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
 
-${DOCKER_COMPOSE_CMD} up -d
+${DOCKER_COMPOSE_CMD} up --detach

paperless.conf.example

@@ -64,11 +64,12 @@
 #PAPERLESS_CONSUMER_IGNORE_PATTERNS=[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]
 #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
 #PAPERLESS_CONSUMER_ENABLE_BARCODES=false
-#PAPERLESS_CONSUMER_ENABLE_BARCODES=PATCHT
+#PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_FILENAME_DATE_ORDER=YMD
 #PAPERLESS_FILENAME_PARSE_TRANSFORMS=[]
+#PAPERLESS_NUMBER_OF_SUGGESTED_DATES=5
 #PAPERLESS_THUMBNAIL_FONT_NAME=
 #PAPERLESS_IGNORE_DATES=
 #PAPERLESS_ENABLE_UPDATE_CHECK=

requirements.txt

@@ -1,108 +0,0 @@
--i https://pypi.python.org/simple
---extra-index-url https://www.piwheels.org/simple
-aioredis==1.3.1
-anyio==3.6.1; python_full_version >= '3.6.2'
-arrow==1.2.2; python_version >= '3.6'
-asgiref==3.5.2; python_version >= '3.7'
-async-timeout==4.0.2; python_version >= '3.6'
-attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-autobahn==22.6.1; python_version >= '3.7'
-automat==20.2.0
-backports.zoneinfo==0.2.1; python_version < '3.9'
-blessed==1.19.1; python_version >= '2.7'
-certifi==2022.6.15; python_version >= '3.6'
-cffi==1.15.1
-channels==3.0.5
-channels-redis==3.4.0
-charset-normalizer==2.1.0; python_version >= '3.6'
-click==8.1.3; python_version >= '3.7'
-coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-concurrent-log-handler==0.9.20
-constantly==15.1.0
-cryptography==37.0.4; python_version >= '3.6'
-daphne==3.0.2; python_version >= '3.6'
-dateparser==1.1.1
-deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-deprecation==2.1.0
-django==4.0.6
-django-cors-headers==3.13.0
-django-extensions==3.2.0
-django-filter==22.1
-django-picklefield==3.1; python_version >= '3'
--e git+https://github.com/paperless-ngx/django-q.git@bf20d57f859a7d872d5979cd8879fac9c9df981c#egg=django-q
-djangorestframework==3.13.1
-filelock==3.7.1
-fuzzywuzzy[speedup]==0.18.0
-gunicorn==20.1.0
-h11==0.13.0; python_version >= '3.6'
-hiredis==2.0.0; python_version >= '3.6'
-httptools==0.4.0
-humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-hyperlink==21.0.0
-idna==3.3; python_version >= '3.5'
-imap-tools==0.56.0
-img2pdf==0.4.4
-importlib-resources==5.8.0; python_version < '3.9'
-incremental==21.3.0
-inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-inotifyrecursive==0.3.5
-joblib==1.1.0; python_version >= '3.6'
-langdetect==1.0.9
-lxml==4.9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-msgpack==1.0.4
-numpy==1.23.1; python_version >= '3.8'
-ocrmypdf==13.6.0
-packaging==21.3; python_version >= '3.6'
-pathvalidate==2.5.0
-pdf2image==1.16.0
-pdfminer.six==20220524
-pikepdf==5.3.1
-pillow==9.2.0
-pluggy==1.0.0; python_version >= '3.6'
-portalocker==2.5.1; python_version >= '3'
-psycopg2==2.9.3
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycparser==2.21
-pyopenssl==22.0.0
-pyparsing==3.0.9; python_full_version >= '3.6.8'
-python-dateutil==2.8.2
-python-dotenv==0.20.0
-python-gnupg==0.4.9
-python-levenshtein==0.12.2
-python-magic==0.4.27
-pytz==2022.1
-pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-pyyaml==6.0
-pyzbar==0.1.9
-redis==4.3.4
-regex==2022.3.2; python_version >= '3.6'
-reportlab==3.6.11; python_version >= '3.7' and python_version < '4'
-requests==2.28.1; python_version >= '3.7' and python_version < '4'
-scikit-learn==1.1.1
-scipy==1.8.1; python_version < '3.11' and python_version >= '3.8'
-service-identity==21.1.0
-setuptools==63.1.0; python_version >= '3.7'
-six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-sniffio==1.2.0; python_version >= '3.5'
-sqlparse==0.4.2; python_version >= '3.5'
-threadpoolctl==3.1.0; python_version >= '3.6'
-tika==1.24
-tqdm==4.64.0
-twisted[tls]==22.4.0; python_full_version >= '3.6.7'
-txaio==22.2.1; python_version >= '3.6'
-typing-extensions==4.3.0; python_version >= '3.7'
-tzdata==2022.1; python_version >= '3.6'
-tzlocal==4.2; python_version >= '3.6'
-urllib3==1.26.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
-uvicorn[standard]==0.18.2
-uvloop==0.16.0
-watchdog==2.1.9
-watchfiles==0.15.0
-wcwidth==0.2.5
-websockets==10.3
-whitenoise==6.2.0
-whoosh==2.7.4
-wrapt==1.14.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-zipp==3.8.0; python_version < '3.9'
-zope.interface==5.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

scripts/paperless-scheduler.service

@@ -1,12 +1,12 @@
 [Unit]
-Description=Paperless scheduler
+Description=Paperless Celery Beat
 Requires=redis.service
 
 [Service]
 User=paperless
 Group=paperless
 WorkingDirectory=/opt/paperless/src
-ExecStart=python3 manage.py qcluster
+ExecStart=celery --app paperless beat --loglevel INFO
 
 [Install]
 WantedBy=multi-user.target

scripts/paperless-task-queue.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Paperless Celery Workers
+Requires=redis.service
+
+[Service]
+User=paperless
+Group=paperless
+WorkingDirectory=/opt/paperless/src
+ExecStart=celery --app paperless worker --loglevel INFO
+
+[Install]
+WantedBy=multi-user.target

scripts/start_services.sh

@@ -2,5 +2,5 @@
 
 docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
-docker run -p 3000:3000 -d gotenberg/gotenberg:7.4
+docker run -p 3000:3000 -d gotenberg/gotenberg:7.6
 docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/cypress/e2e/documents/document-detail.cy.ts

@@ -17,6 +17,32 @@ describe('document-detail', () => {
       req.reply({ result: 'OK' })
     }).as('saveDoc')
 
+    cy.fixture('documents/1/comments.json').then((commentsJson) => {
+      cy.intercept(
+        'GET',
+        'http://localhost:8000/api/documents/1/comments/',
+        (req) => {
+          req.reply(commentsJson.filter((c) => c.id != 10)) // 3
+        }
+      )
+
+      cy.intercept(
+        'DELETE',
+        'http://localhost:8000/api/documents/1/comments/?id=9',
+        (req) => {
+          req.reply(commentsJson.filter((c) => c.id != 9 && c.id != 10)) // 2
+        }
+      )
+
+      cy.intercept(
+        'POST',
+        'http://localhost:8000/api/documents/1/comments/',
+        (req) => {
+          req.reply(commentsJson) // 4
+        }
+      )
+    })
+
     cy.viewport(1024, 1024)
     cy.visit('/documents/1/')
   })
@@ -39,4 +65,30 @@ describe('document-detail', () => {
     cy.contains('button', 'Save').click().wait('@saveDoc').wait(2000) // navigates away after saving
     cy.contains('You have unsaved changes').should('not.exist')
   })
+
+  it('should show a list of comments', () => {
+    cy.wait(1000).get('a').contains('Comments').click().wait(1000)
+    cy.get('app-document-comments').find('.card').its('length').should('eq', 3)
+  })
+
+  it('should support comment deletion', () => {
+    cy.wait(1000).get('a').contains('Comments').click().wait(1000)
+    cy.get('app-document-comments')
+      .find('.card')
+      .first()
+      .find('button')
+      .click({ force: true })
+      .wait(500)
+    cy.get('app-document-comments').find('.card').its('length').should('eq', 2)
+  })
+
+  it('should support comment insertion', () => {
+    cy.wait(1000).get('a').contains('Comments').click().wait(1000)
+    cy.get('app-document-comments')
+      .find('form textarea')
+      .type('Testing new comment')
+      .wait(500)
+    cy.get('app-document-comments').find('form button').click().wait(1500)
+    cy.get('app-document-comments').find('.card').its('length').should('eq', 4)
+  })
 })

src-ui/cypress/e2e/settings/settings.cy.ts

@@ -46,7 +46,7 @@ describe('settings', () => {
       })
     })
 
-    cy.viewport(1024, 1024)
+    cy.viewport(1024, 1600)
     cy.visit('/settings')
     cy.wait('@savedViews')
   })

src-ui/cypress/fixtures/documents/1/comments.json

@@ -0,0 +1,46 @@
+[
+    {
+        "id": 10,
+        "comment": "Testing new comment",
+        "created": "2022-08-08T04:24:55.176008Z",
+        "user": {
+            "id": 1,
+            "username": "user2",
+            "firstname": "",
+            "lastname": ""
+        }
+    },
+    {
+        "id": 9,
+        "comment": "Testing one more time",
+        "created": "2022-02-18T04:24:55.176008Z",
+        "user": {
+            "id": 2,
+            "username": "user1",
+            "firstname": "",
+            "lastname": ""
+        }
+    },
+    {
+        "id": 8,
+        "comment": "Another comment",
+        "created": "2021-11-08T04:24:47.925042Z",
+        "user": {
+            "id": 2,
+            "username": "user33",
+            "firstname": "",
+            "lastname": ""
+        }
+    },
+    {
+        "id": 7,
+        "comment": "Cupcake ipsum dolor sit amet cheesecake candy cookie tiramisu. Donut chocolate chupa chups macaroon brownie halvah pie cheesecake gummies. Sweet chocolate bar candy donut gummi bears bear claw liquorice bonbon shortbread.\n\nDonut chocolate bar candy wafer wafer tiramisu. Gummies chocolate cake muffin toffee carrot cake macaroon. Toffee toffee jelly beans danish lollipop cake.",
+        "created": "2021-02-08T02:37:49.724132Z",
+        "user": {
+            "id": 3,
+            "username": "admin",
+            "firstname": "",
+            "lastname": ""
+        }
+    }
+]

src-ui/package.json

@@ -13,48 +13,49 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/common": "~14.0.4",
-    "@angular/compiler": "~14.0.4",
-    "@angular/core": "~14.0.4",
-    "@angular/forms": "~14.0.4",
-    "@angular/localize": "~14.0.4",
-    "@angular/platform-browser": "~14.0.4",
-    "@angular/platform-browser-dynamic": "~14.0.4",
-    "@angular/router": "~14.0.4",
-    "@ng-bootstrap/ng-bootstrap": "^13.0.0-beta.1",
+    "@angular/common": "~14.2.8",
+    "@angular/compiler": "~14.2.8",
+    "@angular/core": "~14.2.8",
+    "@angular/forms": "~14.2.8",
+    "@angular/localize": "~14.2.8",
+    "@angular/platform-browser": "~14.2.8",
+    "@angular/platform-browser-dynamic": "~14.2.8",
+    "@angular/router": "~14.2.8",
+    "@ng-bootstrap/ng-bootstrap": "^13.0.0",
     "@ng-select/ng-select": "^9.0.2",
     "@ngneat/dirty-check-forms": "^3.0.2",
-    "@popperjs/core": "^2.11.5",
-    "bootstrap": "^5.1.3",
+    "@popperjs/core": "^2.11.6",
+    "bootstrap": "^5.2.1",
     "file-saver": "^2.0.5",
-    "ng2-pdf-viewer": "^9.0.0",
-    "ngx-color": "^7.3.3",
+    "ng2-pdf-viewer": "^9.1.2",
+    "ngx-color": "^8.0.3",
     "ngx-cookie-service": "^14.0.1",
-    "ngx-file-drop": "^13.0.0",
-    "rxjs": "~7.5.5",
+    "ngx-file-drop": "^14.0.1",
+    "ngx-ui-tour-ng-bootstrap": "^11.1.0",
+    "rxjs": "~7.5.7",
     "tslib": "^2.3.1",
-    "uuid": "^8.3.1",
-    "zone.js": "~0.11.6"
+    "uuid": "^9.0.0",
+    "zone.js": "~0.11.8"
   },
   "devDependencies": {
-    "@angular-builders/jest": "14.0.0",
-    "@angular-devkit/build-angular": "~14.0.4",
-    "@angular/cli": "~14.0.4",
-    "@angular/compiler-cli": "~14.0.4",
-    "@types/jest": "28.1.4",
-    "@types/node": "^18.0.0",
+    "@angular-builders/jest": "14.0.1",
+    "@angular-devkit/build-angular": "~14.2.7",
+    "@angular/cli": "~14.2.7",
+    "@angular/compiler-cli": "~14.2.8",
+    "@types/jest": "28.1.6",
+    "@types/node": "^18.7.23",
     "codelyzer": "^6.0.2",
-    "concurrently": "7.2.2",
-    "jest": "28.1.2",
-    "jest-environment-jsdom": "^28.1.2",
-    "jest-preset-angular": "^12.1.0",
-    "ts-node": "~10.8.1",
+    "concurrently": "7.4.0",
+    "jest": "28.1.3",
+    "jest-environment-jsdom": "^29.2.2",
+    "jest-preset-angular": "^12.2.2",
+    "ts-node": "~10.9.1",
     "tslint": "~6.1.3",
-    "typescript": "~4.6.3",
+    "typescript": "~4.8.4",
     "wait-on": "~6.0.1"
   },
   "optionalDependencies": {
-    "@cypress/schematic": "^2.0.0",
-    "cypress": "~10.3.0"
+    "@cypress/schematic": "^2.1.1",
+    "cypress": "~10.9.0"
   }
 }

src-ui/src/app/app-routing.module.ts

@@ -14,16 +14,27 @@ import { DocumentAsnComponent } from './components/document-asn/document-asn.com
 import { DirtyFormGuard } from './guards/dirty-form.guard'
 import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component'
 import { TasksComponent } from './components/manage/tasks/tasks.component'
+import { DirtyDocGuard } from './guards/dirty-doc.guard'
+import { DirtySavedViewGuard } from './guards/dirty-saved-view.guard'
 
 const routes: Routes = [
   { path: '', redirectTo: 'dashboard', pathMatch: 'full' },
   {
     path: '',
     component: AppFrameComponent,
+    canDeactivate: [DirtyDocGuard],
     children: [
       { path: 'dashboard', component: DashboardComponent },
-      { path: 'documents', component: DocumentListComponent },
-      { path: 'view/:id', component: DocumentListComponent },
+      {
+        path: 'documents',
+        component: DocumentListComponent,
+        canDeactivate: [DirtySavedViewGuard],
+      },
+      {
+        path: 'view/:id',
+        component: DocumentListComponent,
+        canDeactivate: [DirtySavedViewGuard],
+      },
       { path: 'documents/:id', component: DocumentDetailComponent },
       { path: 'asn/:id', component: DocumentAsnComponent },
       { path: 'tags', component: TagListComponent },

src-ui/src/app/app.component.html

@@ -11,3 +11,28 @@
         </div>
     </ng-template>
 </ngx-file-drop>
+
+<tour-step-template>
+    <ng-template #tourStep let-step="step">
+        <p class="tour-step-content" [innerHTML]="step?.content"></p>
+        <hr/>
+        <div class="d-flex justify-content-between align-items-center">
+            <span class="badge bg-light text-dark">{{ tourService.steps?.indexOf(step) + 1 }} / {{ tourService.steps?.length }}</span>
+            <div class="tour-step-navigation btn-toolbar" role="toolbar" aria-label="Controls">
+                <div class="btn-group btn-group-sm me-2" role="group" aria-label="Dismiss">
+                    <button class="btn btn-outline-danger" (click)="tourService.end()">
+                        {{ step?.endBtnTitle }}
+                    </button>
+                </div>
+                <div class="btn-group btn-group-sm align-self-end" role="group" aria-label="Previous / Next">
+                    <button *ngIf="tourService.hasPrev(step)" class="btn btn-outline-primary" (click)="tourService.prev()">
+                        « {{ step?.prevBtnTitle }}
+                    </button>
+                    <button *ngIf="tourService.hasNext(step)" class="btn btn-outline-primary" (click)="tourService.next()">
+                        {{ step?.nextBtnTitle }} »
+                    </button>
+                </div>
+            </div>
+        </div>
+    </ng-template>
+</tour-step-template>

src-ui/src/app/app.component.ts

@@ -1,6 +1,6 @@
 import { SettingsService } from './services/settings.service'
 import { SETTINGS_KEYS } from './data/paperless-uisettings'
-import { Component, OnDestroy, OnInit } from '@angular/core'
+import { Component, OnDestroy, OnInit, Renderer2 } from '@angular/core'
 import { Router } from '@angular/router'
 import { Subscription } from 'rxjs'
 import { ConsumerStatusService } from './services/consumer-status.service'
@@ -8,6 +8,7 @@ import { ToastService } from './services/toast.service'
 import { NgxFileDropEntry } from 'ngx-file-drop'
 import { UploadDocumentsService } from './services/upload-documents.service'
 import { TasksService } from './services/tasks.service'
+import { TourService } from 'ngx-ui-tour-ng-bootstrap'
 
 @Component({
   selector: 'app-root',
@@ -29,7 +30,9 @@ export class AppComponent implements OnInit, OnDestroy {
     private toastService: ToastService,
     private router: Router,
     private uploadDocumentsService: UploadDocumentsService,
-    private tasksService: TasksService
+    private tasksService: TasksService,
+    public tourService: TourService,
+    private renderer: Renderer2
   ) {
     let anyWindow = window as any
     anyWindow.pdfWorkerSrc = 'assets/js/pdf.worker.min.js'
@@ -112,6 +115,121 @@ export class AppComponent implements OnInit, OnDestroy {
           })
         }
       })
+
+    const prevBtnTitle = $localize`Prev`
+    const nextBtnTitle = $localize`Next`
+    const endBtnTitle = $localize`End`
+
+    this.tourService.initialize([
+      {
+        anchorId: 'tour.dashboard',
+        content: $localize`The dashboard can be used to show saved views, such as an 'Inbox'. Those settings are found under Settings > Saved Views once you have created some.`,
+        route: '/dashboard',
+        enableBackdrop: true,
+        delayAfterNavigation: 500,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.upload-widget',
+        content: $localize`Drag-and-drop documents here to start uploading or place them in the consume folder. You can also drag-and-drop documents anywhere on all other pages of the web app. Once you do, Paperless-ngx will start training its machine learning algorithms.`,
+        route: '/dashboard',
+        enableBackdrop: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.documents',
+        content: $localize`The documents list shows all of your documents and allows for filtering as well as bulk-editing. There are three different view styles: list, small cards and large cards. A list of documents currently opened for editing is shown in the sidebar.`,
+        route: '/documents?sort=created&reverse=1&page=1',
+        delayAfterNavigation: 500,
+        placement: 'bottom',
+        enableBackdrop: true,
+        disableScrollToAnchor: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.documents-filter-editor',
+        content: $localize`The filtering tools allow you to quickly find documents using various searches, dates, tags, etc.`,
+        route: '/documents?sort=created&reverse=1&page=1',
+        placement: 'bottom',
+        enableBackdrop: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.documents-views',
+        content: $localize`Any combination of filters can be saved as a 'view' which can then be displayed on the dashboard and / or sidebar.`,
+        route: '/documents?sort=created&reverse=1&page=1',
+        enableBackdrop: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.tags',
+        content: $localize`Tags, correspondents, document types and storage paths can all be managed using these pages. They can also be created from the document edit view.`,
+        route: '/tags',
+        enableBackdrop: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.file-tasks',
+        content: $localize`File Tasks shows you documents that have been consumed, are waiting to be, or may have failed during the process.`,
+        route: '/tasks',
+        enableBackdrop: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.settings',
+        content: $localize`Check out the settings for various tweaks to the web app or to toggle settings for saved views.`,
+        route: '/settings',
+        enableBackdrop: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.admin',
+        content: $localize`The Admin area contains more advanced controls as well as the settings for automatic e-mail fetching.`,
+        enableBackdrop: true,
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+      {
+        anchorId: 'tour.outro',
+        title: $localize`Thank you! 🙏`,
+        content:
+          $localize`There are <em>tons</em> more features and info we didn't cover here, but this should get you started. Check out the documentation or visit the project on GitHub to learn more or to report issues.` +
+          '<br/><br/>' +
+          $localize`Lastly, on behalf of every contributor to this community-supported project, thank you for using Paperless-ngx!`,
+        route: '/dashboard',
+        prevBtnTitle,
+        nextBtnTitle,
+        endBtnTitle,
+      },
+    ])
+
+    this.tourService.start$.subscribe(() => {
+      this.renderer.addClass(document.body, 'tour-active')
+    })
+
+    this.tourService.end$.subscribe(() => {
+      // animation time
+      setTimeout(() => {
+        this.renderer.removeClass(document.body, 'tour-active')
+      }, 500)
+    })
   }
 
   public get dragDropEnabled(): boolean {

src-ui/src/app/app.module.ts

@@ -24,6 +24,7 @@ import { CorrespondentEditDialogComponent } from './components/common/edit-dialo
 import { TagEditDialogComponent } from './components/common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component'
 import { DocumentTypeEditDialogComponent } from './components/common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
 import { TagComponent } from './components/common/tag/tag.component'
+import { ClearableBadge } from './components/common/clearable-badge/clearable-badge.component'
 import { PageHeaderComponent } from './components/common/page-header/page-header.component'
 import { AppFrameComponent } from './components/app-frame/app-frame.component'
 import { ToastsComponent } from './components/common/toasts/toasts.component'
@@ -67,6 +68,14 @@ import { ApiVersionInterceptor } from './interceptors/api-version.interceptor'
 import { ColorSliderModule } from 'ngx-color/slider'
 import { ColorComponent } from './components/common/input/color/color.component'
 import { DocumentAsnComponent } from './components/document-asn/document-asn.component'
+import { DocumentCommentsComponent } from './components/document-comments/document-comments.component'
+import { DirtyDocGuard } from './guards/dirty-doc.guard'
+import { DirtySavedViewGuard } from './guards/dirty-saved-view.guard'
+import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component'
+import { StoragePathEditDialogComponent } from './components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
+import { SettingsService } from './services/settings.service'
+import { TasksComponent } from './components/manage/tasks/tasks.component'
+import { TourNgBootstrapModule } from 'ngx-ui-tour-ng-bootstrap'
 
 import localeBe from '@angular/common/locales/be'
 import localeCs from '@angular/common/locales/cs'
@@ -87,10 +96,6 @@ import localeSr from '@angular/common/locales/sr'
 import localeSv from '@angular/common/locales/sv'
 import localeTr from '@angular/common/locales/tr'
 import localeZh from '@angular/common/locales/zh'
-import { StoragePathListComponent } from './components/manage/storage-path-list/storage-path-list.component'
-import { StoragePathEditDialogComponent } from './components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
-import { SettingsService } from './services/settings.service'
-import { TasksComponent } from './components/manage/tasks/tasks.component'
 
 registerLocaleData(localeBe)
 registerLocaleData(localeCs)
@@ -138,6 +143,7 @@ function initializeApp(settings: SettingsService) {
     DocumentTypeEditDialogComponent,
     StoragePathEditDialogComponent,
     TagComponent,
+    ClearableBadge,
     PageHeaderComponent,
     AppFrameComponent,
     ToastsComponent,
@@ -172,6 +178,7 @@ function initializeApp(settings: SettingsService) {
     DateComponent,
     ColorComponent,
     DocumentAsnComponent,
+    DocumentCommentsComponent,
     TasksComponent,
   ],
   imports: [
@@ -185,6 +192,7 @@ function initializeApp(settings: SettingsService) {
     PdfViewerModule,
     NgSelectModule,
     ColorSliderModule,
+    TourNgBootstrapModule.forRoot(),
   ],
   providers: [
     {
@@ -209,6 +217,8 @@ function initializeApp(settings: SettingsService) {
     DocumentTitlePipe,
     { provide: NgbDateAdapter, useClass: ISODateAdapter },
     { provide: NgbDateParserFormatter, useClass: LocalizedDateParserFormatter },
+    DirtyDocGuard,
+    DirtySavedViewGuard,
   ],
   bootstrap: [AppComponent],
 })

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -4,11 +4,11 @@
     (click)="isMenuCollapsed = !isMenuCollapsed">
     <span class="navbar-toggler-icon"></span>
   </button>
-  <a class="navbar-brand col-auto col-md-3 col-lg-2 me-0 px-3 py-3 order-sm-0" routerLink="/dashboard">
+  <a class="navbar-brand col-auto col-md-3 col-lg-2 me-0 px-3 py-3 order-sm-0" [ngClass]="slimSidebarEnabled ? 'slim' : 'col-auto col-md-3 col-lg-2'" routerLink="/dashboard" tourAnchor="tour.intro">
     <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 198.43 238.91" width="1em" class="me-2" fill="currentColor">
       <path d="M194.7,0C164.22,70.94,17.64,79.74,64.55,194.06c.58,1.47-10.85,17-18.47,29.9-1.76-6.45-3.81-13.48-3.52-14.07,38.11-45.14-27.26-70.65-30.78-107.58C-4.64,131.62-10.5,182.92,39,212.53c.3,0,2.64,11.14,3.81,16.71a58.55,58.55,0,0,0-2.93,6.45c-1.17,2.93,7.62,2.64,7.62,3.22.88-.29,21.7-36.93,22.28-37.23C187.67,174.72,208.48,68.6,194.7,0ZM134.61,74.75C79.5,124,70.12,160.64,71.88,178.53,53.41,134.85,107.64,86.77,134.61,74.75ZM28.2,145.11c10.55,9.67,28.14,39.28,13.19,56.57C44.91,193.77,46.08,175.89,28.2,145.11Z" transform="translate(0 0)"/>
     </svg>
-    <ng-container i18n="app title">Paperless-ngx</ng-container>
+    <span class="ms-2" [class.visually-hidden]="slimSidebarEnabled" i18n="app title">Paperless-ngx</span>
   </a>
   <div class="search-form-container flex-grow-1 py-2 pb-3 pb-sm-2 px-3 ps-md-4 me-sm-auto order-3 order-sm-1">
     <form (ngSubmit)="search()" class="form-inline flex-grow-1">
@@ -16,12 +16,17 @@
         <use xlink:href="assets/bootstrap-icons.svg#search"/>
       </svg>
       <input class="form-control form-control-sm" type="text" placeholder="Search documents" aria-label="Search"
-        [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (selectItem)="itemSelected($event)" i18n-placeholder>
+        [formControl]="searchField" [ngbTypeahead]="searchAutoComplete" (keyup)="searchFieldKeyup($event)" (selectItem)="itemSelected($event)" i18n-placeholder>
+      <button type="button" *ngIf="!searchFieldEmpty" class="btn btn-link btn-sm px-0 position-absolute top-0 end-0" (click)="resetSearchField()">
+        <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-x me-1" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+          <path fill-rule="evenodd" d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z"/>
+        </svg>
+      </button>
     </form>
   </div>
   <ul ngbNav class="order-sm-3">
     <li ngbDropdown class="nav-item dropdown">
-      <button class="btn" id="userDropdown" ngbDropdownToggle>
+      <button class="btn border-0" id="userDropdown" ngbDropdownToggle>
         <span class="small me-2 d-none d-sm-inline">
           {{this.settingsService.displayName}}
         </span>
@@ -51,48 +56,54 @@
 
 <div class="container-fluid">
   <div class="row">
-    <nav id="sidebarMenu" class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse" [ngbCollapse]="isMenuCollapsed">
+    <nav id="sidebarMenu" class="d-md-block bg-light sidebar collapse" [ngClass]="slimSidebarEnabled ? 'slim' : 'col-md-3 col-lg-2'" [class.animating]="slimSidebarAnimating" [ngbCollapse]="isMenuCollapsed">
+      <button class="btn btn-sm btn-dark sidebar-slim-toggler" (click)="toggleSlimSidebar()">
+        <svg class="sidebaricon-sm" fill="currentColor">
+          <use *ngIf="slimSidebarEnabled" xlink:href="assets/bootstrap-icons.svg#chevron-double-right"/>
+          <use *ngIf="!slimSidebarEnabled" xlink:href="assets/bootstrap-icons.svg#chevron-double-left"/>
+        </svg>
+      </button>
       <div class="sidebar-sticky pt-3 d-flex flex-column justify-space-around">
         <ul class="nav flex-column">
           <li class="nav-item">
-            <a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Dashboard" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#house"/>
-              </svg>&nbsp;<ng-container i18n>Dashboard</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Dashboard</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" routerLink="documents" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="documents" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Documents" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#files"/>
-              </svg>&nbsp;<ng-container i18n>Documents</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Documents</ng-container></span>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='savedViewService.loading || savedViewService.sidebarViews.length > 0'>
-          <ng-container i18n>Saved views</ng-container>
+          <span i18n>Saved views</span>
           <div *ngIf="savedViewService.loading" class="spinner-border spinner-border-sm fw-normal ms-2" role="status"></div>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item w-100" *ngFor="let view of savedViewService.sidebarViews">
-            <a class="nav-link text-truncate" routerLink="view/{{view.id}}" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" [class.text-truncate]="!slimSidebarEnabled" routerLink="view/{{view.id}}" routerLinkActive="active" (click)="closeMenu()" [ngbPopover]="view.name" [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#funnel"/>
-              </svg>&nbsp;{{view.name}}
+              </svg><span>&nbsp;{{view.name}}</span>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted" *ngIf='openDocuments.length > 0'>
-          <ng-container i18n>Open documents</ng-container>
+          <span i18n>Open documents</span>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item w-100" *ngFor='let d of openDocuments'>
-            <a class="nav-link text-truncate" routerLink="documents/{{d.id}}" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" [class.text-truncate]="!slimSidebarEnabled" routerLink="documents/{{d.id}}" routerLinkActive="active" (click)="closeMenu()" [ngbPopover]="d.title | documentTitle" [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#file-text"/>
-              </svg>&nbsp;{{d.title | documentTitle}}
+              </svg><span>&nbsp;{{d.title | documentTitle}}</span>
               <span class="close" (click)="closeDocument(d); $event.preventDefault()">
                 <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="bi bi-x" viewBox="0 0 16 16">
                   <use xlink:href="assets/bootstrap-icons.svg#x"/>
@@ -101,95 +112,96 @@
             </a>
           </li>
           <li class="nav-item w-100" *ngIf="openDocuments.length >= 1">
-            <a class="nav-link text-truncate" [routerLink]="[]" (click)="closeAll()">
+            <a class="nav-link" [class.text-truncate]="!slimSidebarEnabled" [routerLink]="[]" (click)="closeAll()" ngbPopover="Close all" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#x"/>
-              </svg>&nbsp;<ng-container i18n>Close all</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Close all</ng-container></span>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
-          <ng-container i18n>Manage</ng-container>
+          <span i18n>Manage</span>
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item">
-            <a class="nav-link" routerLink="correspondents" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="correspondents" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Correspondents" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#person"/>
-              </svg>&nbsp;<ng-container i18n>Correspondents</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Correspondents</ng-container></span>
             </a>
           </li>
-          <li class="nav-item">
-            <a class="nav-link" routerLink="tags" routerLinkActive="active" (click)="closeMenu()">
+          <li class="nav-item" tourAnchor="tour.tags">
+            <a class="nav-link" routerLink="tags" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Tags" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#tags"/>
-              </svg>&nbsp;<ng-container i18n>Tags</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Tags</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" routerLink="documenttypes" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="documenttypes" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Document types" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#hash"/>
-              </svg>&nbsp;<ng-container i18n>Document types</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Document types</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" routerLink="storagepaths" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="storagepaths" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Storage paths" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#folder"/>
-              </svg>&nbsp;<ng-container i18n>Storage paths</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Storage paths</ng-container></span>
             </a>
           </li>
-          <li class="nav-item">
-            <a class="nav-link" routerLink="tasks" routerLinkActive="active" (click)="closeMenu()">
+          <li class="nav-item" tourAnchor="tour.file-tasks">
+            <a class="nav-link" routerLink="tasks" routerLinkActive="active" (click)="closeMenu()" ngbPopover="File Tasks" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
+              <span *ngIf="tasksService.failedFileTasks.length > 0 && slimSidebarEnabled" class="badge bg-danger position-absolute top-0 end-0">{{tasksService.failedFileTasks.length}}</span>
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#list-task"/>
-              </svg>&nbsp;<ng-container i18n>File Tasks<ng-container *ngIf="tasksService.failedFileTasks.length > 0"><span class="badge bg-danger ms-2">{{tasksService.failedFileTasks.length}}</span></ng-container></ng-container>
+              </svg><span>&nbsp;<ng-container i18n>File Tasks<span *ngIf="tasksService.failedFileTasks.length > 0"><span class="badge bg-danger ms-2">{{tasksService.failedFileTasks.length}}</span></span></ng-container></span>
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" routerLink="logs" routerLinkActive="active" (click)="closeMenu()">
+            <a class="nav-link" routerLink="logs" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Logs" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#text-left"/>
-              </svg>&nbsp;<ng-container i18n>Logs</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Logs</ng-container></span>
             </a>
           </li>
-          <li class="nav-item">
-            <a class="nav-link" routerLink="settings" routerLinkActive="active" (click)="closeMenu()">
+          <li class="nav-item" tourAnchor="tour.settings">
+            <a class="nav-link" routerLink="settings" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Settings" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#gear"/>
-              </svg>&nbsp;<ng-container i18n>Settings</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Settings</ng-container></span>
             </a>
           </li>
-          <li class="nav-item">
-            <a class="nav-link" href="admin/">
+          <li class="nav-item" tourAnchor="tour.admin">
+            <a class="nav-link" href="admin/" ngbPopover="Admin" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#toggles"/>
-              </svg>&nbsp;<ng-container i18n>Admin</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Admin</ng-container></span>
             </a>
           </li>
         </ul>
 
         <h6 class="sidebar-heading px-3 mt-auto pt-4 mb-1 text-muted">
-          <ng-container i18n>Info</ng-container>
+          <span i18n>Info</span>
         </h6>
         <ul class="nav flex-column mb-2">
-          <li class="nav-item">
-            <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://paperless-ngx.readthedocs.io/en/latest/">
+          <li class="nav-item" tourAnchor="tour.outro">
+            <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://paperless-ngx.readthedocs.io/en/latest/" ngbPopover="Documentation" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#question-circle"/>
-              </svg>&nbsp;<ng-container i18n>Documentation</ng-container>
+              </svg><span>&nbsp;<ng-container i18n>Documentation</ng-container></span>
             </a>
           </li>
           <li class="nav-item">
             <div class="d-flex w-100 flex-wrap">
-              <a class="nav-link pe-2 pb-1" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx">
+              <a class="nav-link pe-2 pb-1" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx" ngbPopover="GitHub" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end" container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
                 <svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" class="sidebaricon" viewBox="0 0 16 16">
                   <use xlink:href="assets/bootstrap-icons.svg#github" />
-                </svg>&nbsp;<ng-container i18n>GitHub</ng-container>
+                </svg><span>&nbsp;<ng-container i18n>GitHub</ng-container></span>
               </a>
-              <a class="nav-link-additional small text-muted ms-3" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests" title="Suggest an idea">
+              <a class="nav-link-additional small text-muted ms-3" [class.visually-hidden]="slimSidebarEnabled" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests" title="Suggest an idea" i18n-title>
                 <svg xmlns="http://www.w3.org/2000/svg" width="1.1em" height="1.1em" fill="currentColor" class="me-1" viewBox="0 0 16 16">
                   <use xlink:href="assets/bootstrap-icons.svg#lightbulb" />
                 </svg>
@@ -197,17 +209,28 @@
               </a>
             </div>
           </li>
-          <li class="nav-item mt-2">
+          <li class="nav-item mt-2" [class.visually-hidden]="slimSidebarEnabled">
             <div class="px-3 py-2 text-muted small d-flex align-items-center flex-wrap">
               <div class="me-3">{{ versionString }}</div>
-              <div *ngIf="appRemoteVersion" class="version-check">
+              <div *ngIf="!settingsService.updateCheckingIsSet || appRemoteVersion" class="version-check">
                 <ng-template #updateAvailablePopContent>
                   <span class="small">Paperless-ngx {{ appRemoteVersion.version }} <ng-container i18n>is available.</ng-container><br/><ng-container i18n>Click to view.</ng-container></span>
                 </ng-template>
                 <ng-template #updateCheckingNotEnabledPopContent>
-                  <span class="small"><ng-container i18n>Checking for updates is disabled.</ng-container><br/><ng-container i18n>Click for more information.</ng-container></span>
+                  <p class="small mb-2">
+                    <ng-container i18n>Paperless-ngx can automatically check for updates</ng-container>
+                  </p>
+                  <div class="btn-group btn-group-xs flex-fill w-100">
+                    <button class="btn btn-outline-primary" (click)="setUpdateChecking(true)">Enable</button>
+                    <button class="btn btn-outline-secondary" (click)="setUpdateChecking(false)">Disable</button>
+                  </div>
+                  <p class="small mb-0 mt-2">
+                    <a class="small text-decoration-none fst-italic" routerLink="/settings" fragment="update-checking" i18n>
+                      How does this work?
+                    </a>
+                  </p>
                 </ng-template>
-                <ng-container *ngIf="appRemoteVersion.feature_is_set; else updateCheckNotSet">
+                <ng-container *ngIf="settingsService.updateCheckingIsSet; else updateCheckNotSet">
                   <a *ngIf="appRemoteVersion.update_available" class="small text-decoration-none" target="_blank" rel="noopener noreferrer" href="https://github.com/paperless-ngx/paperless-ngx/releases"
                   [ngbPopover]="updateAvailablePopContent" popoverClass="shadow" triggers="mouseenter:mouseleave" container="body">
                     <svg fill="currentColor" class="me-1" width="1.2em" height="1.2em" style="vertical-align: text-top;" viewBox="0 0 16 16">
@@ -217,8 +240,8 @@
                   </a>
                 </ng-container>
                 <ng-template #updateCheckNotSet>
-                  <a class="small text-decoration-none" target="_blank" rel="noopener noreferrer" href="https://paperless-ngx.readthedocs.io/en/latest/configuration.html#update-checking"
-                  [ngbPopover]="updateCheckingNotEnabledPopContent" popoverClass="shadow" triggers="mouseenter:mouseleave" container="body">
+                  <a class="small text-decoration-none" routerLink="/settings" fragment="update-checking"
+                  [ngbPopover]="updateCheckingNotEnabledPopContent" popoverClass="shadow" triggers="mouseenter" container="body">
                     <svg fill="currentColor" class="me-1" width="1.2em" height="1.2em" style="vertical-align: text-top;" viewBox="0 0 16 16">
                       <use xlink:href="assets/bootstrap-icons.svg#info-circle" />
                     </svg>
@@ -231,7 +254,7 @@
       </div>
     </nav>
 
-    <main role="main" class="col-md-9 ms-sm-auto col-lg-10 px-md-4">
+    <main role="main" class="ms-sm-auto px-md-4" [ngClass]="slimSidebarEnabled ? 'col-slim' : 'col-md-9 col-lg-10'">
       <router-outlet></router-outlet>
     </main>
   </div>

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -1,3 +1,6 @@
+@import "node_modules/bootstrap/scss/functions";
+@import "node_modules/bootstrap/scss/variables";
+
 /*
  * Sidebar
  */
@@ -14,6 +17,17 @@
     width: 0.8em;
     height: 0.8em;
   }
+
+  // These come from the col-md-3 col-lg-2 classes for regular sidebar, needed for animation
+  @media (min-width: 768px) {
+    max-width: 25%;
+  }
+
+  @media (min-width: 992px) {
+    max-width: 16.66666667%;
+  }
+
+  transition: all .2s ease;
 }
 @media (max-width: 767.98px) {
   .sidebar {
@@ -21,6 +35,90 @@
   }
 }
 
+main {
+  transition: all .2s ease;
+}
+
+.sidebar-slim-toggler {
+  display: none; // hide on mobile
+}
+
+.sidebar li.nav-item span,
+.sidebar .sidebar-heading span {
+  transition: all .1s ease;
+}
+
+@media(min-width: 768px) {
+  .sidebar.slim {
+    max-width: 50px;
+
+    li.nav-item span.badge {
+      display: inline-block;
+      margin-right: 2px;
+    }
+  }
+
+  .sidebar.slim:not(.animating) {
+    li.nav-item span,
+    .sidebar-heading span {
+      display: none;
+    }
+  }
+
+  .sidebar.animating {
+    li.nav-item span,
+    .sidebar-heading span {
+      display: unset;
+      position: absolute;
+      opacity: 0;
+      overflow: hidden;
+    }
+  }
+
+  .sidebar:not(.slim):not(.animating) {
+    li.nav-item span,
+    .sidebar-heading span {
+      position: unset;
+      opacity: 1;
+      overflow: auto;
+    }
+  }
+
+  .sidebar.slim,
+  .sidebar.animating {
+    .text-truncate {
+      text-overflow: unset !important;
+      word-wrap: break-word !important;
+    }
+  }
+
+  .sidebar.slim {
+    li.nav-item span.badge {
+      display: inline-block;
+      margin-right: 2px;
+    }
+  }
+
+  .col-slim {
+    padding-left: calc(50px + $grid-gutter-width) !important;
+  }
+
+  .sidebar-slim-toggler {
+    display: block;
+    position: absolute;
+    right: -12px;
+    top: 60px;
+    z-index: 996;
+    --bs-btn-padding-x: 0.35rem;
+    --bs-btn-padding-y: 0.125rem;
+  }
+}
+
+::ng-deep .popover-slim .popover-body {
+  --bs-popover-body-padding-x: .5rem;
+  --bs-popover-body-padding-y: .5rem;
+}
+
 .sidebar-sticky {
   position: relative;
   top: 0;
@@ -77,7 +175,7 @@
 
   .close {
     display: none;
-    position: absolute;
+    position: absolute !important;
     cursor: pointer;
     opacity: 1;
     top: 0;
@@ -145,17 +243,18 @@
 
   form {
     position: relative;
+
+    > svg {
+      position: absolute;
+      left: 0.6rem;
+      top: 0.5rem;
+      color: rgba(255, 255, 255, 0.6);
+    }
   }
 
-  svg {
-    position: absolute;
-    left: 0.6rem;
-    top: 0.5rem;
-    color: rgba(255, 255, 255, 0.6);
-  }
 
   &:focus-within {
-    svg {
+    form > svg {
       display: none;
     }
 

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -1,6 +1,6 @@
-import { Component } from '@angular/core'
+import { Component, HostListener, OnInit } from '@angular/core'
 import { FormControl } from '@angular/forms'
-import { ActivatedRoute, Router, Params } from '@angular/router'
+import { ActivatedRoute, Router } from '@angular/router'
 import { from, Observable } from 'rxjs'
 import {
   debounceTime,
@@ -23,13 +23,16 @@ import {
 } from 'src/app/services/rest/remote-version.service'
 import { SettingsService } from 'src/app/services/settings.service'
 import { TasksService } from 'src/app/services/tasks.service'
+import { ComponentCanDeactivate } from 'src/app/guards/dirty-doc.guard'
+import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings'
+import { ToastService } from 'src/app/services/toast.service'
 
 @Component({
   selector: 'app-app-frame',
   templateUrl: './app-frame.component.html',
   styleUrls: ['./app-frame.component.scss'],
 })
-export class AppFrameComponent {
+export class AppFrameComponent implements OnInit, ComponentCanDeactivate {
   constructor(
     public router: Router,
     private activatedRoute: ActivatedRoute,
@@ -39,14 +42,15 @@ export class AppFrameComponent {
     private remoteVersionService: RemoteVersionService,
     private list: DocumentListViewService,
     public settingsService: SettingsService,
-    public tasksService: TasksService
-  ) {
-    this.remoteVersionService
-      .checkForUpdates()
-      .subscribe((appRemoteVersion: AppRemoteVersion) => {
-        this.appRemoteVersion = appRemoteVersion
-      })
-    tasksService.reload()
+    public tasksService: TasksService,
+    private readonly toastService: ToastService
+  ) {}
+
+  ngOnInit(): void {
+    if (this.settingsService.get(SETTINGS_KEYS.UPDATE_CHECKING_ENABLED)) {
+      this.checkForUpdates()
+    }
+    this.tasksService.reload()
   }
 
   versionString = `${environment.appTitle} ${environment.version}`
@@ -54,16 +58,64 @@ export class AppFrameComponent {
 
   isMenuCollapsed: boolean = true
 
+  slimSidebarAnimating: boolean = false
+
+  toggleSlimSidebar(): void {
+    this.slimSidebarAnimating = true
+    this.slimSidebarEnabled = !this.slimSidebarEnabled
+    setTimeout(() => {
+      this.slimSidebarAnimating = false
+    }, 200) // slightly longer than css animation for slim sidebar
+  }
+
+  get slimSidebarEnabled(): boolean {
+    return this.settingsService.get(SETTINGS_KEYS.SLIM_SIDEBAR)
+  }
+
+  set slimSidebarEnabled(enabled: boolean) {
+    this.settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, enabled)
+    this.settingsService
+      .storeSettings()
+      .pipe(first())
+      .subscribe({
+        error: (error) => {
+          this.toastService.showError(
+            $localize`An error occurred while saving settings.`
+          )
+          console.log(error)
+        },
+      })
+  }
+
   closeMenu() {
     this.isMenuCollapsed = true
   }
 
-  searchField = new FormControl('')
-
   get openDocuments(): PaperlessDocument[] {
     return this.openDocumentsService.getOpenDocuments()
   }
 
+  @HostListener('window:beforeunload')
+  canDeactivate(): Observable<boolean> | boolean {
+    return !this.openDocumentsService.hasDirty()
+  }
+
+  searchField = new FormControl('')
+
+  get searchFieldEmpty(): boolean {
+    return this.searchField.value.trim().length == 0
+  }
+
+  resetSearchField() {
+    this.searchField.reset('')
+  }
+
+  searchFieldKeyup(event: KeyboardEvent) {
+    if (event.key == 'Escape') {
+      this.resetSearchField()
+    }
+  }
+
   searchAutoComplete = (text$: Observable<string>) =>
     text$.pipe(
       debounceTime(200),
@@ -144,4 +196,30 @@ export class AppFrameComponent {
         }
       })
   }
+
+  private checkForUpdates() {
+    this.remoteVersionService
+      .checkForUpdates()
+      .subscribe((appRemoteVersion: AppRemoteVersion) => {
+        this.appRemoteVersion = appRemoteVersion
+      })
+  }
+
+  setUpdateChecking(enable: boolean) {
+    this.settingsService.set(SETTINGS_KEYS.UPDATE_CHECKING_ENABLED, enable)
+    this.settingsService
+      .storeSettings()
+      .pipe(first())
+      .subscribe({
+        error: (error) => {
+          this.toastService.showError(
+            $localize`An error occurred while saving update checking settings.`
+          )
+          console.log(error)
+        },
+      })
+    if (enable) {
+      this.checkForUpdates()
+    }
+  }
 }

src-ui/src/app/components/common/clearable-badge/clearable-badge.component.html

@@ -0,0 +1,9 @@
+<button *ngIf="active" class="position-absolute top-0 start-100 translate-middle badge bg-secondary border border-light rounded-pill p-1" title="Clear" i18n-title (click)="onClick($event)">
+    <svg *ngIf="!isNumbered && selected" width="1em" height="1em" class="check m-0 p-0 opacity-75" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+        <use xlink:href="assets/bootstrap-icons.svg#check-lg"/>
+    </svg>
+    <div *ngIf="isNumbered" class="number">{{number}}<span class="visually-hidden">selected</span></div>
+    <svg width=".9em" height="1em" class="x m-0 p-0 opacity-75" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+        <use xlink:href="assets/bootstrap-icons.svg#x-lg"/>
+    </svg>
+</button>

src-ui/src/app/components/common/clearable-badge/clearable-badge.component.scss

@@ -0,0 +1,28 @@
+.badge {
+    min-width: 20px;
+    min-height: 20px;
+}
+
+.x {
+    display: none;
+}
+
+.number {
+    min-width: 1em;
+    min-height: 1em;
+    display: inline-block;
+}
+
+button:hover {
+    .check,
+    .number {
+        opacity: 0 !important;
+    }
+
+    .x {
+        display: inline-block;
+        position: absolute;
+        top: 5px;
+        left: calc(50% - 4px);
+    }
+}

src-ui/src/app/components/common/clearable-badge/clearable-badge.component.ts

@@ -0,0 +1,33 @@
+import { Component, Input, Output, EventEmitter } from '@angular/core'
+
+@Component({
+  selector: 'app-clearable-badge',
+  templateUrl: './clearable-badge.component.html',
+  styleUrls: ['./clearable-badge.component.scss'],
+})
+export class ClearableBadge {
+  constructor() {}
+
+  @Input()
+  number: number
+
+  @Input()
+  selected: boolean
+
+  @Output()
+  cleared: EventEmitter<boolean> = new EventEmitter()
+
+  get active(): boolean {
+    return this.selected || this.number > -1
+  }
+
+  get isNumbered(): boolean {
+    return this.number > -1
+  }
+
+  onClick(event: PointerEvent) {
+    this.cleared.emit(true)
+    event.stopImmediatePropagation()
+    event.preventDefault()
+  }
+}

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html

@@ -16,4 +16,7 @@
         <ngb-progressbar *ngIf="!confirmButtonEnabled" style="height: 1px;" type="dark" [max]="secondsTotal" [value]="seconds"></ngb-progressbar>
         <span class="visually-hidden">{{ seconds | number: '1.0-0' }} seconds</span>
       </button>
+      <button *ngIf="alternativeBtnCaption" type="button" class="btn" [class]="alternativeBtnClass" (click)="alternative()" [disabled]="!alternativeButtonEnabled || !buttonsEnabled">
+        {{alternativeBtnCaption}}
+      </button>
     </div>

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts

@@ -13,6 +13,9 @@ export class ConfirmDialogComponent {
   @Output()
   public confirmClicked = new EventEmitter()
 
+  @Output()
+  public alternativeClicked = new EventEmitter()
+
   @Input()
   title = $localize`Confirmation`
 
@@ -28,14 +31,22 @@ export class ConfirmDialogComponent {
   @Input()
   btnCaption = $localize`Confirm`
 
+  @Input()
+  alternativeBtnClass = 'btn-secondary'
+
+  @Input()
+  alternativeBtnCaption
+
   @Input()
   buttonsEnabled = true
 
   confirmButtonEnabled = true
+  alternativeButtonEnabled = true
   seconds = 0
   secondsTotal = 0
 
   confirmSubject: Subject<boolean>
+  alternativeSubject: Subject<boolean>
 
   delayConfirm(seconds: number) {
     const refreshInterval = 0.15 // s
@@ -68,4 +79,10 @@ export class ConfirmDialogComponent {
     this.confirmSubject?.next(true)
     this.confirmSubject?.complete()
   }
+
+  alternative() {
+    this.alternativeClicked.emit()
+    this.alternativeSubject?.next(true)
+    this.alternativeSubject?.complete()
+  }
 }

src-ui/src/app/components/common/date-dropdown/date-dropdown.component.html

@@ -1,11 +1,17 @@
   <div class="btn-group w-100" ngbDropdown role="group">
   <button class="btn btn-sm" id="dropdown{{title}}" ngbDropdownToggle [ngClass]="dateBefore || dateAfter ? 'btn-primary' : 'btn-outline-primary'">
     {{title}}
+    <app-clearable-badge [selected]="isActive" (cleared)="reset()"></app-clearable-badge><span class="visually-hidden">selected</span>
   </button>
   <div class="dropdown-menu date-dropdown shadow pt-0" ngbDropdownMenu attr.aria-labelledby="dropdown{{title}}">
     <div class="list-group list-group-flush">
-        <button *ngFor="let qf of quickFilters" class="list-group-item small list-goup list-group-item-action d-flex p-2 ps-3" role="menuitem" (click)="setDateQuickFilter(qf.id)">
-          {{qf.name}}
+        <button *ngFor="let rd of relativeDates" class="list-group-item small list-goup list-group-item-action d-flex p-2" role="menuitem" (click)="setRelativeDate(rd.date)">
+          <div _ngcontent-hga-c166="" class="selected-icon me-1">
+            <svg *ngIf="relativeDate === rd.date" xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" fill="currentColor" class="bi bi-check" viewBox="0 0 16 16">
+              <path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
+            </svg>
+          </div>
+          {{rd.name}}
         </button>
         <div class="list-group-item d-flex flex-column align-items-start" role="menuitem">
 

src-ui/src/app/components/common/date-dropdown/date-dropdown.component.scss

@@ -5,3 +5,8 @@
     line-height: 1;
   }
 }
+
+.selected-icon {
+  min-width: 1em;
+  min-height: 1em;
+}

src-ui/src/app/components/common/date-dropdown/date-dropdown.component.ts

@@ -16,12 +16,15 @@ import { ISODateAdapter } from 'src/app/utils/ngb-iso-date-adapter'
 export interface DateSelection {
   before?: string
   after?: string
+  relativeDateID?: number
 }
 
-const LAST_7_DAYS = 0
-const LAST_MONTH = 1
-const LAST_3_MONTHS = 2
-const LAST_YEAR = 3
+export enum RelativeDate {
+  LAST_7_DAYS = 0,
+  LAST_MONTH = 1,
+  LAST_3_MONTHS = 2,
+  LAST_YEAR = 3,
+}
 
 @Component({
   selector: 'app-date-dropdown',
@@ -34,11 +37,23 @@ export class DateDropdownComponent implements OnInit, OnDestroy {
     this.datePlaceHolder = settings.getLocalizedDateInputFormat()
   }
 
-  quickFilters = [
-    { id: LAST_7_DAYS, name: $localize`Last 7 days` },
-    { id: LAST_MONTH, name: $localize`Last month` },
-    { id: LAST_3_MONTHS, name: $localize`Last 3 months` },
-    { id: LAST_YEAR, name: $localize`Last year` },
+  relativeDates = [
+    {
+      date: RelativeDate.LAST_7_DAYS,
+      name: $localize`Last 7 days`,
+    },
+    {
+      date: RelativeDate.LAST_MONTH,
+      name: $localize`Last month`,
+    },
+    {
+      date: RelativeDate.LAST_3_MONTHS,
+      name: $localize`Last 3 months`,
+    },
+    {
+      date: RelativeDate.LAST_YEAR,
+      name: $localize`Last year`,
+    },
   ]
 
   datePlaceHolder: string
@@ -55,12 +70,26 @@ export class DateDropdownComponent implements OnInit, OnDestroy {
   @Output()
   dateAfterChange = new EventEmitter<string>()
 
+  @Input()
+  relativeDate: RelativeDate
+
+  @Output()
+  relativeDateChange = new EventEmitter<number>()
+
   @Input()
   title: string
 
   @Output()
   datesSet = new EventEmitter<DateSelection>()
 
+  get isActive(): boolean {
+    return (
+      this.relativeDate !== null ||
+      this.dateAfter?.length > 0 ||
+      this.dateBefore?.length > 0
+    )
+  }
+
   private datesSetDebounce$ = new Subject()
 
   private sub: Subscription
@@ -77,37 +106,33 @@ export class DateDropdownComponent implements OnInit, OnDestroy {
     }
   }
 
-  setDateQuickFilter(qf: number) {
+  reset() {
     this.dateBefore = null
-    let date = new Date()
-    switch (qf) {
-      case LAST_7_DAYS:
-        date.setDate(date.getDate() - 7)
-        break
+    this.dateAfter = null
+    this.relativeDate = null
+    this.onChange()
+  }
 
-      case LAST_MONTH:
-        date.setMonth(date.getMonth() - 1)
-        break
-
-      case LAST_3_MONTHS:
-        date.setMonth(date.getMonth() - 3)
-        break
-
-      case LAST_YEAR:
-        date.setFullYear(date.getFullYear() - 1)
-        break
-    }
-    this.dateAfter = formatDate(date, 'yyyy-MM-dd', 'en-us', 'UTC')
+  setRelativeDate(rd: RelativeDate) {
+    this.dateBefore = null
+    this.dateAfter = null
+    this.relativeDate = this.relativeDate == rd ? null : rd
     this.onChange()
   }
 
   onChange() {
-    this.dateAfterChange.emit(this.dateAfter)
     this.dateBeforeChange.emit(this.dateBefore)
-    this.datesSet.emit({ after: this.dateAfter, before: this.dateBefore })
+    this.dateAfterChange.emit(this.dateAfter)
+    this.relativeDateChange.emit(this.relativeDate)
+    this.datesSet.emit({
+      after: this.dateAfter,
+      before: this.dateBefore,
+      relativeDateID: this.relativeDate,
+    })
   }
 
   onChangeDebounce() {
+    this.relativeDate = null
     this.datesSetDebounce$.next({
       after: this.dateAfter,
       before: this.dateBefore,

src-ui/src/app/components/common/edit-dialog/correspondent-edit-dialog/correspondent-edit-dialog.component.ts

@@ -2,6 +2,7 @@ import { Component } from '@angular/core'
 import { FormControl, FormGroup } from '@angular/forms'
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
 import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component'
+import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
 import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
 import { ToastService } from 'src/app/services/toast.service'
@@ -31,7 +32,7 @@ export class CorrespondentEditDialogComponent extends EditDialogComponent<Paperl
   getForm(): FormGroup {
     return new FormGroup({
       name: new FormControl(''),
-      matching_algorithm: new FormControl(1),
+      matching_algorithm: new FormControl(DEFAULT_MATCHING_ALGORITHM),
       match: new FormControl(''),
       is_insensitive: new FormControl(true),
     })

src-ui/src/app/components/common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component.ts

@@ -2,6 +2,7 @@ import { Component } from '@angular/core'
 import { FormControl, FormGroup } from '@angular/forms'
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
 import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component'
+import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
 import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
 import { ToastService } from 'src/app/services/toast.service'
@@ -31,7 +32,7 @@ export class DocumentTypeEditDialogComponent extends EditDialogComponent<Paperle
   getForm(): FormGroup {
     return new FormGroup({
       name: new FormControl(''),
-      matching_algorithm: new FormControl(1),
+      matching_algorithm: new FormControl(DEFAULT_MATCHING_ALGORITHM),
       match: new FormControl(''),
       is_insensitive: new FormControl(true),
     })

src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts

@@ -2,6 +2,7 @@ import { Component } from '@angular/core'
 import { FormControl, FormGroup } from '@angular/forms'
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
 import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component'
+import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
 import { StoragePathService } from 'src/app/services/rest/storage-path.service'
 import { ToastService } from 'src/app/services/toast.service'
@@ -42,7 +43,7 @@ export class StoragePathEditDialogComponent extends EditDialogComponent<Paperles
     return new FormGroup({
       name: new FormControl(''),
       path: new FormControl(''),
-      matching_algorithm: new FormControl(1),
+      matching_algorithm: new FormControl(DEFAULT_MATCHING_ALGORITHM),
       match: new FormControl(''),
       is_insensitive: new FormControl(true),
     })

src-ui/src/app/components/common/edit-dialog/tag-edit-dialog/tag-edit-dialog.component.ts

@@ -6,6 +6,7 @@ import { PaperlessTag } from 'src/app/data/paperless-tag'
 import { TagService } from 'src/app/services/rest/tag.service'
 import { ToastService } from 'src/app/services/toast.service'
 import { randomColor } from 'src/app/utils/color'
+import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model'
 
 @Component({
   selector: 'app-tag-edit-dialog',
@@ -34,7 +35,7 @@ export class TagEditDialogComponent extends EditDialogComponent<PaperlessTag> {
       name: new FormControl(''),
       color: new FormControl(randomColor()),
       is_inbox_tag: new FormControl(false),
-      matching_algorithm: new FormControl(1),
+      matching_algorithm: new FormControl(DEFAULT_MATCHING_ALGORITHM),
       match: new FormControl(''),
       is_insensitive: new FormControl(true),
     })

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.html

@@ -5,12 +5,7 @@
     </svg>
     <div class="d-none d-sm-inline">&nbsp;{{title}}</div>
     <ng-container *ngIf="!editing && selectionModel.selectionSize() > 0">
-      <div *ngIf="multiple" class="position-absolute top-0 start-100 translate-middle badge bg-secondary border border-light text-light rounded-pill">
-        {{selectionModel.totalCount}}<span class="visually-hidden">selected</span>
-      </div>
-      <div *ngIf="!multiple" class="position-absolute top-0 start-100 p-2 translate-middle badge bg-secondary border border-light rounded-circle">
-        <span class="visually-hidden">selected</span>
-      </div>
+      <app-clearable-badge [number]="multiple ? selectionModel.totalCount : undefined" [selected]="!multiple && selectionModel.selectionSize() > 0" (cleared)="reset()"></app-clearable-badge>
     </ng-container>
   </button>
   <div class="dropdown-menu py-0 shadow" ngbDropdownMenu attr.aria-labelledby="dropdown{{title}}">

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.scss

@@ -17,25 +17,6 @@
   }
 }
 
-.btn-group-xs {
-  > .btn {
-    padding: 0.2rem 0.25rem;
-    font-size: 0.675rem;
-    line-height: 1.2;
-    border-radius: 0.15rem;
-  }
-
-  > .btn:not(:first-child) {
-    border-top-left-radius: 0;
-    border-bottom-left-radius: 0;
-  }
-
-  > .btn:not(:last-child) {
-    border-top-right-radius: 0;
-    border-bottom-right-radius: 0;
-  }
-}
-
 .btn-group > label.disabled {
   filter: brightness(0.5);
 

src-ui/src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts

@@ -384,4 +384,9 @@ export class FilterableDropdownComponent {
       this.selectionModel.exclude(itemID)
     }
   }
+
+  reset() {
+    this.selectionModel.reset()
+    this.selectionModelChange.emit(this.selectionModel)
+  }
 }

src-ui/src/app/components/common/input/date/date.component.html

@@ -12,4 +12,10 @@
   </div>
   <div class="invalid-feedback" i18n>Invalid date.</div>
   <small *ngIf="hint" class="form-text text-muted">{{hint}}</small>
+  <small *ngIf="getSuggestions().length > 0">
+    <span i18n>Suggestions:</span>&nbsp;
+    <ng-container *ngFor="let s of getSuggestions()">
+      <a (click)="onSuggestionClick(s)" [routerLink]="[]">{{s}}</a>&nbsp;
+    </ng-container>
+  </small>
 </div>

src-ui/src/app/components/common/input/date/date.component.ts

@@ -1,8 +1,10 @@
-import { Component, forwardRef, OnInit } from '@angular/core'
+import { Component, forwardRef, Input, OnInit } from '@angular/core'
 import { NG_VALUE_ACCESSOR } from '@angular/forms'
-import { NgbDateParserFormatter } from '@ng-bootstrap/ng-bootstrap'
+import {
+  NgbDateAdapter,
+  NgbDateParserFormatter,
+} from '@ng-bootstrap/ng-bootstrap'
 import { SettingsService } from 'src/app/services/settings.service'
-import { LocalizedDateParserFormatter } from 'src/app/utils/ngb-date-parser-formatter'
 import { AbstractInputComponent } from '../abstract-input'
 
 @Component({
@@ -23,11 +25,34 @@ export class DateComponent
 {
   constructor(
     private settings: SettingsService,
-    private ngbDateParserFormatter: NgbDateParserFormatter
+    private ngbDateParserFormatter: NgbDateParserFormatter,
+    private isoDateAdapter: NgbDateAdapter<string>
   ) {
     super()
   }
 
+  @Input()
+  suggestions: string[]
+
+  getSuggestions() {
+    return this.suggestions == null
+      ? []
+      : this.suggestions
+          .map((s) => this.ngbDateParserFormatter.parse(s))
+          .filter(
+            (d) =>
+              this.value === null || // if value is not set, take all suggestions
+              this.value != this.isoDateAdapter.toModel(d) // otherwise filter out current date
+          )
+          .map((s) => this.ngbDateParserFormatter.format(s))
+  }
+
+  onSuggestionClick(dateString: string) {
+    const parsedDate = this.ngbDateParserFormatter.parse(dateString)
+    this.writeValue(this.isoDateAdapter.toModel(parsedDate))
+    this.onChange(this.value)
+  }
+
   ngOnInit(): void {
     super.ngOnInit()
     this.placeholder = this.settings.getLocalizedDateInputFormat()
@@ -43,9 +68,10 @@ export class DateComponent
       let pastedText = clipboardData.getData('text')
       pastedText = pastedText.replace(/[\sa-z#!$%\^&\*;:{}=\-_`~()]+/g, '')
       const parsedDate = this.ngbDateParserFormatter.parse(pastedText)
-      const formattedDate = this.ngbDateParserFormatter.format(parsedDate)
-      this.writeValue(formattedDate)
-      this.onChange(formattedDate)
+      if (parsedDate) {
+        this.writeValue(this.isoDateAdapter.toModel(parsedDate))
+        this.onChange(this.value)
+      }
     }
   }
 

src-ui/src/app/components/dashboard/dashboard.component.html

@@ -19,17 +19,20 @@
   </svg>
 </app-page-header>
 
-<div class='row'>
+<div class="row">
   <div class="col-lg-8">
     <ng-container *ngIf="savedViewService.loading">
       <div class="spinner-border spinner-border-sm me-2" role="status"></div>
       <ng-container i18n>Loading...</ng-container>
     </ng-container>
 
-    <app-welcome-widget *ngIf="!savedViewService.loading && savedViewService.dashboardViews.length == 0"></app-welcome-widget>
+    <app-welcome-widget *ngIf="settingsService.offerTour()" tourAnchor="tour.dashboard"></app-welcome-widget>
 
-    <ng-container *ngFor="let v of savedViewService.dashboardViews">
-      <app-saved-view-widget [savedView]="v"></app-saved-view-widget>
+    <ng-container *ngFor="let v of savedViewService.dashboardViews; first as isFirst">
+      <app-saved-view-widget *ngIf="isFirst; else noTour" [savedView]="v" tourAnchor="tour.dashboard"></app-saved-view-widget>
+      <ng-template #noTour>
+        <app-saved-view-widget [savedView]="v"></app-saved-view-widget>
+      </ng-template>
     </ng-container>
 
   </div>

src-ui/src/app/components/dashboard/dashboard.component.ts

@@ -1,5 +1,4 @@
-import { Component, OnInit } from '@angular/core'
-import { Meta } from '@angular/platform-browser'
+import { Component } from '@angular/core'
 import { SavedViewService } from 'src/app/services/rest/saved-view.service'
 import { SettingsService } from 'src/app/services/settings.service'
 
@@ -16,9 +15,9 @@ export class DashboardComponent {
 
   get subtitle() {
     if (this.settingsService.displayName) {
-      return $localize`Hello ${this.settingsService.displayName}, welcome to Paperless-ngx!`
+      return $localize`Hello ${this.settingsService.displayName}, welcome to Paperless-ngx`
     } else {
-      return $localize`Welcome to Paperless-ngx!`
+      return $localize`Welcome to Paperless-ngx`
     }
   }
 }

src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html

@@ -8,7 +8,7 @@
       </svg>
     </a>
   </div>
-  <div content>
+  <div content tourAnchor="tour.upload-widget">
     <form>
       <ngx-file-drop dropZoneLabel="Drop documents here or" browseBtnLabel="Browse files" (onFileDrop)="dropped($event)"
         (onFileOver)="fileOver($event)" (onFileLeave)="fileLeave($event)" dropZoneClassName="bg-light card"

src-ui/src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.html

@@ -1,16 +1,11 @@
-<app-widget-frame title="First steps" i18n-title>
-
-  <ng-container content>
-    <img src="assets/save-filter.png" class="float-right">
-    <p i18n>Paperless is running! :)</p>
-    <p i18n>You can start uploading documents by dropping them in the file upload box to the right or by dropping them in the configured consumption folder and they'll start showing up in the documents list.
-      After you've added some metadata to your documents, use the filtering mechanisms of paperless to create custom views (such as 'Recently added', 'Tagged TODO') and they will appear on the dashboard instead of this message.</p>
-    <p i18n>Paperless offers some more features that try to make your life easier:</p>
-    <ul>
-      <li i18n>Once you've got a couple documents in paperless and added metadata to them, paperless can assign that metadata to new documents automatically.</li>
-      <li i18n>You can configure paperless to read your mails and add documents from attached files.</li>
-    </ul>
-    <p i18n>Consult the documentation on how to use these features. The section on basic usage also has some information on how to use paperless in general.</p>
-  </ng-container>
-
-</app-widget-frame>
+<ngb-alert type="primary" [dismissible]="false">
+  <!-- [dismissible]="isFinished(status)" (closed)="dismiss(status)" -->
+  <h4 class="alert-heading"><ng-container i18n>Paperless-ngx is running!</ng-container> 🎉</h4>
+  <p i18n>You're ready to start uploading documents! Explore the various features of this web app on your own, or start a quick tour using the button below.</p>
+  <p i18n>More detail on how to use and configure Paperless-ngx is always available in the <a href="https://paperless-ngx.readthedocs.io" target="_blank">documentation</a>.</p>
+  <hr>
+  <div class="d-flex align-items-end">
+    <p class="lead fs-6 m-0"><em i18n>Thanks for being a part of the Paperless-ngx community!</em></p>
+    <button class="btn btn-primary ms-auto flex-shrink-0" (click)="tourService.start()"><ng-container i18n>Start the tour</ng-container> &rarr;</button>
+  </div>
+</ngb-alert>

src-ui/src/app/components/dashboard/widgets/welcome-widget/welcome-widget.component.ts

@@ -1,4 +1,5 @@
 import { Component, OnInit } from '@angular/core'
+import { TourService } from 'ngx-ui-tour-ng-bootstrap'
 
 @Component({
   selector: 'app-welcome-widget',
@@ -6,7 +7,7 @@ import { Component, OnInit } from '@angular/core'
   styleUrls: ['./welcome-widget.component.scss'],
 })
 export class WelcomeWidgetComponent implements OnInit {
-  constructor() {}
+  constructor(public readonly tourService: TourService) {}
 
   ngOnInit(): void {}
 }

src-ui/src/app/components/document-comments/document-comments.component.html

@@ -0,0 +1,28 @@
+<div *ngIf="comments">
+    <form [formGroup]="commentForm" class="needs-validation mt-3" novalidate>
+        <div class="form-group">
+            <textarea class="form-control form-control-sm" [class.is-invalid]="newCommentError" rows="3" formControlName="newComment" placeholder="Enter comment" i18n-placeholder required></textarea>
+            <div class="invalid-feedback" i18n>
+                Please enter a comment.
+            </div>
+        </div>
+        <div class="form-group mt-2 d-flex justify-content-end align-items-center">
+            <div *ngIf="networkActive" class="spinner-border spinner-border-sm fw-normal me-auto" role="status"></div>
+            <button type="button" class="btn btn-primary btn-sm" [disabled]="networkActive" (click)="addComment()" i18n>Add comment</button>
+        </div>
+    </form>
+    <hr>
+    <div *ngFor="let comment of comments" class="card border mb-3">
+        <div class="card-body text-dark">
+            <p class="card-text">{{comment.comment}}</p>
+        </div>
+        <div class="d-flex card-footer small bg-light text-primary justify-content-between align-items-center">
+            <span>{{displayName(comment)}} - {{ comment.created | customDate}}</span>
+            <button type="button" class="btn btn-link btn-sm p-0 fade" (click)="deleteComment(comment.id)">
+                <svg width="13" height="13" fill="currentColor">
+                    <use xlink:href="assets/bootstrap-icons.svg#trash" />
+                </svg>
+            </button>
+        </div>
+    </div>
+</div>

src-ui/src/app/components/document-comments/document-comments.component.scss

@@ -0,0 +1,9 @@
+.card-body {
+    max-height: 12rem;
+    overflow: scroll;
+    white-space: pre-wrap;
+}
+
+.card:hover .fade {
+    opacity: 1;
+}