Bump version to beta

* New translations messages.xlf (German)
[ci skip]

* New translations messages.xlf (German)
[ci skip]

* New translations messages.xlf (German)
[ci skip]

.build-config.json

@@ -0,0 +1,9 @@
+{
+  "qpdf": {
+      "version": "10.6.3"
+    },
+  "jbig2enc": {
+      "version": "0.29",
+      "git_tag": "0.29"
+    }
+}

.editorconfig

@@ -33,5 +33,5 @@ indent_style = space
 [**/test_*.py]
 max_line_length = off
 
-[Dockerfile]
+[Dockerfile*]
 indent_style = space

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,86 @@
+name: Bug report
+description: Something is not working
+title: "[BUG] Concise description of the issue"
+labels: ["bug", "unconfirmed"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Have a question? 👉 [Start a new discussion](https://github.com/paperless-ngx/paperless-ngx/discussions/new) or [ask in chat](https://matrix.to/#/#paperless:adnidor.de).
+
+        Before opening an issue, please double check:
+
+        - [The troubleshooting documentation](https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html).
+        - [The installation instructions](https://paperless-ngx.readthedocs.io/en/latest/setup.html#installation).
+        - [Existing issues and discussions](https://github.com/paperless-ngx/paperless-ngx/search?q=&type=issues).
+
+        If you encounter issues while installing or configuring Paperless-ngx, please post in the ["Support" section of the discussions](https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=support).
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is. If applicable, add screenshots to help explain your problem.
+      placeholder: |
+        Currently Paperless does not work when...
+
+        [Screenshot if applicable]
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: Steps to reproduce the behavior.
+      placeholder: |
+        1. Go to '...'
+        2. Click on '....'
+        3. See error
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Webserver logs
+      description: If available, post any logs from the web server related to your issue.
+      render: bash
+  - type: input
+    id: version
+    attributes:
+      label: Paperless-ngx version
+      placeholder: e.g. 1.6.0
+    validations:
+      required: true
+  - type: input
+    id: host-os
+    attributes:
+      label: Host OS
+      description: Host OS of the machine running paperless-ngx. Please add the architecture (uname -m) if applicable.
+      placeholder: e.g. Archlinux / Ubuntu 20.04 / Raspberry Pi `arm64`
+    validations:
+      required: true
+  - type: dropdown
+    id: install-method
+    attributes:
+      label: Installation method
+      options:
+        - Docker
+        - Bare metal
+        - Other (please describe above)
+    validations:
+      required: true
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Which browser you are using, if relevant.
+      placeholder: e.g. Chrome, Safari
+  - type: input
+    id: config-changes
+    attributes:
+      label: Configuration changes
+      description: Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.
+  - type: input
+    id: other
+    attributes:
+      label: Other
+      description: Any other relevant details.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,50 +0,0 @@
----
-name: Bug report
-about: Something is not working
-title: '[BUG] Concise description of the issue'
-labels: ''
-assignees: ''
----
-
-<!---
-=> Before opening an issue, please check the documentation and see if it helps you resolve your issue: https://paperless-ngx.readthedocs.io/en/latest/troubleshooting.html
-=> Please also make sure that you followed the installation instructions.
-=> Please search the issues and look for similar issues before opening a bug report.
-
-=> If you would like to submit a feature request please submit one under https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests
-
-=> If you encounter issues while installing of configuring Paperless-ngx, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably an issue with your system, and not an issue of paperless.
-
-=> Don't remove the [BUG] prefix from the title.
--->
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Webserver logs**
-
-```
-If available, post any logs from the web server related to your issue.
-```
-
-**Relevant information**
-
-- Host OS of the machine running paperless: [e.g. Archlinux / Ubuntu 20.04]
-- Browser [e.g. chrome, safari]
-- Version [e.g. 1.0.0]
-- Installation method: [docker / bare metal]
-- Any configuration changes you made in `docker-compose.yml`, `docker-compose.env` or `paperless.conf`.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 🤔 Questions and Help
+    url: https://github.com/paperless-ngx/paperless-ngx/discussions
+    about: This issue tracker is not for support questions. Please refer to our Discussions.
+  - name: 💬 Chat
+    url: https://matrix.to/#/#paperless:adnidor.de
+    about: Want to discuss Paperless-ngx with others? Check out our chat.
+  - name: 🚀 Feature Request
+    url: https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=feature-requests
+    about: Remember to search for existing feature requests and "up-vote" any you like

.github/ISSUE_TEMPLATE/other.md

@@ -1,19 +0,0 @@
----
-name: Other
-about: Anything that is not a feature request or bug.
-title: '[Other] Title of your issue'
-labels: ''
-assignees: ''
----
-
-<!--
-
-=> Discussions, Feedback and other suggestions belong in the "Discussion" section and not on the issue tracker.
-
-=> If you would like to submit a feature request please submit one under https://github.com/paperless-ngx/paperless-ngx/discussions/categories/feature-requests
-
-=> If you encounter issues while installing of configuring Paperless-ngx, please post that in the "Support" section of the discussions. Remember that Paperless successfully runs on a variety of different systems. If paperless does not start, it's probably is an issue with your system, and not an issue of paperless.
-
-=> Don't remove the [Other] prefix from the title.
-
--->

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,23 +1,15 @@
-<!-- 
+<!--
 Note: All PRs with code changes should be targeted to the `dev` branch, pure documentation changes can target `main`
 -->
 
 ## Proposed change
 
 <!--
-Please include a summary of the change and which issue is fixed (if any) and any relevant motivation / context. List any dependencies that are required for this change. If appropriate, please include an explanation of how your poposed change can be tested. Screenshots and / or videos can also be helpful if appropriate.
+Please include a summary of the change and which issue is fixed (if any) and any relevant motivation / context. List any dependencies that are required for this change. If appropriate, please include an explanation of how your proposed change can be tested. Screenshots and / or videos can also be helpful if appropriate.
 -->
 
 Fixes # (issue)
 
-<!--
-Please also tag the relevant team to help with review. You can tag any of the following:
-@paperless-ngx/backend (Python / django, database, etc.)
-@paperless-ngx/frontend (JavaScript/Typescript, HTML, CSS, etc.)
-@paperless-ngx/ci-cd (GitHub Actions, deployment)
-@paperless-ngx/test (General testing for larger PRs)
--->
-
 ## Type of change
 
 <!--

.github/dependabot.yml

@@ -6,11 +6,14 @@ updates:
   # Enable version updates for npm
   - package-ecosystem: "npm"
     target-branch: "dev"
-    # Look for `package.json` and `lock` files in the `root` directory
+    # Look for `package.json` and `lock` files in the `/src-ui` directory
     directory: "/src-ui"
     # Check the npm registry for updates every month
     schedule:
       interval: "monthly"
+    labels:
+      - "frontend"
+      - "dependencies"
     # Add reviewers
     reviewers:
       - "paperless-ngx/frontend"
@@ -26,9 +29,13 @@ updates:
     labels:
       - "backend"
       - "dependencies"
+    # Add reviewers
+    reviewers:
+      - "paperless-ngx/backend"
 
   # Enable updates for Github Actions
   - package-ecosystem: "github-actions"
+    target-branch: "dev"
     directory: "/"
     schedule:
       # Check for updates to GitHub Actions every month
@@ -38,4 +45,4 @@ updates:
       - "dependencies"
     # Add reviewers
     reviewers:
-      - "paperless-ngx/backend"
+      - "paperless-ngx/ci-cd"

.github/release-drafter.yml

@@ -0,0 +1,37 @@
+categories:
+  - title: 'Breaking Changes'
+    labels:
+      - 'breaking-change'
+  - title: 'Features'
+    labels:
+      - 'enhancement'
+  - title: 'Bug Fixes'
+    labels:
+      - 'bug'
+  - title: 'Documentation'
+    label: 'documentation'
+  - title: 'Maintenance'
+    labels:
+      - 'chore'
+      - 'deployment'
+      - 'translation'
+  - title: 'Dependencies'
+    collapse-after: 3
+    label: 'dependencies'
+include-labels:
+  - 'enhancement'
+  - 'bug'
+  - 'chore'
+  - 'deployment'
+  - 'translation'
+  - 'dependencies'
+replacers: # Changes "Feature: Update checker" to "Update checker"
+  - search: '/Feature:|Feat:|\[feature\]/gi'
+    replace: ''
+category-template: '### $TITLE'
+change-template: '- $TITLE [@$AUTHOR](https://github.com/$AUTHOR) ([#$NUMBER]($URL))'
+change-title-escapes: '\<*_&#@'
+template: |
+  ## paperless-ngx $RESOLVED_VERSION
+
+  $CHANGES

.github/scripts/cleanup-tags.py

@@ -0,0 +1,254 @@
+import logging
+import os
+from argparse import ArgumentParser
+from typing import Final
+from typing import List
+from urllib.parse import quote
+
+import requests
+from common import get_log_level
+
+logger = logging.getLogger("cleanup-tags")
+
+
+class GithubContainerRegistry:
+    def __init__(
+        self,
+        session: requests.Session,
+        token: str,
+        owner_or_org: str,
+    ):
+        self._session: requests.Session = session
+        self._token = token
+        self._owner_or_org = owner_or_org
+        # https://docs.github.com/en/rest/branches/branches
+        self._BRANCHES_ENDPOINT = "https://api.github.com/repos/{OWNER}/{REPO}/branches"
+        if self._owner_or_org == "paperless-ngx":
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-an-organization
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-package-version-for-an-organization
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+        else:
+            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-the-authenticated-user
+            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
+            # https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
+            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
+
+    def __enter__(self):
+        self._session.headers.update(
+            {
+                "Accept": "application/vnd.github.v3+json",
+                "Authorization": f"token {self._token}",
+            },
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if "Accept" in self._session.headers:
+            del self._session.headers["Accept"]
+        if "Authorization" in self._session.headers:
+            del self._session.headers["Authorization"]
+
+    def _read_all_pages(self, endpoint):
+        internal_data = []
+
+        while True:
+            resp = self._session.get(endpoint)
+            if resp.status_code == 200:
+                internal_data += resp.json()
+                if "next" in resp.links:
+                    endpoint = resp.links["next"]["url"]
+                else:
+                    logger.debug("Exiting pagination loop")
+                    break
+            else:
+                logger.warning(f"Request to {endpoint} return HTTP {resp.status_code}")
+                break
+
+        return internal_data
+
+    def get_branches(self, repo: str):
+        endpoint = self._BRANCHES_ENDPOINT.format(OWNER=self._owner_or_org, REPO=repo)
+        internal_data = self._read_all_pages(endpoint)
+        return internal_data
+
+    def filter_branches_by_name_pattern(self, branch_data, pattern: str):
+        matches = {}
+
+        for branch in branch_data:
+            if branch["name"].startswith(pattern):
+                matches[branch["name"]] = branch
+
+        return matches
+
+    def get_package_versions(
+        self,
+        package_name: str,
+        package_type: str = "container",
+    ) -> List:
+        package_name = quote(package_name, safe="")
+        endpoint = self._PACKAGES_VERSIONS_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_type,
+            PACKAGE_NAME=package_name,
+        )
+
+        internal_data = self._read_all_pages(endpoint)
+
+        return internal_data
+
+    def filter_packages_by_tag_pattern(self, package_data, pattern: str):
+        matches = {}
+
+        for package in package_data:
+            if "metadata" in package and "container" in package["metadata"]:
+                container_metadata = package["metadata"]["container"]
+                if "tags" in container_metadata:
+                    container_tags = container_metadata["tags"]
+                    for tag in container_tags:
+                        if tag.startswith(pattern):
+                            matches[tag] = package
+                            break
+
+        return matches
+
+    def filter_packages_untagged(self, package_data):
+        matches = {}
+
+        for package in package_data:
+            if "metadata" in package and "container" in package["metadata"]:
+                container_metadata = package["metadata"]["container"]
+                if "tags" in container_metadata:
+                    container_tags = container_metadata["tags"]
+                    if not len(container_tags):
+                        matches[package["name"]] = package
+
+        return matches
+
+    def delete_package_version(self, package_name, package_data):
+        package_name = quote(package_name, safe="")
+        endpoint = self._PACKAGE_VERSION_DELETE_ENDPOINT.format(
+            ORG=self._owner_or_org,
+            PACKAGE_TYPE=package_data["metadata"]["package_type"],
+            PACKAGE_NAME=package_name,
+            PACKAGE_VERSION_ID=package_data["id"],
+        )
+        resp = self._session.delete(endpoint)
+        if resp.status_code != 204:
+            logger.warning(
+                f"Request to delete {endpoint} returned HTTP {resp.status_code}",
+            )
+
+
+def _main():
+    parser = ArgumentParser(
+        description="Using the GitHub API locate and optionally delete container"
+        " tags which no longer have an associated feature branch",
+    )
+
+    parser.add_argument(
+        "--delete",
+        action="store_true",
+        default=False,
+        help="If provided, actually delete the container tags",
+    )
+
+    # TODO There's a lot of untagged images, do those need to stay for anything?
+    parser.add_argument(
+        "--untagged",
+        action="store_true",
+        default=False,
+        help="If provided, delete untagged containers as well",
+    )
+
+    parser.add_argument(
+        "--loglevel",
+        default="info",
+        help="Configures the logging level",
+    )
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=get_log_level(args),
+        datefmt="%Y-%m-%d %H:%M:%S",
+        format="%(asctime)s %(levelname)-8s %(message)s",
+    )
+
+    repo_owner: Final[str] = os.environ["GITHUB_REPOSITORY_OWNER"]
+    repo: Final[str] = os.environ["GITHUB_REPOSITORY"]
+    gh_token: Final[str] = os.environ["GITHUB_TOKEN"]
+
+    with requests.session() as sess:
+        with GithubContainerRegistry(sess, gh_token, repo_owner) as gh_api:
+            all_branches = gh_api.get_branches("paperless-ngx")
+            logger.info(f"Located {len(all_branches)} branches of {repo_owner}/{repo} ")
+
+            feature_branches = gh_api.filter_branches_by_name_pattern(
+                all_branches,
+                "feature-",
+            )
+            logger.info(f"Located {len(feature_branches)} feature branches")
+
+            for package_name in ["paperless-ngx", "paperless-ngx/builder/cache/app"]:
+
+                all_package_versions = gh_api.get_package_versions(package_name)
+                logger.info(
+                    f"Located {len(all_package_versions)} versions of package {package_name}",
+                )
+
+                packages_tagged_feature = gh_api.filter_packages_by_tag_pattern(
+                    all_package_versions,
+                    "feature-",
+                )
+                logger.info(
+                    f'Located {len(packages_tagged_feature)} versions of package {package_name} tagged "feature-"',
+                )
+
+                untagged_packages = gh_api.filter_packages_untagged(
+                    all_package_versions,
+                )
+                logger.info(
+                    f"Located {len(untagged_packages)} untagged versions of package {package_name}",
+                )
+
+                to_delete = list(
+                    set(packages_tagged_feature.keys()) - set(feature_branches.keys()),
+                )
+                logger.info(
+                    f"Located {len(to_delete)} versions of package {package_name} to delete",
+                )
+
+                for tag_to_delete in to_delete:
+                    package_version_info = packages_tagged_feature[tag_to_delete]
+
+                    if args.delete:
+                        logger.info(
+                            f"Deleting {tag_to_delete} (id {package_version_info['id']})",
+                        )
+                        gh_api.delete_package_version(
+                            package_name,
+                            package_version_info,
+                        )
+
+                    else:
+                        logger.info(
+                            f"Would delete {tag_to_delete} (id {package_version_info['id']})",
+                        )
+
+                if args.untagged:
+                    logger.info(f"Deleting untagged packages of {package_name}")
+                    for to_delete_name in untagged_packages:
+                        to_delete_version = untagged_packages[to_delete_name]
+                        logger.info(f"Deleting id {to_delete_version['id']}")
+                        if args.delete:
+                            gh_api.delete_package_version(
+                                package_name,
+                                to_delete_version,
+                            )
+                else:
+                    logger.info("Leaving untagged images untouched")
+
+
+if __name__ == "__main__":
+    _main()

.github/scripts/common.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+import logging
+from argparse import ArgumentError
+
+
+def get_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+) -> str:
+    """
+    Returns a string representing the normal image for a given package
+    """
+    return f"ghcr.io/{repo_name.lower()}/builder/{pkg_name}:{pkg_version}"
+
+
+def get_cache_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+    branch_name: str,
+) -> str:
+    """
+    Returns a string representing the expected image cache tag for a given package
+
+    Registry type caching is utilized for the builder images, to allow fast
+    rebuilds, generally almost instant for the same version
+    """
+    return f"ghcr.io/{repo_name.lower()}/builder/cache/{pkg_name}:{pkg_version}"
+
+
+def get_log_level(args) -> int:
+    levels = {
+        "critical": logging.CRITICAL,
+        "error": logging.ERROR,
+        "warn": logging.WARNING,
+        "warning": logging.WARNING,
+        "info": logging.INFO,
+        "debug": logging.DEBUG,
+    }
+    level = levels.get(args.loglevel.lower())
+    if level is None:
+        level = logging.INFO
+    return level

.github/scripts/get-build-json.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+This is a helper script for the mutli-stage Docker image builder.
+It provides a single point of configuration for package version control.
+The output JSON object is used by the CI workflow to determine what versions
+to build and pull into the final Docker image.
+
+Python package information is obtained from the Pipfile.lock.  As this is
+kept updated by dependabot, it usually will need no further configuration.
+The sole exception currently is pikepdf, which has a dependency on qpdf,
+and is configured here to use the latest version of qpdf built by the workflow.
+
+Other package version information is configured directly below, generally by
+setting the version and Git information, if any.
+
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Final
+
+from common import get_cache_image_tag
+from common import get_image_tag
+
+
+def _main():
+    parser = argparse.ArgumentParser(
+        description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
+    )
+    parser.add_argument(
+        "package",
+        help="The name of the package to generate JSON for",
+    )
+
+    PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock")
+    BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json")
+
+    # Read the main config file
+    build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text())
+
+    # Read Pipfile.lock file
+    pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text())
+
+    args: Final = parser.parse_args()
+
+    # Read from environment variables set by GitHub Actions
+    repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"]
+    branch_name: Final[str] = os.environ["GITHUB_REF_NAME"]
+
+    # Default output values
+    version = None
+    extra_config = {}
+
+    if args.package in pipfile_data["default"]:
+        # Read the version from Pipfile.lock
+        pkg_data = pipfile_data["default"][args.package]
+        pkg_version = pkg_data["version"].split("==")[-1]
+        version = pkg_version
+
+        # Any extra/special values needed
+        if args.package == "pikepdf":
+            extra_config["qpdf_version"] = build_json["qpdf"]["version"]
+
+    elif args.package in build_json:
+        version = build_json[args.package]["version"]
+
+    else:
+        raise NotImplementedError(args.package)
+
+    # The JSON object we'll output
+    output = {
+        "name": args.package,
+        "version": version,
+        "image_tag": get_image_tag(repo_name, args.package, version),
+        "cache_tag": get_cache_image_tag(
+            repo_name,
+            args.package,
+            version,
+            branch_name,
+        ),
+    }
+
+    # Add anything special a package may need
+    output.update(extra_config)
+
+    # Output the JSON info to stdout
+    print(json.dumps(output))
+
+
+if __name__ == "__main__":
+    _main()

.github/stale.yml

@@ -2,11 +2,8 @@
 daysUntilStale: 30
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
-# Issues with these labels will never be considered stale
-exemptLabels:
-  - pinned
-  - security
-  - fixpending
+onlyLabels:
+  - unconfirmed
 # Label to use when marking an issue as stale
 staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable

.github/workflows/ci.yml

@@ -3,8 +3,10 @@ name: ci
 on:
   push:
     tags:
-      - ngx-*
-      - beta-*
+      # https://semver.org/#spec-item-2
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # https://semver.org/#spec-item-9
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+'
     branches-ignore:
       - 'translations**'
   pull_request:
@@ -24,7 +26,7 @@ jobs:
         run: pipx install pipenv
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
           cache: "pipenv"
@@ -45,179 +47,158 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  code-checks-backend:
-    name: "Backend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Install checkers
-        run: |
-          pipx install reorder-python-imports
-          pipx install yesqa
-          pipx install add-trailing-comma
-          pipx install flake8
-      -
-        name: Run reorder-python-imports
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-      -
-        name: Run yesqa
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-      -
-        name: Run add-trailing-comma
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
-      # black is placed after add-trailing-comma because it may format differently
-      # if a trailing comma is added
-      -
-        name: Run black
-        uses: psf/black@stable
-        with:
-          options: "--check --diff"
-          version: "22.3.0"
-      -
-        name: Run flake8 checks
-        run: |
-          cd src/
-          flake8 --max-line-length=88 --ignore=E203,W503
+  ci-backend:
+    uses: ./.github/workflows/reusable-ci-backend.yml
 
-  code-checks-frontend:
-    name: "Frontend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-      -
-        name: Install prettier
-        run: |
-          npm install prettier
-      -
-        name: Run prettier
-        run:
-          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  ci-frontend:
+    uses: ./.github/workflows/reusable-ci-frontend.yml
 
-  tests-backend:
-    needs: [code-checks-backend]
-    name: "Backend Tests (${{ matrix.python-version }})"
+  prepare-docker-build:
+    name: Prepare Docker Pipeline Data
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
     runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ['3.8', '3.9']
-      fail-fast: false
+    # If the push triggered the installer library workflow, wait for it to
+    # complete here.  This ensures the required versions for the final
+    # image have been built, while not waiting at all if the versions haven't changed
+    concurrency:
+      group: build-installer-library
+      cancel-in-progress: false
+    needs:
+      - documentation
+      - ci-backend
+      - ci-frontend
     steps:
+      -
+        name: Set ghcr repository name
+        id: set-ghcr-repository
+        run: |
+          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
+          echo ::set-output name=repository::${ghcr_name}
       -
         name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      -
-        name: Install pipenv
-        run: pipx install pipenv
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
-          python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
+          python-version: "3.9"
       -
-        name: Install system dependencies
+        name: Setup qpdf image
+        id: qpdf-setup
         run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng
-      -
-        name: Install Python dependencies
-        run: |
-          pipenv sync --dev
-      -
-        name: Tests
-        run: |
-          cd src/
-          pipenv run pytest
-      -
-        name: Get changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v18.1
-        with:
-          files: |
-            src/**
-      -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
 
-  tests-frontend:
-    needs: [code-checks-frontend]
-    name: "Frontend Tests"
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-      - run: cd src-ui && npm ci
-      - run: cd src-ui && npm run test
-      - run: cd src-ui && npm run e2e:ci
+          echo ${build_json}
+
+          echo ::set-output name=qpdf-json::${build_json}
+      -
+        name: Setup psycopg2 image
+        id: psycopg2-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo ::set-output name=psycopg2-json::${build_json}
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=pikepdf-json::${build_json}
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo ::set-output name=jbig2enc-json::${build_json}
+
+    outputs:
+
+      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
 
   # build and push image to docker hub.
   build-docker-image:
-    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
     runs-on: ubuntu-20.04
-    needs: [tests-backend, tests-frontend]
+    concurrency:
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
+      cancel-in-progress: true
+    needs:
+      - prepare-docker-build
     steps:
+      -
+        name: Check pushing to Docker Hub
+        id: docker-hub
+        # Only push to Dockerhub from the main repo AND the ref is either:
+        #  main
+        #  dev
+        #  beta
+        #  a tag
+        # Otherwise forks would require a Docker Hub account and secrets setup
+        run: |
+          if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
+            echo "Enabling DockerHub image push"
+            echo ::set-output name=enable::"true"
+          else
+            echo "Not pushing to DockerHub"
+            echo ::set-output name=enable::"false"
+          fi
       -
         name: Gather Docker metadata
         id: docker-meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v4
         with:
-          images: ghcr.io/${{ github.repository }}
+          images: |
+            ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}
+            name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
           tags: |
-            type=match,pattern=ngx-(\d.\d.\d),group=1
-            type=semver,pattern=ngx-{{version}}
-            type=semver,pattern=ngx-{{major}}.{{minor}}
-            type=ref
+            # Tag branches with branch name
+            type=ref,event=branch
+            # Process semver tags
+            # For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
       -
         name: Checkout
         uses: actions/checkout@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       -
         name: Login to Github Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
+        # Don't attempt to login is not pushing to Docker Hub
+        if: steps.docker-hub.outputs.enable == 'true'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./Dockerfile
@@ -225,8 +206,19 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          build-args: |
+            JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+            QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+            PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+            PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+          # Get cache layers from this branch, then dev, then main
+          # This allows new branches to get at least some cache benefits, generally from dev
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev
+            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main
+          cache-to: |
+            type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
       -
         name: Inspect image
         run: |
@@ -244,7 +236,8 @@ jobs:
           path: src/documents/static/frontend/
 
   build-release:
-    needs: [build-docker-image, documentation]
+    needs:
+      - build-docker-image
     runs-on: ubuntu-20.04
     steps:
       -
@@ -252,7 +245,7 @@ jobs:
         uses: actions/checkout@v3
       -
         name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
       -
@@ -312,8 +305,9 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
-    needs: build-release
-    if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
+    needs:
+      - build-release
+    if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc'))
     steps:
       -
         name: Download release artifact
@@ -325,27 +319,24 @@ jobs:
         name: Get version
         id: get_version
         run: |
-          if [[ $GITHUB_REF == refs/tags/ngx-* ]]; then
-            echo ::set-output name=version::${GITHUB_REF#refs/tags/ngx-}
-            echo ::set-output name=prerelease::false
-            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://paperless-ngx.readthedocs.io/en/latest/changelog.html"
-          elif [[ $GITHUB_REF == refs/tags/beta-* ]]; then
-            echo ::set-output name=version::${GITHUB_REF#refs/tags/beta-}
+          echo ::set-output name=version::${{ github.ref_name }}
+          if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then
             echo ::set-output name=prerelease::true
-            echo ::set-output name=body::"For a complete list of changes, see the changelog at https://github.com/paperless-ngx/paperless-ngx/blob/beta/docs/changelog.rst"
+          else
+            echo ::set-output name=prerelease::false
           fi
       -
-        name: Create release
-        id: create_release
-        uses: actions/create-release@v1
+        name: Create Release and Changelog
+        id: create-release
+        uses: release-drafter/release-drafter@v5
+        with:
+          name: Paperless-ngx ${{ steps.get_version.outputs.version }}
+          tag: ${{ steps.get_version.outputs.version }}
+          version: ${{ steps.get_version.outputs.version }}
+          prerelease: ${{ steps.get_version.outputs.prerelease }}
+          publish: true # ensures release is not marked as draft
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ngx-${{ steps.get_version.outputs.version }}
-          release_name: Paperless-ngx ${{ steps.get_version.outputs.version }}
-          draft: false
-          prerelease: ${{ steps.get_version.outputs.prerelease }}
-          body: ${{ steps.get_version.outputs.body }}
       -
         name: Upload release archive
         id: upload-release-asset
@@ -353,7 +344,25 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+          upload_url: ${{ steps.create-release.outputs.upload_url }}
           asset_path: ./paperless-ngx.tar.xz
           asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz
           asset_content_type: application/x-xz
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+      -
+        name: Append Changelog to docs
+        id: append-Changelog
+        working-directory: docs
+        run: |
+          echo -e "# Changelog\n\n${{ steps.create-release.outputs.body }}\n" > changelog-new.md
+          CURRENT_CHANGELOG=`tail --lines +2 changelog.md`
+          echo -e "$CURRENT_CHANGELOG" >> changelog-new.md
+          mv changelog-new.md changelog.md
+          git config --global user.name "github-actions"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git commit -am "Changelog ${{ steps.get_version.outputs.version }} - GHA"
+          git push origin HEAD:main

.github/workflows/cleanup-tags.yml

@@ -0,0 +1,48 @@
+name: Cleanup Image Tags
+
+on:
+  schedule:
+    - cron: '0 0 * * SAT'
+  delete:
+  pull_request:
+    types:
+      - closed
+  push:
+    paths:
+      - ".github/workflows/cleanup-tags.yml"
+      - ".github/scripts/cleanup-tags.py"
+      - ".github/scripts/common.py"
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  cleanup:
+    name: Cleanup Image Tags
+    runs-on: ubuntu-20.04
+    permissions:
+      packages: write
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.9"
+      -
+        name: Install requests
+        run: |
+          python -m pip install requests
+      -
+        name: Cleanup feature tags
+        run: |
+          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --loglevel info --delete

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,4 +51,4 @@ jobs:
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/installer-library.yml

@@ -0,0 +1,147 @@
+# This workflow will run to update the installer library of
+# Docker images.  These are the images which provide updated wheels
+# .deb installation packages or maybe just some compiled library
+
+name: Build Image Library
+
+on:
+  push:
+    # Must match one of these branches AND one of the paths
+    # to be triggered
+    branches:
+      - "main"
+      - "dev"
+      - "library-*"
+      - "feature-*"
+    paths:
+      # Trigger the workflow if a Dockerfile changed
+      - "docker-builders/**"
+      # Trigger if a package was updated
+      - ".build-config.json"
+      - "Pipfile.lock"
+      # Also trigger on workflow changes related to the library
+      - ".github/workflows/installer-library.yml"
+      - ".github/workflows/reusable-workflow-builder.yml"
+      - ".github/scripts/**"
+
+# Set a workflow level concurrency group so primary workflow
+# can wait for this to complete if needed
+# DO NOT CHANGE without updating main workflow group
+concurrency:
+  group: build-installer-library
+  cancel-in-progress: false
+
+jobs:
+  prepare-docker-build:
+    name: Prepare Docker Image Version Data
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Set ghcr repository name
+        id: set-ghcr-repository
+        run: |
+          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
+          echo ::set-output name=repository::${ghcr_name}
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      -
+        name: Setup qpdf image
+        id: qpdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=qpdf-json::${build_json}
+      -
+        name: Setup psycopg2 image
+        id: psycopg2-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo ::set-output name=psycopg2-json::${build_json}
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=pikepdf-json::${build_json}
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo ::set-output name=jbig2enc-json::${build_json}
+
+    outputs:
+
+      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+
+  build-qpdf-debs:
+    name: qpdf
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+      build-args: |
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+  build-jbig2enc:
+    name: jbig2enc
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.jbig2enc
+      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+      build-args: |
+        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+  build-psycopg2-wheel:
+    name: psycopg2
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.psycopg2
+      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+      build-args: |
+        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+  build-pikepdf-wheel:
+    name: pikepdf
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.pikepdf
+      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+      build-args: |
+        REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }}
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}

.github/workflows/reusable-ci-backend.yml

@@ -0,0 +1,108 @@
+name: Backend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-backend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Install checkers
+        run: |
+          pipx install reorder-python-imports
+          pipx install yesqa
+          pipx install add-trailing-comma
+          pipx install flake8
+      -
+        name: Run reorder-python-imports
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
+      -
+        name: Run yesqa
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
+      -
+        name: Run add-trailing-comma
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
+      # black is placed after add-trailing-comma because it may format differently
+      # if a trailing comma is added
+      -
+        name: Run black
+        uses: psf/black@stable
+        with:
+          options: "--check --diff"
+          version: "22.3.0"
+      -
+        name: Run flake8 checks
+        run: |
+          cd src/
+          flake8 --max-line-length=88 --ignore=E203,W503
+
+  tests-backend:
+    name: "Tests (${{ matrix.python-version }})"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-backend
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+      fail-fast: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      -
+        name: Install pipenv
+        run: pipx install pipenv
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Tests
+        run: |
+          cd src/
+          pipenv run pytest
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v23.1
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          pipenv run coveralls --service=github

.github/workflows/reusable-ci-frontend.yml

@@ -0,0 +1,42 @@
+name: Frontend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-frontend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      -
+        name: Install prettier
+        run: |
+          npm install prettier
+      -
+        name: Run prettier
+        run:
+          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  tests-frontend:
+    name: "Tests"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-frontend
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci

.github/workflows/reusable-workflow-builder.yml

@@ -0,0 +1,53 @@
+name: Reusable Image Builder
+
+on:
+  workflow_call:
+    inputs:
+      dockerfile:
+        required: true
+        type: string
+      build-json:
+        required: true
+        type: string
+      build-args:
+        required: false
+        default: ""
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
+  cancel-in-progress: false
+
+jobs:
+  build-image:
+    name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Build ${{ fromJSON(inputs.build-json).name }}
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          tags: ${{ fromJSON(inputs.build-json).image_tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          build-args: ${{ inputs.build-args }}
+          push: true
+          cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}
+          cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }}

.gitignore

@@ -63,11 +63,14 @@ target/
 
 # VS Code
 .vscode
+/src-ui/.vscode
+/docs/.vscode
 
 # Other stuff that doesn't belong
 .virtualenv
 virtualenv
 /venv
+.venv/
 /docker-compose.env
 /docker-compose.yml
 
@@ -84,8 +87,9 @@ scripts/nuke
 /paperless.conf
 /consume/
 /export/
-/src-ui/.vscode
 
 # this is where the compiled frontend is moved to.
 /src/documents/static/frontend/
-/docs/.vscode/settings.json
+
+# mac os
+.DS_Store

.pre-commit-config.yaml

@@ -5,7 +5,7 @@
 repos:
   # General hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.3.0
     hooks:
       - id: check-docstring-first
       - id: check-json
@@ -27,7 +27,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.6.1"
+    rev: "v2.7.1"
     hooks:
       - id: prettier
         types_or:
@@ -37,7 +37,7 @@ repos:
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
   - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.0.1
+    rev: v3.8.1
     hooks:
       - id: reorder-python-imports
         exclude: "(migrations)"
@@ -47,7 +47,7 @@ repos:
       - id: yesqa
         exclude: "(migrations)"
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: "v2.2.1"
+    rev: "v2.2.3"
     hooks:
       - id: add-trailing-comma
         exclude: "(migrations)"
@@ -59,14 +59,28 @@ repos:
         args:
           - "--config=./src/setup.cfg"
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.6.0
     hooks:
       - id: black
-  # Dockerfile hooks
-  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
-    rev: "v0.1.0"
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.37.1
     hooks:
-      - id: dockerfilelint
+      - id: pyupgrade
+        exclude: "(migrations)"
+        args:
+          - "--py38-plus"
+  # Dockerfile hooks
+  - repo: https://github.com/AleksaC/hadolint-py
+    rev: v2.10.0
+    hooks:
+      - id: hadolint
+        args:
+          - --ignore
+          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
+          - --ignore
+          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
+          - --ignore
+          - DL3003      # https://github.com/hadolint/hadolint/wiki/DL3003  (seems excessive to use WORKDIR so much)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1

CODEOWNERS

@@ -0,0 +1,10 @@
+/.github/workflows/ @paperless-ngx/ci-cd
+/docker/ @paperless-ngx/ci-cd
+/scripts/ @paperless-ngx/ci-cd
+
+/src-ui/ @paperless-ngx/frontend
+
+/src/ @paperless-ngx/backend
+Pipfile* @paperless-ngx/backend
+*.py @paperless-ngx/backend
+requirements.txt @paperless-ngx/backend

Dockerfile

@@ -1,12 +1,36 @@
-FROM node:16 AS compile-frontend
+# syntax=docker/dockerfile:1.4
 
-COPY . /src
+# Pull the installer images from the library
+# These are all built previously
+# They provide either a .deb or .whl
+
+ARG JBIG2ENC_VERSION
+ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION
+
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
+
+FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
+
+# This stage compiles the frontend
+# This stage runs once for the native platform, as the outputs are not
+# dependent on target arch
+# Inputs: None
+
+COPY ./src-ui /src/src-ui
 
 WORKDIR /src/src-ui
-RUN npm update npm -g && npm ci --no-optional
-RUN ./node_modules/.bin/ng build --configuration production
+RUN set -eux \
+  && npm update npm -g \
+  && npm ci --omit=optional
+RUN set -eux \
+  && ./node_modules/.bin/ng build --configuration production
 
-FROM ghcr.io/paperless-ngx/builder/ngx-base:1.0 as main-app
+FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
 LABEL org.opencontainers.image.documentation="https://paperless-ngx.readthedocs.io/en/latest/"
@@ -14,45 +38,179 @@ LABEL org.opencontainers.image.source="https://github.com/paperless-ngx/paperles
 LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-ngx"
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
-WORKDIR /usr/src/paperless/src/
+ARG DEBIAN_FRONTEND=noninteractive
 
-COPY requirements.txt ../
+#
+# Begin installation and configuration
+# Order the steps below from least often changed to most
+#
 
-# Python dependencies
-RUN apt-get update \
-  # python-Levenshtein still needs to be compiled here
-  && apt-get -y --no-install-recommends install \
-    build-essential \
-    && python3 -m pip install --upgrade --no-cache-dir pip wheel \
-  && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
-  && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
-  && apt-get -y purge build-essential \
-  && apt-get -y autoremove --purge \
-  && rm -rf /var/lib/apt/lists/*
+# copy jbig2enc
+# Basically will never change again
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
+
+# Packages need for running
+ARG RUNTIME_PACKAGES="\
+  curl \
+  file \
+  # fonts for text file thumbnail generation
+  fonts-liberation \
+  gettext \
+  ghostscript \
+  gnupg \
+  gosu \
+  icc-profiles-free \
+  imagemagick \
+  media-types \
+  liblept5 \
+  libpq5 \
+  libxml2 \
+  liblcms2-2 \
+  libtiff5 \
+  libxslt1.1 \
+  libfreetype6 \
+  libwebp6 \
+  libopenjp2-7 \
+  libimagequant0 \
+  libraqm0 \
+  libgnutls30 \
+  libjpeg62-turbo \
+  python3 \
+  python3-pip \
+  python3-setuptools \
+  postgresql-client \
+  # For Numpy
+  libatlas3-base \
+  # OCRmyPDF dependencies
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-deu \
+  tesseract-ocr-fra \
+  tesseract-ocr-ita \
+  tesseract-ocr-spa \
+  tzdata \
+  unpaper \
+  # Mime type detection
+  zlib1g \
+  # Barcode splitter
+  libzbar0 \
+  poppler-utils"
+
+# Install basic runtime packages.
+# These change very infrequently
+RUN set -eux \
+  echo "Installing system packages" \
+    && apt-get update \
+    && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
+    && rm -rf /var/lib/apt/lists/* \
+  && echo "Installing supervisor" \
+    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.4
+
+# Copy gunicorn config
+# Changes very infrequently
+WORKDIR /usr/src/paperless/
+
+COPY gunicorn.conf.py .
 
 # setup docker-specific things
-COPY docker/ ./docker/
+# Use mounts to avoid copying installer files into the image
+# These change sometimes, but rarely
+ARG DOCKER_SRC=/usr/src/paperless/src/docker/
+WORKDIR ${DOCKER_SRC}
 
-RUN cd docker \
-    && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
-  && mkdir /var/log/supervisord /var/run/supervisord \
-  && cp supervisord.conf /etc/supervisord.conf \
-  && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
-  && chmod 755 /sbin/docker-entrypoint.sh \
-  && cp docker-prepare.sh /sbin/docker-prepare.sh \
-  && chmod 755 /sbin/docker-prepare.sh \
-  && chmod +x install_management_commands.sh \
-  && ./install_management_commands.sh \
-  && cd .. \
-  && rm -rf docker/
+COPY [ \
+	"docker/imagemagick-policy.xml", \
+	"docker/supervisord.conf", \
+	"docker/docker-entrypoint.sh", \
+	"docker/docker-prepare.sh", \
+	"docker/paperless_cmd.sh", \
+	"docker/wait-for-redis.py", \
+	"docker/management_script.sh", \
+	"docker/install_management_commands.sh", \
+	"${DOCKER_SRC}" \
+]
 
-COPY gunicorn.conf.py ../
+RUN set -eux \
+  && echo "Configuring ImageMagick" \
+    && mv imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+  && echo "Configuring supervisord" \
+    && mkdir /var/log/supervisord /var/run/supervisord \
+    && mv supervisord.conf /etc/supervisord.conf \
+  && echo "Setting up Docker scripts" \
+    && mv docker-entrypoint.sh /sbin/docker-entrypoint.sh \
+    && chmod 755 /sbin/docker-entrypoint.sh \
+    && mv docker-prepare.sh /sbin/docker-prepare.sh \
+    && chmod 755 /sbin/docker-prepare.sh \
+    && mv wait-for-redis.py /sbin/wait-for-redis.py \
+    && chmod 755 /sbin/wait-for-redis.py \
+    && mv paperless_cmd.sh /usr/local/bin/paperless_cmd.sh \
+    && chmod 755 /usr/local/bin/paperless_cmd.sh \
+  && echo "Installing managment commands" \
+    && chmod +x install_management_commands.sh \
+    && ./install_management_commands.sh
 
-# copy app
-COPY --from=compile-frontend /src/src/ ./
+# Install the built packages from the installer library images
+# Use mounts to avoid copying installer files into the image
+# These change sometimes
+RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
+    --mount=type=bind,from=psycopg2-builder,target=/psycopg2 \
+    --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
+  set -eux \
+  && echo "Installing qpdf" \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \
+    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
+  && echo "Installing pikepdf and dependencies" \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \
+    && python -m pip list \
+  && echo "Installing psycopg2" \
+    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
+    && python -m pip list
+
+# Python dependencies
+# Change pretty frequently
+COPY requirements.txt ../
+
+# Packages needed only for building a few quick Python
+# dependencies
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  python3-dev"
+
+RUN set -eux \
+  && echo "Installing build system packages" \
+    && apt-get update \
+    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+    && python3 -m pip install --no-cache-dir --upgrade wheel \
+  && echo "Installing Python requirements" \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && apt-get clean --yes \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* \
+    && rm -rf /var/tmp/* \
+    && rm -rf /var/cache/apt/archives/* \
+    && truncate -s 0 /var/log/*log
+
+WORKDIR /usr/src/paperless/src/
+
+# copy backend
+COPY ./src ./
+
+# copy frontend
+COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
 
 # add users, setup scripts
-RUN addgroup --gid 1000 paperless \
+RUN set -eux \
+  && addgroup --gid 1000 paperless \
   && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
   && chown -R paperless:paperless ../ \
   && gosu paperless python3 manage.py collectstatic --clear --no-input \
@@ -67,4 +225,4 @@ ENTRYPOINT ["/sbin/docker-entrypoint.sh"]
 
 EXPOSE 8000
 
-CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf"]
+CMD ["/usr/local/bin/paperless_cmd.sh"]

Pipfile

@@ -13,8 +13,8 @@ dateparser = "~=1.1"
 django = "~=4.0"
 django-cors-headers = "*"
 django-extensions = "*"
-django-filter = "~=21.1"
-django-q = "~=1.3"
+django-filter = "~=22.1"
+django-q = {editable = true, ref = "paperless-main", git = "https://github.com/paperless-ngx/django-q.git"}
 djangorestframework = "~=3.13"
 filelock = "*"
 fuzzywuzzy = {extras = ["speedup"], version = "*"}
@@ -22,20 +22,17 @@ gunicorn = "*"
 imap-tools = "*"
 langdetect = "*"
 pathvalidate = "*"
-pillow = "~=9.0"
-# Any version update to pikepdf requires a base image update
+pillow = "~=9.2"
 pikepdf = "~=5.1"
 python-gnupg = "*"
 python-dotenv = "*"
 python-dateutil = "*"
 python-magic = "*"
-# Any version update to psycopg2 requires a base image update
 psycopg2 = "*"
 redis = "*"
-# Pinned because aarch64 wheels and updates cause warnings when loading the classifier model.
-scikit-learn="==1.0.2"
-whitenoise = "~=6.0.0"
-watchdog = "~=2.1.0"
+scikit-learn="~=1.1"
+whitenoise = "~=6.2.0"
+watchdog = "~=2.1.9"
 whoosh="~=2.7.4"
 inotifyrecursive = "~=0.3"
 ocrmypdf = "~=13.4"
@@ -51,6 +48,8 @@ concurrent-log-handler = "*"
 "backports.zoneinfo" = {version = "*", markers = "python_version < '3.9'"}
 "importlib-resources" = {version = "*", markers = "python_version < '3.9'"}
 zipp = {version = "*", markers = "python_version < '3.9'"}
+pyzbar = "*"
+pdf2image = "*"
 
 [dev-packages]
 coveralls = "*"
@@ -62,8 +61,10 @@ pytest-django = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-sphinx = "~=4.4.0"
+sphinx = "~=5.0.2"
 sphinx_rtd_theme = "*"
 tox = "*"
 black = "*"
 pre-commit = "*"
+sphinx-autobuild = "*"
+myst-parser = "*"

README.md

@@ -102,7 +102,7 @@ For bugs please [open an issue](https://github.com/paperless-ngx/paperless-ngx/i
 
 Paperless has been around a while now, and people are starting to build stuff on top of it. If you're one of those people, we can add your project to this list:
 
-- [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ngx.
+- [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS app for Paperless-ngx. Also works with the original Paperless and Paperless-ng.
 - [Paperless Share](https://github.com/qcasey/paperless_share). Share any files from your Android application with paperless. Very simple, but works with all of the mobile scanning apps out there that allow you to share scanned documents.
 - [Scan to Paperless](https://github.com/sbrunner/scan-to-paperless): Scan and prepare (crop, deskew, OCR, ...) your documents for Paperless.
 

build-docker-image.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+# Helper script for building the Docker image locally.
+# Parses and provides the nessecary versions of other images to Docker
+# before passing in the rest of script args.
+
+# First Argument: The Dockerfile to build
+# Other Arguments: Additional arguments to docker build
+
+# Example Usage:
+#	./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature
+
+set -eux
+
+if ! command -v jq;  then
+	echo "jq required"
+	exit 1
+elif [ ! -f "$1" ]; then
+	echo "$1 is not a file, please provide the Dockerfile"
+	exit 1
+fi
+
+# Parse what we can from Pipfile.lock
+pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
+psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+# Read this from the other config file
+qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
+jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
+# Get the branch name (used for caching)
+branch_name=$(git rev-parse --abbrev-ref HEAD)
+
+# https://docs.docker.com/develop/develop-images/build_enhancements/
+# Required to use cache-from
+export DOCKER_BUILDKIT=1
+
+docker build --file "$1" \
+	--progress=plain \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
+	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
+	--build-arg QPDF_VERSION="${qpdf_version}" \
+	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" .

docker-builders/Dockerfile.frontend

@@ -0,0 +1,14 @@
+# This Dockerfile compiles the frontend
+# Inputs: None
+
+FROM node:16-bullseye-slim AS compile-frontend
+
+COPY ./src /src/src
+COPY ./src-ui /src/src-ui
+
+WORKDIR /src/src-ui
+RUN set -eux \
+  && npm update npm -g \
+  && npm ci --omit=optional
+RUN set -eux \
+  && ./node_modules/.bin/ng build --configuration production

docker-builders/Dockerfile.jbig2enc

@@ -0,0 +1,39 @@
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+#    - JBIG2ENC_VERSION - the Git tag to checkout and build
+
+FROM debian:bullseye-slim as main
+
+LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  automake \
+  libtool \
+  libleptonica-dev \
+  zlib1g-dev \
+  git \
+  ca-certificates"
+
+WORKDIR /usr/src/jbig2enc
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+RUN apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG JBIG2ENC_VERSION
+
+RUN set -eux \
+  && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
+RUN set -eux \
+  && ./autogen.sh
+RUN set -eux \
+  && ./configure && make

docker-builders/Dockerfile.pikepdf

@@ -0,0 +1,88 @@
+# This Dockerfile builds the pikepdf wheel
+# Inputs:
+#    - REPO - Docker repository to pull qpdf from
+#    - QPDF_VERSION - The image qpdf version to copy .deb files from
+#    - PIKEPDF_VERSION - Version of pikepdf to build wheel for
+
+# Default to pulling from the main repo registry when manually building
+ARG REPO="paperless-ngx/paperless-ngx"
+
+ARG QPDF_VERSION
+FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+
+# This does nothing, except provide a name for a copy below
+
+FROM python:3.9-slim-bullseye as main
+
+LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  python3-dev \
+  python3-pip \
+  # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers
+  libgnutls28-dev \
+  # lxml requrements - https://lxml.de/installation.html
+  libxml2-dev \
+  libxslt1-dev \
+  # Pillow requirements - https://pillow.readthedocs.io/en/stable/installation.html#external-libraries
+  # JPEG functionality
+  libjpeg62-turbo-dev \
+  # conpressed PNG
+  zlib1g-dev \
+  # compressed TIFF
+  libtiff-dev \
+  # type related services
+  libfreetype-dev \
+  # color management
+  liblcms2-dev \
+  # WebP format
+  libwebp-dev \
+  # JPEG 2000
+  libopenjp2-7-dev \
+  # improved color quantization
+  libimagequant-dev \
+  # complex text layout support
+  libraqm-dev"
+
+WORKDIR /usr/src
+
+COPY --from=qpdf-builder /usr/src/qpdf/*.deb ./
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && dpkg --install libqpdf28_*.deb \
+  && dpkg --install libqpdf-dev_*.deb \
+  && python3 -m pip install --no-cache-dir --upgrade \
+    pip \
+    wheel \
+    # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
+    pybind11 \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG PIKEPDF_VERSION
+
+RUN set -eux \
+  && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \
+  && mkdir wheels \
+  && python3 -m pip wheel \
+    # Build the package at the required version
+    pikepdf==${PIKEPDF_VERSION} \
+    # Output the *.whl into this directory
+    --wheel-dir wheels \
+    # Do not use a binary packge for the package being built
+    --no-binary=pikepdf \
+    # Do use binary packages for dependencies
+    --prefer-binary \
+    --no-cache-dir \
+  && ls -ahl wheels

docker-builders/Dockerfile.psycopg2

@@ -0,0 +1,49 @@
+# This Dockerfile builds the psycopg2 wheel
+# Inputs:
+#    - PSYCOPG2_VERSION - Version to build
+
+FROM python:3.9-slim-bullseye as main
+
+LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  python3-dev \
+  python3-pip \
+  # https://www.psycopg.org/docs/install.html#prerequisites
+  libpq-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/* \
+  && python3 -m pip install --no-cache-dir --upgrade pip wheel
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG PSYCOPG2_VERSION
+
+RUN set -eux \
+  && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \
+  && cd /usr/src \
+  && mkdir wheels \
+  && python3 -m pip wheel \
+    # Build the package at the required version
+    psycopg2==${PSYCOPG2_VERSION} \
+    # Output the *.whl into this directory
+    --wheel-dir wheels \
+    # Do not use a binary packge for the package being built
+    --no-binary=psycopg2 \
+    # Do use binary packages for dependencies
+    --prefer-binary \
+    --no-cache-dir \
+  && ls -ahl wheels/

docker-builders/Dockerfile.qpdf

@@ -0,0 +1,53 @@
+FROM debian:bullseye-slim as main
+
+LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  debhelper \
+  debian-keyring \
+  devscripts \
+  equivs  \
+  libtool \
+  # https://qpdf.readthedocs.io/en/stable/installation.html#system-requirements
+  libjpeg62-turbo-dev \
+  libgnutls28-dev \
+  packaging-dev \
+  zlib1g-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
+
+# In order to get the required version of qpdf, it is backported from bookwork
+# and then built from source
+RUN set -eux \
+  && echo "Building qpdf" \
+  && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
+  && apt-get update \
+  && mkdir qpdf \
+  && cd qpdf \
+  && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
+  && rm -rf /var/lib/apt/lists/* \
+  && cd qpdf-$QPDF_VERSION \
+  # We don't need to build the tests (also don't run them)
+  && rm -rf libtests \
+  && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
+  && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+  && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
+  && ls -ahl ../*.deb

docker/compose/docker-compose.env

@@ -22,6 +22,10 @@
 # Docker setup does not use the configuration file.
 # A few commonly adjusted settings are provided below.
 
+# This is required if you will be exposing Paperless-ngx on a public domain
+# (if doing so please consider security measures such as reverse proxy)
+#PAPERLESS_URL=https://paperless.example.com
+
 # Adjust this key if you plan to make paperless available publicly. It should
 # be a very long sequence of random characters. You don't need to remember it.
 #PAPERLESS_SECRET_KEY=change-me

docker/compose/docker-compose.portainer.yml

@@ -31,13 +31,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
@@ -55,7 +55,7 @@ services:
     ports:
       - 8010:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/compose/docker-compose.postgres-tika.yml

@@ -1,7 +1,6 @@
 # docker-compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware. The apache/tika image
-# does not support arm or arm64, however.
+# Paperless supports amd64, arm and arm64 hardware.
 #
 # All compose files of paperless configure paperless in the following way:
 #
@@ -34,13 +33,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
@@ -60,7 +59,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -78,14 +77,14 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7
+    image: docker.io/gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"
       - "--chromium-disable-routes=true"
 
   tika:
-    image: apache/tika
+    image: ghcr.io/paperless-ngx/tika:latest
     restart: unless-stopped
 
 volumes:

docker/compose/docker-compose.postgres.yml

@@ -29,13 +29,13 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
 
   db:
-    image: postgres:13
+    image: docker.io/library/postgres:13
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
@@ -53,7 +53,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/compose/docker-compose.sqlite-tika.arm.yml

@@ -1,84 +0,0 @@
-# docker-compose file for running paperless from the docker container registry.
-# This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware.
-#
-# All compose files of paperless configure paperless in the following way:
-#
-# - Paperless is (re)started on system boot, if it was running before shutdown.
-# - Docker volumes for storing data are managed by Docker.
-# - Folders for importing and exporting files are created in the same directory
-#   as this file and mounted to the correct folders inside the container.
-# - Paperless listens on port 8000.
-#
-# SQLite is used as the database. The SQLite file is stored in the data volume.
-#
-# iwishiwasaneagle/apache-tika-arm docker image is used to enable arm64 arch
-# which apache/tika does not currently support.
-#
-# In addition to that, this docker-compose file adds the following optional
-# configurations:
-#
-# - Apache Tika and Gotenberg servers are started with paperless and paperless
-#   is configured to use these services. These provide support for consuming
-#   Office documents (Word, Excel, Power Point and their LibreOffice counter-
-#   parts.
-#
-# To install and update paperless with this file, do the following:
-#
-# - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
-#   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
-#
-# For more extensive installation and update instructions, refer to the
-# documentation.
-
-version: "3.4"
-services:
-  broker:
-    image: redis:6.0
-    restart: unless-stopped
-    volumes:
-      - redisdata:/data
-
-  webserver:
-    image: ghcr.io/paperless-ngx/paperless-ngx:latest
-    restart: unless-stopped
-    depends_on:
-      - broker
-      - gotenberg
-      - tika
-    ports:
-      - 8000:8000
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-    volumes:
-      - data:/usr/src/paperless/data
-      - media:/usr/src/paperless/media
-      - ./export:/usr/src/paperless/export
-      - ./consume:/usr/src/paperless/consume
-    env_file: docker-compose.env
-    environment:
-      PAPERLESS_REDIS: redis://broker:6379
-      PAPERLESS_TIKA_ENABLED: 1
-      PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
-      PAPERLESS_TIKA_ENDPOINT: http://tika:9998
-
-  gotenberg:
-    image: thecodingmachine/gotenberg
-    restart: unless-stopped
-    environment:
-      DISABLE_GOOGLE_CHROME: 1
-
-  tika:
-    image: iwishiwasaneagle/apache-tika-arm@sha256:a78c25ffe57ecb1a194b2859d42a61af46e9e845191512b8f1a4bf90578ffdfd
-    restart: unless-stopped
-
-volumes:
-  data:
-  media:
-  redisdata:

docker/compose/docker-compose.sqlite-tika.yml

@@ -1,8 +1,6 @@
 # docker-compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
-# Paperless supports amd64, arm and arm64 hardware. The apache/tika image
-# does not support arm or arm64, however.
-#
+# Paperless supports amd64, arm and arm64 hardware.
 # All compose files of paperless configure paperless in the following way:
 #
 # - Paperless is (re)started on system boot, if it was running before shutdown.
@@ -35,7 +33,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
@@ -50,7 +48,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -67,14 +65,14 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: gotenberg/gotenberg:7
+    image: docker.io/gotenberg/gotenberg:7.4
     restart: unless-stopped
     command:
       - "gotenberg"
       - "--chromium-disable-routes=true"
 
   tika:
-    image: apache/tika
+    image: ghcr.io/paperless-ngx/tika:latest
     restart: unless-stopped
 
 volumes:

docker/compose/docker-compose.sqlite.yml

@@ -26,7 +26,7 @@
 version: "3.4"
 services:
   broker:
-    image: redis:6.0
+    image: docker.io/library/redis:6.0
     restart: unless-stopped
     volumes:
       - redisdata:/data
@@ -39,7 +39,7 @@ services:
     ports:
       - 8000:8000
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000"]
+      test: ["CMD", "curl", "-fs", "-S", "--max-time", "2", "http://localhost:8000"]
       interval: 30s
       timeout: 10s
       retries: 5

docker/docker-entrypoint.sh

@@ -2,6 +2,37 @@
 
 set -e
 
+# Adapted from:
+# https://github.com/docker-library/postgres/blob/master/docker-entrypoint.sh
+# usage: file_env VAR
+#    ie: file_env 'XYZ_DB_PASSWORD' will allow for "$XYZ_DB_PASSWORD_FILE" to
+# fill in the value of "$XYZ_DB_PASSWORD" from a file, especially for Docker's
+# secrets feature
+file_env() {
+	local var="$1"
+	local fileVar="${var}_FILE"
+
+	# Basic validation
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+		exit 1
+	fi
+
+	# Only export var if the _FILE exists
+	if [ "${!fileVar:-}" ]; then
+		# And the file exists
+		if [[ -f ${!fileVar} ]]; then
+			echo "Setting ${var} from file"
+			val="$(< "${!fileVar}")"
+			export "$var"="$val"
+		else
+			echo "File ${!fileVar} doesn't exist"
+			exit 1
+		fi
+	fi
+
+}
+
 # Source: https://github.com/sameersbn/docker-gitlab/
 map_uidgid() {
 	USERMAP_ORIG_UID=$(id -u paperless)
@@ -15,26 +46,56 @@ map_uidgid() {
 	fi
 }
 
+map_folders() {
+	# Export these so they can be used in docker-prepare.sh
+	export DATA_DIR="${PAPERLESS_DATA_DIR:-/usr/src/paperless/data}"
+	export MEDIA_ROOT_DIR="${PAPERLESS_MEDIA_ROOT:-/usr/src/paperless/media}"
+}
+
 initialize() {
+
+	# Setup environment from secrets before anything else
+	for env_var in \
+		PAPERLESS_DBUSER \
+		PAPERLESS_DBPASS \
+		PAPERLESS_SECRET_KEY \
+		PAPERLESS_AUTO_LOGIN_USERNAME \
+		PAPERLESS_ADMIN_USER \
+		PAPERLESS_ADMIN_MAIL \
+		PAPERLESS_ADMIN_PASSWORD; do
+		# Check for a version of this var with _FILE appended
+		# and convert the contents to the env var value
+		file_env ${env_var}
+	done
+
+	# Change the user and group IDs if needed
 	map_uidgid
 
-	for dir in export data data/index media media/documents media/documents/originals media/documents/thumbnails; do
-		if [[ ! -d "../$dir" ]]; then
-			echo "Creating directory ../$dir"
-			mkdir ../$dir
+	# Check for overrides of certain folders
+	map_folders
+
+	local export_dir="/usr/src/paperless/export"
+
+	for dir in "${export_dir}" "${DATA_DIR}" "${DATA_DIR}/index" "${MEDIA_ROOT_DIR}" "${MEDIA_ROOT_DIR}/documents" "${MEDIA_ROOT_DIR}/documents/originals" "${MEDIA_ROOT_DIR}/documents/thumbnails"; do
+		if [[ ! -d "${dir}" ]]; then
+			echo "Creating directory ${dir}"
+			mkdir "${dir}"
 		fi
 	done
 
-	echo "Creating directory /tmp/paperless"
-	mkdir -p /tmp/paperless
+	local tmp_dir="/tmp/paperless"
+	echo "Creating directory ${tmp_dir}"
+	mkdir -p "${tmp_dir}"
 
 	set +e
 	echo "Adjusting permissions of paperless files. This may take a while."
-	chown -R paperless:paperless /tmp/paperless
-	find .. -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	chown -R paperless:paperless ${tmp_dir}
+	for dir in "${export_dir}" "${DATA_DIR}" "${MEDIA_ROOT_DIR}"; do
+		find "${dir}" -not \( -user paperless -and -group paperless \) -exec chown paperless:paperless {} +
+	done
 	set -e
 
-	gosu paperless /sbin/docker-prepare.sh
+	${gosu_cmd[@]} /sbin/docker-prepare.sh
 }
 
 install_languages() {
@@ -76,6 +137,11 @@ install_languages() {
 
 echo "Paperless-ngx docker container starting..."
 
+gosu_cmd=(gosu paperless)
+if [ $(id -u) == $(id -u paperless) ]; then
+	gosu_cmd=()
+fi
+
 # Install additional languages if specified
 if [[ -n "$PAPERLESS_OCR_LANGUAGES" ]]; then
 	install_languages "$PAPERLESS_OCR_LANGUAGES"
@@ -85,7 +151,7 @@ initialize
 
 if [[ "$1" != "/"* ]]; then
 	echo Executing management command "$@"
-	exec gosu paperless python3 manage.py "$@"
+	exec ${gosu_cmd[@]} python3 manage.py "$@"
 else
 	echo Executing "$@"
 	exec "$@"

docker/docker-prepare.sh

@@ -1,16 +1,19 @@
 #!/usr/bin/env bash
 
+set -e
+
 wait_for_postgres() {
-	attempt_num=1
-	max_attempts=5
+	local attempt_num=1
+	local max_attempts=5
 
 	echo "Waiting for PostgreSQL to start..."
 
-	host="${PAPERLESS_DBHOST:=localhost}"
-	port="${PAPERLESS_DBPORT:=5342}"
+	local host="${PAPERLESS_DBHOST:-localhost}"
+	local port="${PAPERLESS_DBPORT:-5432}"
 
-
-	while [ ! "$(pg_isready -h $host -p $port)" ]; do
+	# Disable warning, host and port can't have spaces
+	# shellcheck disable=SC2086
+	while [ ! "$(pg_isready -h ${host} -p ${port})" ]; do
 
 		if [ $attempt_num -eq $max_attempts ]; then
 			echo "Unable to connect to database."
@@ -25,6 +28,14 @@ wait_for_postgres() {
 	done
 }
 
+wait_for_redis() {
+	# We use a Python script to send the Redis ping
+	# instead of installing redis-tools just for 1 thing
+	if ! python3 /sbin/wait-for-redis.py; then
+		exit 1
+	fi
+}
+
 migrations() {
 	(
 		# flock is in place to prevent multiple containers from doing migrations
@@ -33,17 +44,18 @@ migrations() {
 		flock 200
 		echo "Apply database migrations..."
 		python3 manage.py migrate
-	) 200>/usr/src/paperless/data/migration_lock
+	) 200>"${DATA_DIR}/migration_lock"
 }
 
 search_index() {
-	index_version=1
-	index_version_file=/usr/src/paperless/data/.index_version
 
-	if [[ (! -f "$index_version_file") || $(<$index_version_file) != "$index_version" ]]; then
+	local index_version=1
+	local index_version_file=${DATA_DIR}/.index_version
+
+	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
 		echo "Search index out of date. Updating..."
-		python3 manage.py document_index reindex
-		echo $index_version | tee $index_version_file >/dev/null
+		python3 manage.py document_index reindex --no-progress-bar
+		echo ${index_version} | tee "${index_version_file}" >/dev/null
 	fi
 }
 
@@ -58,6 +70,8 @@ do_work() {
 		wait_for_postgres
 	fi
 
+	wait_for_redis
+
 	migrations
 
 	search_index

docker/install_management_commands.sh

@@ -1,6 +1,19 @@
 #!/usr/bin/env bash
 
-for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
+set -eu
+
+for command in decrypt_documents \
+	document_archiver \
+	document_exporter \
+	document_importer \
+	mail_fetcher \
+	document_create_classifier \
+	document_index \
+	document_renamer \
+	document_retagger \
+	document_thumbnails \
+	document_sanity_checker \
+	manage_superuser;
 do
 	echo "installing $command..."
 	sed "s/management_command/$command/g" management_script.sh > /usr/local/bin/$command

docker/paperless_cmd.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+rootless_args=()
+if [ $(id -u) == $(id -u paperless) ]; then
+	rootless_args=(
+		--user
+		paperless
+		--logfile
+		supervisord.log
+		--pidfile
+		supervisord.pid
+	)
+fi
+
+/usr/local/bin/supervisord -c /etc/supervisord.conf ${rootless_args[@]}

docker/supervisord.conf

@@ -28,6 +28,7 @@ stderr_logfile_maxbytes=0
 [program:scheduler]
 command=python3 manage.py qcluster
 user=paperless
+stopasgroup = true
 
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

docker/wait-for-redis.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""
+Simple script which attempts to ping the Redis broker as set in the environment for
+a certain number of times, waiting a little bit in between
+
+"""
+import os
+import sys
+import time
+from typing import Final
+
+from redis import Redis
+
+if __name__ == "__main__":
+
+    MAX_RETRY_COUNT: Final[int] = 5
+    RETRY_SLEEP_SECONDS: Final[int] = 5
+
+    REDIS_URL: Final[str] = os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")
+
+    print(f"Waiting for Redis: {REDIS_URL}", flush=True)
+
+    attempt = 0
+    with Redis.from_url(url=REDIS_URL) as client:
+        while attempt < MAX_RETRY_COUNT:
+            try:
+                client.ping()
+                break
+            except Exception as e:
+                print(
+                    f"Redis ping #{attempt} failed.\n"
+                    f"Error: {str(e)}.\n"
+                    f"Waiting {RETRY_SLEEP_SECONDS}s",
+                    flush=True,
+                )
+                time.sleep(RETRY_SLEEP_SECONDS)
+                attempt += 1
+
+    if attempt >= MAX_RETRY_COUNT:
+        print(f"Failed to connect to: {REDIS_URL}")
+        sys.exit(os.EX_UNAVAILABLE)
+    else:
+        print(f"Connected to Redis broker: {REDIS_URL}")
+        sys.exit(os.EX_OK)

docs/Dockerfile

@@ -1,10 +1,10 @@
 FROM python:3.5.1
 
 # Install Sphinx and Pygments
-RUN pip install Sphinx Pygments
+RUN pip install --no-cache-dir Sphinx Pygments \
+  # Setup directories, copy data
+  && mkdir /build
 
-# Setup directories, copy data
-RUN mkdir /build
 COPY . /build
 WORKDIR /build/docs
 

docs/Makefile

@@ -24,6 +24,7 @@ I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html       to make standalone HTML files"
+	@echo "  livehtml   to preview changes with live reload in your browser"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
 	@echo "  singlehtml to make a single large HTML file"
 	@echo "  pickle     to make pickle files"
@@ -54,6 +55,9 @@ html:
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
+livehtml:
+	sphinx-autobuild "./" "$(BUILDDIR)" $(O)
+
 dirhtml:
 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 	@echo

docs/_static/css/custom.css

@@ -0,0 +1,596 @@
+/* Variables */
+:root {
+  --color-text-body: #5c5962;
+  --color-text-body-light: #fcfcfc;
+  --color-text-anchor: #7253ed;
+  --color-text-alt: rgba(0, 0, 0, 0.3);
+  --color-text-title: #27262b;
+  --color-text-code-inline: #e74c3c;
+  --color-text-code-nt: #062873;
+  --color-text-selection: #b19eff;
+  --color-bg-body: #fcfcfc;
+  --color-bg-body-alt: #f3f6f6;
+  --color-bg-side-nav: #f5f6fa;
+  --color-bg-side-nav-hover: #ebedf5;
+  --color-bg-code-block: var(--color-bg-side-nav);
+  --color-border: #eeebee;
+  --color-btn-neutral-bg: #f3f6f6;
+  --color-btn-neutral-bg-hover: #e5ebeb;
+  --color-success-title: #1abc9c;
+  --color-success-body: #dbfaf4;
+  --color-warning-title: #f0b37e;
+  --color-warning-body: #ffedcc;
+  --color-danger-title: #f29f97;
+  --color-danger-body: #fdf3f2;
+  --color-info-title: #6ab0de;
+  --color-info-body: #e7f2fa;
+}
+
+.dark-mode {
+  --color-text-body: #abb2bf;
+  --color-text-body-light: #9499a2;
+  --color-text-alt: rgba(0255, 255, 255, 0.5);
+  --color-text-title: var(--color-text-anchor);
+  --color-text-code-inline: #abb2bf;
+  --color-text-code-nt: #2063f3;
+  --color-text-selection: #030303;
+  --color-bg-body: #1d1d20 !important;
+  --color-bg-body-alt: #131315;
+  --color-bg-side-nav: #18181a;
+  --color-bg-side-nav-hover: #101216;
+  --color-bg-code-block: #101216;
+  --color-border: #47494f;
+  --color-btn-neutral-bg: #242529;
+  --color-btn-neutral-bg-hover: #101216;
+  --color-success-title: #02120f;
+  --color-success-body: #041b17;
+  --color-warning-title: #1b0e03;
+  --color-warning-body: #371d06;
+  --color-danger-title: #120902;
+  --color-danger-body: #1b0503;
+  --color-info-title: #020608;
+  --color-info-body: #06141e;
+}
+
+* {
+  transition: background-color 0.3s ease, border-color 0.3s ease;
+}
+
+/* Typography */
+body {
+  font-family: system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif;
+  font-size: inherit;
+  line-height: 1.4;
+  color: var(--color-text-body);
+}
+
+.rst-content p {
+  word-break: break-word;
+}
+
+h1, h2, h3, h4, h5, h6 {
+  font-family: inherit;
+}
+
+.rst-content .toctree-wrapper>p.caption, .rst-content h1, .rst-content h2, .rst-content h3, .rst-content h4, .rst-content h5, .rst-content h6 {
+  padding-top: .5em;
+}
+
+p, .main-content-wrap, .rst-content .section ul, .rst-content .toctree-wrapper ul, .rst-content section ul, .wy-plain-list-disc, article ul {
+  line-height: 1.6;
+}
+
+pre, .code, .rst-content .linenodiv pre, .rst-content div[class^=highlight] pre, .rst-content pre.literal-block {
+  font-family: "SFMono-Regular", Menlo,Consolas, Monospace;
+  font-size: 0.75em;
+  line-height: 1.8;
+}
+
+.wy-menu-vertical li.toctree-l3,.wy-menu-vertical li.toctree-l4 {
+  font-size: 1rem
+}
+
+.rst-versions {
+  font-family: inherit;
+  line-height: 1;
+}
+
+footer, footer p {
+  font-size: .8rem;
+}
+
+footer .rst-footer-buttons {
+  font-size: 1rem;
+}
+
+@media (max-width: 400px) {
+  /* break code lines on mobile */
+  pre, code {
+    word-break: break-word;
+  }
+}
+
+
+/* Layout */
+.wy-side-nav-search, .wy-menu-vertical {
+  width: auto;
+}
+
+.wy-nav-side {
+  z-index: 0;
+  display: flex;
+  flex-wrap: wrap;
+  background-color: var(--color-bg-side-nav);
+}
+
+.wy-side-scroll {
+  width: 100%;
+  overflow-y: auto;
+}
+
+@media (min-width: 66.5rem) {
+  .wy-side-scroll {
+      width:264px
+  }
+}
+
+@media (min-width: 50rem) {
+  .wy-nav-side {
+      flex-wrap: nowrap;
+      position: fixed;
+      width: 248px;
+      height: 100%;
+      flex-direction: column;
+      border-right: 1px solid var(--color-border);
+      align-items:flex-end
+  }
+}
+
+@media (min-width: 66.5rem) {
+  .wy-nav-side {
+      width: calc((100% - 1064px) / 2 + 264px);
+      min-width:264px
+  }
+}
+
+@media (min-width: 50rem) {
+  .wy-nav-content-wrap {
+      position: relative;
+      max-width: 800px;
+      margin-left:248px
+  }
+}
+
+@media (min-width: 66.5rem) {
+  .wy-nav-content-wrap {
+      margin-left:calc((100% - 1064px) / 2 + 264px)
+  }
+}
+
+
+/* Colors */
+body.wy-body-for-nav,
+.wy-nav-content {
+  background: var(--color-bg-body);
+}
+
+.wy-nav-side {
+  border-right: 1px solid var(--color-border);
+}
+
+.wy-side-nav-search, .wy-nav-top {
+  background: var(--color-bg-side-nav);
+  border-bottom: 1px solid var(--color-border);
+}
+
+.wy-nav-content-wrap {
+  background: inherit;
+}
+
+.wy-side-nav-search > a, .wy-nav-top a, .wy-nav-top i {
+  color: var(--color-text-title);
+}
+
+.wy-side-nav-search > a:hover, .wy-nav-top a:hover {
+  background: transparent;
+}
+
+.wy-side-nav-search > div.version {
+  color: var(--color-text-alt)
+}
+
+.wy-side-nav-search > div[role="search"] {
+  border-top: 1px solid var(--color-border);
+}
+
+.wy-menu-vertical li.toctree-l2.current>a, .wy-menu-vertical li.toctree-l2.current li.toctree-l3>a,
+.wy-menu-vertical li.toctree-l3.current>a, .wy-menu-vertical li.toctree-l3.current li.toctree-l4>a {
+  background: var(--color-bg-side-nav);
+}
+
+.rst-content .highlighted {
+  background: #eedd85;
+  box-shadow: 0 0 0 2px #eedd85;
+  font-weight: 600;
+}
+
+.wy-side-nav-search input[type=text],
+html.writer-html5 .rst-content table.docutils th {
+  color: var(--color-text-body);
+}
+
+.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td,
+.wy-table-backed,
+.wy-table-odd td,
+.wy-table-striped tr:nth-child(2n-1) td {
+  background-color: var(--color-bg-body-alt);
+}
+
+.rst-content table.docutils,
+.wy-table-bordered-all,
+html.writer-html5 .rst-content table.docutils th,
+.rst-content table.docutils td,
+.wy-table-bordered-all td,
+hr {
+  border-color: var(--color-border) !important;
+}
+
+::selection {
+  background: var(--color-text-selection);
+}
+
+/* Ridiculous rules are taken from sphinx_rtd */
+.rst-content .admonition-title,
+.wy-alert-title {
+  color: var(--color-text-body-light);
+}
+
+.rst-content .hint,
+.rst-content .important,
+.rst-content .tip,
+.rst-content .wy-alert-success,
+.wy-alert.wy-alert-success {
+  background: var(--color-success-body);
+}
+
+.rst-content .hint .admonition-title,
+.rst-content .hint .wy-alert-title,
+.rst-content .important .admonition-title,
+.rst-content .important .wy-alert-title,
+.rst-content .tip .admonition-title,
+.rst-content .tip .wy-alert-title,
+.rst-content .wy-alert-success .admonition-title,
+.rst-content .wy-alert-success .wy-alert-title,
+.wy-alert.wy-alert-success .rst-content .admonition-title,
+.wy-alert.wy-alert-success .wy-alert-title {
+  background-color: var(--color-success-title);
+}
+
+.rst-content .admonition-todo,
+.rst-content .attention,
+.rst-content .caution,
+.rst-content .warning,
+.rst-content .wy-alert-warning,
+.wy-alert.wy-alert-warning {
+  background: var(--color-warning-body);
+}
+
+.rst-content .admonition-todo .admonition-title,
+.rst-content .admonition-todo .wy-alert-title,
+.rst-content .attention .admonition-title,
+.rst-content .attention .wy-alert-title,
+.rst-content .caution .admonition-title,
+.rst-content .caution .wy-alert-title,
+.rst-content .warning .admonition-title,
+.rst-content .warning .wy-alert-title,
+.rst-content .wy-alert-warning .admonition-title,
+.rst-content .wy-alert-warning .wy-alert-title,
+.rst-content .wy-alert.wy-alert-warning .admonition-title,
+.wy-alert.wy-alert-warning .rst-content .admonition-title,
+.wy-alert.wy-alert-warning .wy-alert-title {
+  background: var(--color-warning-title);
+}
+
+.rst-content .danger,
+.rst-content .error,
+.rst-content .wy-alert-danger,
+.wy-alert.wy-alert-danger {
+  background: var(--color-danger-body);
+}
+
+.rst-content .danger .admonition-title,
+.rst-content .danger .wy-alert-title,
+.rst-content .error .admonition-title,
+.rst-content .error .wy-alert-title,
+.rst-content .wy-alert-danger .admonition-title,
+.rst-content .wy-alert-danger .wy-alert-title,
+.wy-alert.wy-alert-danger .rst-content .admonition-title,
+.wy-alert.wy-alert-danger .wy-alert-title {
+  background: var(--color-danger-title);
+}
+
+.rst-content .note,
+.rst-content .seealso,
+.rst-content .wy-alert-info,
+.wy-alert.wy-alert-info {
+  background: var(--color-info-body);
+}
+
+.rst-content .note .admonition-title,
+.rst-content .note .wy-alert-title,
+.rst-content .seealso .admonition-title,
+.rst-content .seealso .wy-alert-title,
+.rst-content .wy-alert-info .admonition-title,
+.rst-content .wy-alert-info .wy-alert-title,
+.wy-alert.wy-alert-info .rst-content .admonition-title,
+.wy-alert.wy-alert-info .wy-alert-title {
+  background: var(--color-info-title);
+}
+
+
+
+/* Links */
+a, a:visited,
+.wy-menu-vertical a,
+a.icon.icon-home,
+.wy-menu-vertical li.toctree-l1.current > a.current {
+  color: var(--color-text-anchor);
+  text-decoration: none;
+}
+
+a:hover, .wy-breadcrumbs-aside a {
+  color: var(--color-text-anchor); /* reset */
+}
+
+.rst-versions a, .rst-versions .rst-current-version {
+  color: #var(--color-text-anchor);
+}
+
+.wy-nav-content a.reference, .wy-nav-content a:not([class]) {
+  background-image: linear-gradient(var(--color-border) 0%, var(--color-border) 100%);
+  background-repeat: repeat-x;
+  background-position: 0 100%;
+  background-size: 1px 1px;
+}
+
+.wy-nav-content a.reference:hover, .wy-nav-content a:not([class]):hover {
+  background-image: linear-gradient(rgba(114,83,237,0.45) 0%, rgba(114,83,237,0.45) 100%);
+  background-size: 1px 1px;
+}
+
+.wy-menu-vertical a:hover,
+.wy-menu-vertical li.current a:hover,
+.wy-menu-vertical a:active {
+  background: var(--color-bg-side-nav-hover) !important;
+  color: var(--color-text-body);
+}
+
+.wy-menu-vertical li.toctree-l1.current>a,
+.wy-menu-vertical li.current>a,
+.wy-menu-vertical li.on a {
+  background-color: var(--color-bg-side-nav-hover);
+  border: none;
+  font-weight: normal;
+}
+
+.wy-menu-vertical li.current {
+  background-color: inherit;
+}
+
+.wy-menu-vertical li.current a {
+  border-right: none;
+}
+
+.wy-menu-vertical li.toctree-l2 a,
+.wy-menu-vertical li.toctree-l3 a,
+.wy-menu-vertical li.toctree-l4 a,
+.wy-menu-vertical li.toctree-l5 a,
+.wy-menu-vertical li.toctree-l6 a,
+.wy-menu-vertical li.toctree-l7 a,
+.wy-menu-vertical li.toctree-l8 a,
+.wy-menu-vertical li.toctree-l9 a,
+.wy-menu-vertical li.toctree-l10 a {
+  color: var(--color-text-body);
+}
+
+a.image-reference, a.image-reference:hover {
+  background: none !important;
+}
+
+a.image-reference img {
+  cursor: zoom-in;
+}
+
+
+/* Code blocks */
+.rst-content code, .rst-content tt, code {
+  padding: 0.25em;
+  font-weight: 400;
+  background-color: var(--color-bg-code-block);
+  border: 1px solid var(--color-border);
+  border-radius: 4px;
+}
+
+.rst-content div[class^=highlight], .rst-content pre.literal-block {
+  padding: 0.7rem;
+  margin-top: 0;
+  margin-bottom: 0.75rem;
+  overflow-x: auto;
+  background-color: var(--color-bg-side-nav);
+  border-color: var(--color-border);
+  border-radius: 4px;
+  box-shadow: none;
+}
+
+.rst-content .admonition-title,
+.rst-content div.admonition,
+.wy-alert-title {
+  padding: 10px 12px;
+  border-top-left-radius: 4px;
+  border-top-right-radius: 4px;
+}
+
+.highlight .go {
+  color: inherit;
+}
+
+.highlight .nt {
+  color: var(--color-text-code-nt);
+}
+
+.rst-content code.literal,
+.rst-content tt.literal {
+  border-color: var(--color-border);
+  background-color: var(--color-border);
+  color: var(--color-text-code-inline)
+}
+
+
+/* Search */
+.wy-side-nav-search input[type=text] {
+  border: none;
+  border-radius: 0;
+  background-color: transparent;
+  font-family: inherit;
+  font-size: .85rem;
+  box-shadow: none;
+  padding: .7rem 1rem .7rem 2.8rem;
+  margin: 0;
+}
+
+#rtd-search-form {
+  position: relative;
+}
+
+#rtd-search-form:before {
+  font: normal normal normal 14px/1 FontAwesome;
+  font-size: inherit;
+  text-rendering: auto;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  content: "\f002";
+  color: var(--color-text-alt);
+  position: absolute;
+  left: 1.5rem;
+  top: .7rem;
+}
+
+/* Side nav */
+.wy-side-nav-search {
+  padding: 1rem 0 0 0;
+}
+
+.wy-menu-vertical li a button.toctree-expand {
+  float: right;
+  margin-right: -1.5em;
+  padding: 0 .5em;
+}
+
+.wy-menu-vertical a,
+.wy-menu-vertical li.current>a,
+.wy-menu-vertical li.current li>a {
+  padding-right: 1.5em !important;
+}
+
+.wy-menu-vertical li.current li>a.current {
+  font-weight: 600;
+}
+
+/* Misc spacing */
+.rst-content .admonition-title, .wy-alert-title {
+  padding: 10px 12px;
+}
+
+/* Buttons */
+.btn {
+  display: inline-block;
+  box-sizing: border-box;
+  padding: 0.3em 1em;
+  margin: 0;
+  font-family: inherit;
+  font-size: inherit;
+  font-weight: 500;
+  line-height: 1.5;
+  color: #var(--color-text-anchor);
+  text-decoration: none;
+  vertical-align: baseline;
+  background-color: #f7f7f7;
+  border-width: 0;
+  border-radius: 4px;
+  box-shadow: 0 1px 2px rgba(0,0,0,0.12),0 3px 10px rgba(0,0,0,0.08);
+  appearance: none;
+}
+
+.btn:active {
+  padding: 0.3em 1em;
+}
+
+.rst-content .btn:focus {
+  outline: 1px solid #ccc;
+}
+
+.rst-content .btn-neutral, .rst-content .btn span.fa {
+  color: var(--color-text-body) !important;
+}
+
+.btn-neutral {
+  background-color: var(--color-btn-neutral-bg) !important;
+  color: var(--color-btn-neutral-text) !important;
+  border: 1px solid var(--color-btn-neutral-bg);
+}
+
+.btn:hover, .btn-neutral:hover {
+  background-color: var(--color-btn-neutral-bg-hover) !important;
+}
+
+
+/* Icon overrides */
+.wy-side-nav-search a.icon-home:before {
+  display: none;
+}
+
+.fa-minus-square-o:before,.wy-menu-vertical li.current>a button.toctree-expand:before,.wy-menu-vertical li.on a button.toctree-expand:before {
+  content: "\f106"; /* fa-angle-up */
+}
+
+.fa-plus-square-o:before, .wy-menu-vertical li button.toctree-expand:before {
+  content: "\f107"; /* fa-angle-down */
+}
+
+
+/* Misc */
+.wy-nav-top {
+  line-height: 36px;
+}
+
+.wy-nav-top > i {
+  font-size: 24px;
+  padding: 8px 0 0 2px;
+  color:#var(--color-text-anchor);
+}
+
+.rst-content table.docutils td,
+.rst-content table.docutils th,
+.rst-content table.field-list td,
+.rst-content table.field-list th,
+.wy-table td,
+.wy-table th {
+  padding: 8px 14px;
+}
+
+.dark-mode-toggle {
+  position: absolute;
+  top: 14px;
+  right: 12px;
+  height: 20px;
+  width: 24px;
+  z-index: 10;
+  border: none;
+  background-color: transparent;
+  color: inherit;
+  opacity: 0.7;
+}
+
+.wy-nav-content-wrap {
+  z-index: 20;
+}

docs/_static/custom.css

@@ -1,14 +0,0 @@
-/* override table width restrictions */
-@media screen and (min-width: 767px) {
-
-  .wy-table-responsive table td {
-    /* !important prevents the common CSS stylesheets from
-       overriding this as on RTD they are loaded after this stylesheet */
-    white-space: normal !important;
-  }
-
-  .wy-table-responsive {
-    overflow: visible !important;
-  }
-
-}

docs/_static/js/darkmode.js

@@ -0,0 +1,47 @@
+let toggleButton
+let icon
+
+function load() {
+	'use strict'
+
+	toggleButton = document.createElement('button')
+	toggleButton.setAttribute('title', 'Toggle dark mode')
+	toggleButton.classList.add('dark-mode-toggle')
+	icon = document.createElement('i')
+	icon.classList.add('fa', darkModeState ? 'fa-sun-o' : 'fa-moon-o')
+	toggleButton.appendChild(icon)
+	document.body.prepend(toggleButton)
+
+	// Listen for changes in the OS settings
+	// addListener is used because older versions of Safari don't support addEventListener
+	// prefersDarkQuery set in <head>
+	if (prefersDarkQuery) {
+		prefersDarkQuery.addListener(function (evt) {
+			toggleDarkMode(evt.matches)
+		})
+	}
+
+	// Initial setting depending on the prefers-color-mode or localstorage
+	// darkModeState should be set in the document <head> to prevent flash
+	if (darkModeState == undefined) darkModeState = false
+	toggleDarkMode(darkModeState)
+
+	// Toggles the "dark-mode" class on click and sets localStorage state
+	toggleButton.addEventListener('click', () => {
+		darkModeState = !darkModeState
+
+		toggleDarkMode(darkModeState)
+		localStorage.setItem('dark-mode', darkModeState)
+	})
+}
+
+function toggleDarkMode(state) {
+	document.documentElement.classList.toggle('dark-mode', state)
+	document.documentElement.classList.toggle('light-mode', !state)
+	icon.classList.remove('fa-sun-o')
+	icon.classList.remove('fa-moon-o')
+	icon.classList.add(state ? 'fa-sun-o' : 'fa-moon-o')
+	darkModeState = state
+}
+
+document.addEventListener('DOMContentLoaded', load)

docs/_templates/layout.html

@@ -0,0 +1,13 @@
+{% extends "!layout.html" %}
+{% block extrahead %}
+    <script>
+        // MediaQueryList object
+        const prefersDarkQuery = window.matchMedia("(prefers-color-scheme: dark)");
+        const lsDark = localStorage.getItem("dark-mode");
+        let darkModeState = lsDark !== null ? lsDark == "true" : prefersDarkQuery.matches;
+
+        document.documentElement.classList.toggle("dark-mode", darkModeState);
+        document.documentElement.classList.toggle("light-mode", !darkModeState);
+    </script>
+    {{ super() }}
+{% endblock %}

docs/administration.rst

@@ -117,6 +117,23 @@ Then you can start paperless-ngx with ``-d`` to have it run in the background.
 
                 image: ghcr.io/paperless-ngx/paperless-ngx:latest
 
+    .. note::
+        In version 1.7.1 and onwards, the Docker image can now be pinned to a release series.
+        This is often combined with automatic updaters such as Watchtower to allow safer
+        unattended upgrading to new bugfix releases only.  It is still recommended to always
+        review release notes before upgrading.  To pin your install to a release series, edit
+        the ``docker-compose.yml`` find the line that says
+
+            .. code::
+
+                image: ghcr.io/paperless-ngx/paperless-ngx:latest
+
+        and replace the version with the series you want to track, for example:
+
+            .. code::
+
+                image: ghcr.io/paperless-ngx/paperless-ngx:1.7
+
 Bare Metal Route
 ================
 
@@ -270,6 +287,10 @@ When you use the provided docker compose script, put the export inside the
 ``export`` folder in your paperless source directory. Specify ``../export``
 as the ``source``.
 
+.. note::
+
+    Importing from a previous version of Paperless may work, but for best results
+    it is suggested to match the versions.
 
 .. _utilities-retagger:
 
@@ -369,8 +390,8 @@ the naming scheme.
 
 .. warning::
 
-    Since this command moves you documents around alot, it is advised to to
-    a backup before. The renaming logic is robust and will never overwrite
+    Since this command moves your documents, it is advised to do
+    a backup beforehand. The renaming logic is robust and will never overwrite
     or delete a file, but you can't ever be careful enough.
 
 .. code::

docs/advanced_usage.rst

@@ -7,12 +7,12 @@ easier.
 
 .. _advanced-matching:
 
-Matching tags, correspondents and document types
-################################################
+Matching tags, correspondents, document types, and storage paths
+################################################################
 
-Paperless will compare the matching algorithms defined by every tag and
-correspondent already set in your database to see if they apply to the text in
-a document.  In other words, if you defined a tag called ``Home Utility``
+Paperless will compare the matching algorithms defined by every tag, correspondent,
+document type, and storage path in your database to see if they apply to the text
+in a document. In other words, if you define a tag called ``Home Utility``
 that had a ``match`` property of ``bc hydro`` and a ``matching_algorithm`` of
 ``literal``, Paperless will automatically tag your newly-consumed document with
 your ``Home Utility`` tag so long as the text ``bc hydro`` appears in the body
@@ -22,10 +22,10 @@ The matching logic is quite powerful. It supports searching the text of your
 document with different algorithms, and as such, some experimentation may be
 necessary to get things right.
 
-In order to have a tag, correspondent, or type assigned automatically to newly
-consumed documents, assign a match and matching algorithm using the web
-interface. These settings define when to assign correspondents, tags, and types
-to documents.
+In order to have a tag, correspondent, document type, or storage path assigned
+automatically to newly consumed documents, assign a match and matching algorithm
+using the web interface. These settings define when to assign tags, correspondents,
+document types, and storage paths to documents.
 
 The following algorithms are available:
 
@@ -37,7 +37,7 @@ The following algorithms are available:
 * **Literal:** Matches only if the match appears exactly as provided (i.e. preserve ordering) in the PDF.
 * **Regular expression:** Parses the match as a regular expression and tries to
   find a match within the document.
-* **Fuzzy match:** I dont know. Look at the source.
+* **Fuzzy match:** I don't know. Look at the source.
 * **Auto:** Tries to automatically match new documents. This does not require you
   to set a match. See the notes below.
 
@@ -47,9 +47,9 @@ defining a match text of ``"Bank of America" BofA`` using the *any* algorithm,
 will match documents that contain either "Bank of America" or "BofA", but will
 not match documents containing "Bank of South America".
 
-Then just save your tag/correspondent and run another document through the
-consumer.  Once complete, you should see the newly-created document,
-automatically tagged with the appropriate data.
+Then just save your tag, correspondent, document type, or storage path and run
+another document through the consumer.  Once complete, you should see the
+newly-created document, automatically tagged with the appropriate data.
 
 
 .. _advanced-automatic_matching:
@@ -58,9 +58,9 @@ Automatic matching
 ==================
 
 Paperless-ngx comes with a new matching algorithm called *Auto*. This matching
-algorithm tries to assign tags, correspondents, and document types to your
-documents based on how you have already assigned these on existing documents. It
-uses a neural network under the hood.
+algorithm tries to assign tags, correspondents, document types, and storage paths
+to your documents based on how you have already assigned these on existing documents.
+It uses a neural network under the hood.
 
 If, for example, all your bank statements of your account 123 at the Bank of
 America are tagged with the tag "bofa_123" and the matching algorithm of this
@@ -80,20 +80,21 @@ feature:
   that the neural network only learns from documents which you have correctly
   tagged before.
 * The matching algorithm can only work if there is a correlation between the
-  tag, correspondent, or document type and the document itself. Your bank
-  statements usually contain your bank account number and the name of the bank,
-  so this works reasonably well, However, tags such as "TODO" cannot be
-  automatically assigned.
+  tag, correspondent, document type, or storage path and the document itself.
+  Your bank statements usually contain your bank account number and the name
+  of the bank, so this works reasonably well, However, tags such as "TODO"
+  cannot be automatically assigned.
 * The matching algorithm needs a reasonable number of documents to identify when
-  to assign tags, correspondents, and types. If one out of a thousand documents
-  has the correspondent "Very obscure web shop I bought something five years
-  ago", it will probably not assign this correspondent automatically if you buy
-  something from them again. The more documents, the better.
+  to assign tags, correspondents, storage paths, and types. If one out of a
+  thousand documents has the correspondent "Very obscure web shop I bought
+  something five years ago", it will probably not assign this correspondent
+  automatically if you buy something from them again. The more documents, the better.
 * Paperless also needs a reasonable amount of negative examples to decide when
-  not to assign a certain tag, correspondent or type. This will usually be the
-  case as you start filling up paperless with documents. Example: If all your
-  documents are either from "Webshop" and "Bank", paperless will assign one of
-  these correspondents to ANY new document, if both are set to automatic matching.
+  not to assign a certain tag, correspondent, document type, or storage path. This will
+  usually be the case as you start filling up paperless with documents.
+  Example: If all your documents are either from "Webshop" and "Bank", paperless
+  will assign one of these correspondents to ANY new document, if both are set
+  to automatic matching.
 
 Hooking into the consumption process
 ####################################
@@ -120,10 +121,10 @@ Pre-consumption script
 ======================
 
 Executed after the consumer sees a new document in the consumption folder, but
-before any processing of the document is performed. This script receives exactly
-one argument:
+before any processing of the document is performed. This script can access the
+following relevant environment variables set:
 
-* Document file name
+* ``DOCUMENT_SOURCE_PATH``
 
 A simple but common example for this would be creating a simple script like
 this:
@@ -133,7 +134,7 @@ this:
 .. code:: bash
 
     #!/usr/bin/env bash
-    pdf2pdfocr.py -i ${1}
+    pdf2pdfocr.py -i ${DOCUMENT_SOURCE_PATH}
 
 ``/etc/paperless.conf``
 
@@ -156,16 +157,20 @@ Post-consumption script
 =======================
 
 Executed after the consumer has successfully processed a document and has moved it
-into paperless. It receives the following arguments:
+into paperless. It receives the following environment variables:
 
-* Document id
-* Generated file name
-* Source path
-* Thumbnail path
-* Download URL
-* Thumbnail URL
-* Correspondent
-* Tags
+* ``DOCUMENT_ID``
+* ``DOCUMENT_FILE_NAME``
+* ``DOCUMENT_CREATED``
+* ``DOCUMENT_MODIFIED``
+* ``DOCUMENT_ADDED``
+* ``DOCUMENT_SOURCE_PATH``
+* ``DOCUMENT_ARCHIVE_PATH``
+* ``DOCUMENT_THUMBNAIL_PATH``
+* ``DOCUMENT_DOWNLOAD_URL``
+* ``DOCUMENT_THUMBNAIL_URL``
+* ``DOCUMENT_CORRESPONDENT``
+* ``DOCUMENT_TAGS``
 
 The script can be in any language, but for a simple shell script
 example, you can take a look at `post-consumption-example.sh`_ in this project.
@@ -200,7 +205,7 @@ Troubleshooting:
 - Check your script's permission e.g. in case of permission error ``sudo chmod 755 post-consumption-example.sh``
 - Pipe your scripts's output to a log file e.g. ``echo "${DOCUMENT_ID}" | tee --append /usr/src/paperless/scripts/post-consumption-example.log``
 
-.. _post-consumption-example.sh: https://github.com/jonaswinkler/paperless-ngx/blob/master/scripts/post-consumption-example.sh
+.. _post-consumption-example.sh: https://github.com/paperless-ngx/paperless-ngx/blob/main/scripts/post-consumption-example.sh
 
 .. _advanced-file_name_handling:
 
@@ -243,7 +248,7 @@ will create a directory structure as follows:
     last filename a document was stored as. If you do rename a file, paperless will
     report your files as missing and won't be able to find them.
 
-Paperless provides the following placeholders withing filenames:
+Paperless provides the following placeholders within filenames:
 
 * ``{asn}``: The archive serial number of the document, or "none".
 * ``{correspondent}``: The name of the correspondent, or "none".
@@ -268,6 +273,17 @@ If paperless detects that two documents share the same filename, paperless will
 append ``_01``, ``_02``, etc to the filename. This happens if all the placeholders in a filename
 evaluate to the same value.
 
+.. hint::
+    You can affect how empty placeholders are treated by changing the following setting to
+    `true`.
+
+    .. code::
+
+        PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=True
+
+    Doing this results in all empty placeholders resolving to "" instead of "none" as stated above.
+    Spaces before empty placeholders are removed as well, empty directories are omitted.
+
 .. hint::
 
     Paperless checks the filename of a document whenever it is saved. Therefore,
@@ -290,3 +306,59 @@ evaluate to the same value.
 
     However, keep in mind that inside docker, if files get stored outside of the
     predefined volumes, they will be lost after a restart of paperless.
+
+
+Storage paths
+#############
+
+One of the best things in Paperless is that you can not only access the documents via the
+web interface, but also via the file system.
+
+When as single storage layout is not sufficient for your use case, storage paths come to
+the rescue. Storage paths allow you to configure more precisely where each document is stored
+in the file system.
+
+- Each storage path is a `PAPERLESS_FILENAME_FORMAT` and follows the rules described above
+- Each document is assigned a storage path using the matching algorithms described above, but
+  can be overwritten at any time
+
+For example, you could define the following two storage paths:
+
+1. Normal communications are put into a folder structure sorted by `year/correspondent`
+2. Communications with insurance companies are stored in a flat structure with longer file names,
+   but containing the full date of the correspondence.
+
+.. code::
+
+    By Year = {created_year}/{correspondent}/{title}
+    Insurances = Insurances/{correspondent}/{created_year}-{created_month}-{created_day} {title}
+
+
+If you then map these storage paths to the documents, you might get the following result.
+For simplicity, `By Year` defines the same structure as in the previous example above.
+
+.. code:: text
+
+   2019/                                   # By Year
+      My bank/
+        Statement January.pdf
+        Statement February.pdf
+
+    Insurances/                           # Insurances
+      Healthcare 123/
+        2022-01-01 Statement January.pdf
+        2022-02-02 Letter.pdf
+        2022-02-03 Letter.pdf
+      Dental 456/
+        2021-12-01 New Conditions.pdf
+
+
+.. hint::
+
+    Defining a storage path is optional. If no storage path is defined for a document, the global
+    `PAPERLESS_FILENAME_FORMAT` is applied.
+
+.. caution::
+
+    If you adjust the format of an existing storage path, old documents don't get relocated automatically.
+    You need to run the :ref:`document renamer <utilities-renamer>` to adjust their pathes.

docs/api.rst

@@ -31,7 +31,8 @@ The objects served by the document endpoint contain the following fields:
 *   ``tags``: List of IDs of tags assigned to this document, or empty list.
 *   ``document_type``: Document type of this document, or null.
 *   ``correspondent``:  Correspondent of this document or null.
-*   ``created``: The date at which this document was created.
+*   ``created``: The date time at which this document was created.
+*   ``created_date``: The date (YYYY-MM-DD) at which this document was created. Optional. If also passed with created, this is ignored.
 *   ``modified``: The date at which this document was last edited in paperless. Read-only.
 *   ``added``: The date at which this document was added to paperless. Read-only.
 *   ``archive_serial_number``: The identifier of this document in a physical document archive.
@@ -240,11 +241,13 @@ be instructed to consume the document from there.
 The endpoint supports the following optional form fields:
 
 *   ``title``: Specify a title that the consumer should use for the document.
+*   ``created``: Specify a DateTime where the document was created (e.g. "2016-04-19" or "2016-04-19 06:15:00+02:00").
 *   ``correspondent``: Specify the ID of a correspondent that the consumer should use for the document.
 *   ``document_type``: Similar to correspondent.
 *   ``tags``: Similar to correspondent. Specify this multiple times to have multiple tags added
     to the document.
 
+
 The endpoint will immediately return "OK" if the document consumption process
 was started successfully. No additional status information about the consumption
 process itself is available, since that happens in a different process.

docs/conf.py

@@ -2,6 +2,8 @@ import sphinx_rtd_theme
 
 
 __version__ = None
+__full_version_str__ = None
+__major_minor_version_str__ = None
 exec(open("../src/paperless/version.py").read())
 
 
@@ -12,13 +14,17 @@ extensions = [
     "sphinx.ext.imgmath",
     "sphinx.ext.viewcode",
     "sphinx_rtd_theme",
+    "myst_parser",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-# templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = ".rst"
+source_suffix = {
+    ".rst": "restructuredtext",
+    ".md": "markdown",
+}
 
 # The encoding of source files.
 # source_encoding = 'utf-8-sig'
@@ -41,9 +47,9 @@ copyright = "2015-2022, Daniel Quinn, Jonas Winkler, and the paperless-ngx team"
 #
 
 # The short X.Y version.
-version = ".".join([str(_) for _ in __version__[:2]])
+version = __major_minor_version_str__
 # The full version, including alpha/beta/rc tags.
-release = ".".join([str(_) for _ in __version__[:3]])
+release = __full_version_str__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -119,6 +125,16 @@ html_theme_path = []
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
+# These paths are either relative to html_static_path
+# or fully qualified paths (eg. https://...)
+html_css_files = [
+    "css/custom.css",
+]
+
+html_js_files = [
+    "js/darkmode.js",
+]
+
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.

docs/configuration.rst

@@ -111,6 +111,14 @@ PAPERLESS_FILENAME_FORMAT=<format>
 
     Default is none, which disables this feature.
 
+PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=<bool>
+    Tells paperless to replace placeholders in `PAPERLESS_FILENAME_FORMAT` that would resolve
+    to 'none' to be omitted from the resulting filename. This also holds true for directory
+    names.
+    See :ref:`advanced-file_name_handling` for details.
+
+    Defaults to `false` which disables this feature.
+
 PAPERLESS_LOGGING_DIR=<path>
     This is where paperless will store log files.
 
@@ -130,6 +138,8 @@ PAPERLESS_LOGROTATE_MAX_BACKUPS=<num>
 
     Defaults to 20.
 
+.. _hosting-and-security:
+
 Hosting & Security
 ##################
 
@@ -142,7 +152,24 @@ PAPERLESS_SECRET_KEY=<key>
 
     Default is listed in the file ``src/paperless/settings.py``.
 
-PAPERLESS_ALLOWED_HOSTS<comma-separated-list>
+PAPERLESS_URL=<url>
+    This setting can be used to set the three options below (ALLOWED_HOSTS,
+    CORS_ALLOWED_HOSTS and CSRF_TRUSTED_ORIGINS). If the other options are
+    set the values will be combined with this one. Do not include a trailing
+    slash. E.g. https://paperless.domain.com
+
+    Defaults to empty string, leaving the other settings unaffected.
+
+PAPERLESS_CSRF_TRUSTED_ORIGINS=<comma-separated-list>
+    A list of trusted origins for unsafe requests (e.g. POST). As of Django 4.0
+    this is required to access the Django admin via the web.
+    See https://docs.djangoproject.com/en/4.0/ref/settings/#csrf-trusted-origins
+
+    Can also be set using PAPERLESS_URL (see above).
+
+    Defaults to empty string, which does not add any origins to the trusted list.
+
+PAPERLESS_ALLOWED_HOSTS=<comma-separated-list>
     If you're planning on putting Paperless on the open internet, then you
     really should set this value to the domain name you're using.  Failing to do
     so leaves you open to HTTP host header attacks:
@@ -151,12 +178,19 @@ PAPERLESS_ALLOWED_HOSTS<comma-separated-list>
     Just remember that this is a comma-separated list, so "example.com" is fine,
     as is "example.com,www.example.com", but NOT " example.com" or "example.com,"
 
+    Can also be set using PAPERLESS_URL (see above).
+
+    If manually set, please remember to include "localhost". Otherwise docker
+    healthcheck will fail.
+
     Defaults to "*", which is all hosts.
 
-PAPERLESS_CORS_ALLOWED_HOSTS<comma-separated-list>
+PAPERLESS_CORS_ALLOWED_HOSTS=<comma-separated-list>
     You need to add your servers to the list of allowed hosts that can do CORS
     calls. Set this to your public domain name.
 
+    Can also be set using PAPERLESS_URL (see above).
+
     Defaults to "http://localhost:8000".
 
 PAPERLESS_FORCE_SCRIPT_NAME=<path>
@@ -185,7 +219,7 @@ PAPERLESS_AUTO_LOGIN_USERNAME=<username>
 PAPERLESS_ADMIN_USER=<username>
     If this environment variable is specified, Paperless automatically creates
     a superuser with the provided username at start. This is useful in cases
-    where you can not run the `createsuperuser` command seperately, such as Kubernetes
+    where you can not run the `createsuperuser` command separately, such as Kubernetes
     or AWS ECS.
 
     Requires `PAPERLESS_ADMIN_PASSWORD` to be set.
@@ -390,14 +424,23 @@ PAPERLESS_OCR_IMAGE_DPI=<num>
     the produced PDF documents are A4 sized.
 
 PAPERLESS_OCR_MAX_IMAGE_PIXELS=<num>
-    Paperless will not OCR images that have more pixels than this limit.
-    This is intended to prevent decompression bombs from overloading paperless.
-    Increasing this limit is desired if you face a DecompressionBombError despite
-    the concerning file not being malicious; this could e.g. be caused by invalidly
-    recognized metadata.
-    If you have enough resources or if you are certain that your uploaded files
-    are not malicious you can increase this value to your needs.
-    The default value is 256000000, an image with more pixels than that would not be parsed.
+    Paperless will raise a warning when OCRing images which are over this limit and
+    will not OCR images which are more than twice this limit.  Note this does not
+    prevent the document from being consumed, but could result in missing text content.
+
+    If unset, will default to the value determined by
+    `Pillow <https://pillow.readthedocs.io/en/stable/reference/Image.html#PIL.Image.MAX_IMAGE_PIXELS>`_.
+
+    .. note::
+
+        Increasing this limit could cause Paperless to consume additional resources
+        when consuming a file.  Be sure you have sufficient system resources.
+
+    .. caution::
+
+        The limit is intended to prevent malicious files from consuming system resources
+        and causing crashes and other errors.  Only increase this value if you are certain
+        your documents are not malicious and you need the text which was not OCRed
 
 PAPERLESS_OCR_USER_ARGS=<json>
     OCRmyPDF offers many more options. Use this parameter to specify any
@@ -448,7 +491,7 @@ PAPERLESS_TIKA_GOTENBERG_ENDPOINT=<url>
     Defaults to "http://localhost:3000".
 
 If you run paperless on docker, you can add those services to the docker-compose
-file (see the provided ``docker-compose.tika.yml`` file for reference). The changes
+file (see the provided ``docker-compose.sqlite-tika.yml`` file for reference). The changes
 requires are as follows:
 
 .. code:: yaml
@@ -469,14 +512,14 @@ requires are as follows:
         # ...
 
         gotenberg:
-            image: gotenberg/gotenberg:7
+            image: gotenberg/gotenberg:7.4
             restart: unless-stopped
             command:
                 - "gotenberg"
                 - "--chromium-disable-routes=true"
 
         tika:
-            image: apache/tika
+            image: ghcr.io/paperless-ngx/tika:latest
             restart: unless-stopped
 
 Add the configuration variables to the environment of the webserver (alternatively
@@ -493,6 +536,8 @@ PAPERLESS_TASK_WORKERS=<num>
     maintain the automatic matching algorithm, check emails, consume documents,
     etc. This variable specifies how many things it will do in parallel.
 
+    Defaults to 1
+
 
 PAPERLESS_THREADS_PER_WORKER=<num>
     Furthermore, paperless uses multiple threads when consuming documents to
@@ -540,6 +585,10 @@ PAPERLESS_WORKER_TIMEOUT=<num>
     large documents within the default 1800 seconds. So extending this timeout
     may prove to be useful on weak hardware setups.
 
+PAPERLESS_WORKER_RETRY=<num>
+    If PAPERLESS_WORKER_TIMEOUT has been configured, the retry time for a task can
+    also be configured.  By default, this value will be set to 10s more than the
+    worker timeout.  This value should never be set less than the worker timeout.
 
 PAPERLESS_TIME_ZONE=<timezone>
     Set the time zone here.
@@ -560,6 +609,28 @@ PAPERLESS_CONSUMER_POLLING=<num>
 
     Defaults to 0, which disables polling and uses filesystem notifications.
 
+PAPERLESS_CONSUMER_POLLING_RETRY_COUNT=<num>
+    If consumer polling is enabled, sets the number of times paperless will check for a
+    file to remain unmodified.
+
+    Defaults to 5.
+
+PAPERLESS_CONSUMER_POLLING_DELAY=<num>
+    If consumer polling is enabled, sets the delay in seconds between each check (above) paperless
+    will do while waiting for a file to remain unmodified.
+
+    Defaults to 5.
+
+.. _configuration-inotify:
+
+PAPERLESS_CONSUMER_INOTIFY_DELAY=<num>
+    Sets the time in seconds the consumer will wait for additional events
+    from inotify before the consumer will consider a file ready and begin consumption.
+    Certain scanners or network setups may generate multiple events for a single file,
+    leading to multiple consumers working on the same file.  Configure this to
+    prevent that.
+
+    Defaults to 0.5 seconds.
 
 PAPERLESS_CONSUMER_DELETE_DUPLICATES=<bool>
     When the consumer detects a duplicate document, it will not touch the
@@ -588,6 +659,37 @@ PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=<bool>
 
     Defaults to false.
 
+PAPERLESS_CONSUMER_ENABLE_BARCODES=<bool>
+    Enables the scanning and page separation based on detected barcodes.
+    This allows for scanning and adding multiple documents per uploaded
+    file, which are separated by one or multiple barcode pages.
+
+    For ease of use, it is suggested to use a standardized separation page,
+    e.g. `here <https://www.alliancegroup.co.uk/patch-codes.htm>`_.
+
+    If no barcodes are detected in the uploaded file, no page separation
+    will happen.
+
+    The original document will be removed and the separated pages will be
+    saved as pdf.
+
+    Defaults to false.
+
+PAPERLESS_CONSUMER_BARCODE_TIFF_SUPPORT=<bool>
+    Whether TIFF image files should be scanned for barcodes.
+    This will automatically convert any TIFF image(s) to pdfs for later
+    processing.
+    This only has an effect, if PAPERLESS_CONSUMER_ENABLE_BARCODES has been
+    enabled.
+
+    Defaults to false.
+
+PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
+  Defines the string to be detected as a separator barcode.
+  If paperless is used with the PATCH-T separator pages, users
+  shouldn't change this.
+
+  Defaults to "PATCHT"
 
 PAPERLESS_CONVERT_MEMORY_LIMIT=<num>
     On smaller systems, or even in the case of Very Large Documents, the consumer
@@ -612,13 +714,6 @@ PAPERLESS_CONVERT_TMPDIR=<path>
 
     Default is none, which disables the temporary directory.
 
-PAPERLESS_OPTIMIZE_THUMBNAILS=<bool>
-    Use optipng to optimize thumbnails. This usually reduces the size of
-    thumbnails by about 20%, but uses considerable compute time during
-    consumption.
-
-    Defaults to true.
-
 PAPERLESS_POST_CONSUME_SCRIPT=<filename>
     After a document is consumed, Paperless can trigger an arbitrary script if
     you like.  This script will be passed a number of arguments for you to work
@@ -634,6 +729,9 @@ PAPERLESS_FILENAME_DATE_ORDER=<format>
     The filename will be checked first, and if nothing is found, the document
     text will be checked as normal.
 
+    A date in a filename must have some separators (`.`, `-`, `/`, etc)
+    for it to be parsed.
+
     Defaults to none, which disables this feature.
 
 PAPERLESS_THUMBNAIL_FONT_NAME=<filename>
@@ -651,10 +749,7 @@ PAPERLESS_IGNORE_DATES=<string>
     this process. This is useful for special dates (like date of birth) that appear
     in documents regularly but are very unlikely to be the documents creation date.
 
-    You may specify dates in a multitude of formats supported by dateparser (see
-    https://dateparser.readthedocs.io/en/latest/#popular-formats) but as the dates
-    need to be comma separated, the options are limited.
-    Example: "2020-12-02,22.04.1999"
+    The date is parsed using the order specified in PAPERLESS_DATE_ORDER
 
     Defaults to an empty string to not ignore any dates.
 
@@ -671,7 +766,7 @@ PAPERLESS_CONSUMER_IGNORE_PATTERNS=<json>
 
     This can be adjusted by configuring a custom json array with patterns to exclude.
 
-    Defautls to ``[".DS_STORE/*", "._*", ".stfolder/*"]``.
+    Defaults to ``[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]``.
 
 Binaries
 ########
@@ -689,9 +784,6 @@ PAPERLESS_CONVERT_BINARY=<path>
 PAPERLESS_GS_BINARY=<path>
     Defaults to "/usr/bin/gs".
 
-PAPERLESS_OPTIPNG_BINARY=<path>
-    Defaults to "/usr/bin/optipng".
-
 
 .. _configuration-docker:
 
@@ -707,9 +799,7 @@ PAPERLESS_WEBSERVER_WORKERS=<num>
     also loads the entire application into memory separately, so increasing this value
     will increase RAM usage.
 
-    Consider configuring this to 1 on low power devices with limited amount of RAM.
-
-    Defaults to 2.
+    Defaults to 1.
 
 PAPERLESS_PORT=<port>
     The port number the webserver will listen on inside the container. There are
@@ -764,3 +854,26 @@ PAPERLESS_OCR_LANGUAGES=<list>
         PAPERLESS_OCR_LANGUAGE=tur
 
     Defaults to none, which does not install any additional languages.
+
+
+.. _configuration-update-checking:
+
+Update Checking
+###############
+
+PAPERLESS_ENABLE_UPDATE_CHECK=<bool>
+    Enable (or disable) the automatic check for available updates. This feature is disabled
+    by default but if it is not explicitly set Paperless-ngx will show a message about this.
+
+    If enabled, the feature works by pinging the the Github API for the latest release e.g.
+    https://api.github.com/repos/paperless-ngx/paperless-ngx/releases/latest
+    to determine whether a new version is available.
+
+    Actual updating of the app must still be performed manually.
+
+    Note that for users of thirdy-party containers e.g. linuxserver.io this notification
+    may be 'ahead' of a new release from the third-party maintainers.
+
+    In either case, no tracking data is collected by the app in any way.
+
+    Defaults to none, which disables the feature.

docs/extending.rst

@@ -334,11 +334,17 @@ directory.
 Building the Docker image
 =========================
 
+The docker image is primarily built by the GitHub actions workflow, but it can be
+faster when developing to build and tag an image locally.
+
+To provide the build arguments automatically, build the image using the helper
+script ``build-docker-image.sh``.
+
 Building the docker image from source:
 
     .. code:: shell-session
 
-        docker build . -t <your-tag>
+        ./build-docker-image.sh Dockerfile -t <your-tag>
 
 Extending Paperless
 ===================

docs/faq.rst

@@ -7,7 +7,7 @@ Frequently asked questions
 
 **A:** While Paperless-ngx is already considered largely "feature-complete" it is a community-driven
 project and development will be guided in this way. New features can be submitted via
-GitHub discussions and "up-voted" by the community but this is not a garauntee the feature
+GitHub discussions and "up-voted" by the community but this is not a guarantee the feature
 will be implemented. This project will always be open to collaboration in the form of PRs,
 ideas etc.
 

docs/index.rst

@@ -52,7 +52,7 @@ resources in the documentation:
 *   Paperless is now integrated with a
     :ref:`task processing queue <setup-task_processor>` that tells you
     at a glance when and why something is not working.
-*   The :ref:`changelog <paperless_changelog>` contains a detailed list of all changes
+*   The :doc:`changelog </changelog>` contains a detailed list of all changes
     in paperless-ngx.
 
 Contents

docs/requirements.txt

@@ -0,0 +1 @@
+myst-parser==0.17.2

docs/scanners.rst

@@ -13,43 +13,45 @@ that works right for you based on recommendations from other Paperless users.
 Physical scanners
 =================
 
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brand   | Model          | Supports                          | Recommended By |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-|         |                | FTP | NFS | SMB | SMTP | API [1]_ |                |
-+=========+================+=====+=====+=====+======+==========+================+
-| Brother | `ADS-1700W`_   | yes |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-1600W`_   | yes |     | yes | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-1500W`_   | yes |     | yes | yes  |          |`danielquinn`_  |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-1100W`_   | yes |     |     |      |          |`ytzelf`_       |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `ADS-2800W`_   | yes | yes |     | yes  | yes      |`philpagel`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-J6930DW`_ | yes |     |     |      |          |`ayounggun`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-L5850DW`_ | yes |     |     | yes  |          |`holzhannes`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-L2750DW`_ | yes |     | yes | yes  |          |`muued`_        |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-J5910DW`_ | yes |     |     |      |          |`bmsleight`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-8950DW`_  | yes |     |     | yes  | yes      |`philpagel`_    |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Brother | `MFC-9142CDN`_ | yes |     | yes |      |          |`REOLDEV`_      |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Fujitsu | `ix500`_       | yes |     | yes |      |          |`eonist`_       |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Epson   | `ES-580W`_     | yes |     | yes | yes  |          |`fignew`_       |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Epson   | `WF-7710DWF`_  | yes |     | yes |      |          |`Skylinar`_     |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Fujitsu | `S1300i`_      | yes |     | yes |      |          |`jonaswinkler`_ |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
-| Doxie   | `Q2`_          |     |     |     |      | yes      |`Unkn0wnCat`_   |
-+---------+----------------+-----+-----+-----+------+----------+----------------+
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brand   | Model             | Supports                                      | Recommended By |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+|         |                   | FTP | SFTP | NFS | SMB      | SMTP | API [1]_ |                |
++=========+===================+=====+======+=====+==========+======+==========+================+
+| Brother | `ADS-1700W`_      | yes |      |     | yes      | yes  |          |`holzhannes`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-1600W`_      | yes |      |     | yes      | yes  |          |`holzhannes`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-1500W`_      | yes |      |     | yes      | yes  |          |`danielquinn`_  |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-1100W`_      | yes |      |     |          |      |          |`ytzelf`_       |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `ADS-2800W`_      | yes | yes  |     | yes      | yes  |          |`philpagel`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-J6930DW`_    | yes |      |     |          |      |          |`ayounggun`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-L5850DW`_    | yes |      |     |          | yes  |          |`holzhannes`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-L2750DW`_    | yes |      |     | yes      | yes  |          |`muued`_        |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-J5910DW`_    | yes |      |     |          |      |          |`bmsleight`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-8950DW`_     | yes |      |     | yes      | yes  |          |`philpagel`_    |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Brother | `MFC-9142CDN`_    | yes |      |     | yes      |      |          |`REOLDEV`_      |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Canon   | `Maxify MB 5350`_ |     |      |     | yes [2]_ | yes  |          |`eingemaischt`_ |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Fujitsu | `ix500`_          | yes |      |     | yes      |      |          |`eonist`_       |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Epson   | `ES-580W`_        | yes |      |     | yes      | yes  |          |`fignew`_       |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Epson   | `WF-7710DWF`_     | yes |      |     | yes      |      |          |`Skylinar`_     |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Fujitsu | `S1300i`_         | yes |      |     | yes      |      |          |`jonaswinkler`_ |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
+| Doxie   | `Q2`_             |     |      |     |          |      | yes      |`Unkn0wnCat`_   |
++---------+-------------------+-----+------+-----+----------+------+----------+----------------+
 
 .. _MFC-L5850DW: https://www.brother-usa.com/products/mfcl5850dw
 .. _MFC-L2750DW: https://www.brother.de/drucker/laserdrucker/mfc-l2750dw
@@ -58,6 +60,7 @@ Physical scanners
 .. _ADS-1500W: https://www.brother.ca/en/p/ads1500w
 .. _ADS-1100W: https://support.brother.com/g/b/downloadtop.aspx?c=fr&lang=fr&prod=ads1100w_eu_as_cn
 .. _ADS-2800W: https://www.brother-usa.com/products/ads2800w
+.. _Maxify MB 5350: https://www.canon.de/printers/inkjet/maxify/maxify_mb5350/specification.html
 .. _MFC-J6930DW: https://www.brother.ca/en/p/MFCJ6930DW
 .. _MFC-J5910DW: https://www.brother.co.uk/printers/inkjet-printers/mfcj5910dw
 .. _MFC-8950DW: https://www.brother-usa.com/products/mfc8950dw
@@ -81,8 +84,11 @@ Physical scanners
 .. _Unkn0wnCat: https://github.com/Unkn0wnCat
 .. _muued: https://github.com/muued
 .. _philpagel: https://github.com/philpagel
+.. _eingemaischt: https://github.com/eingemaischt
 
 .. [1] Scanners with API Integration allow to push scanned documents directly to :ref:`Paperless API <api-file_uploads>`, sometimes referred to as Webhook or Document POST.
+.. [2] Canon Multi Function Printers show strange behavior over SMB. They close and reopen the file after every page. It's recommended to tune the
+       :ref:`polling <configuration-polling>` and :ref:`inotify <configuration-inotify>` configuration values for your scanner. The scanner timeout is 3 minutes, so ``180`` is a good starting point.
 
 Mobile phone software
 =====================
@@ -105,6 +111,9 @@ You can use your phone to "scan" documents. The regular camera app will work, bu
 
 On Android, you can use these applications in combination with one of the :ref:`Paperless-ngx compatible apps <usage-mobile_upload>` to "Share" the documents produced by these scanner apps with paperless. On iOS, you can share the scanned documents via iOS-Sharing to other mail, WebDav or FTP apps.
 
+There is also an iOS Shortcut that allows you to directly upload text, PDF and image documents available here: https://www.icloud.com/shortcuts/d234abc0885040129d9d75fa45fe1154
+Please note this only works for documents downloaded to iCloud / the device, in other words not directly from a URL.
+
 .. _Office Lens: https://play.google.com/store/apps/details?id=com.microsoft.office.officelens
 .. _Genius Scan: https://play.google.com/store/apps/details?id=com.thegrizzlylabs.geniusscan.free
 .. _OCR Scanner - QuickScan: https://apps.apple.com/us/app/quickscan-scanner-text-ocr/id1513790291
@@ -131,4 +140,3 @@ This part assumes your Doxie is connected to WiFi and you know its IP.
 6. Click *Submit* at the bottom of the page
 
 Congrats, you can now scan directly from your Doxie to your Paperless-ngx instance!
-

docs/screenshots.rst

@@ -4,41 +4,60 @@
 Screenshots
 ***********
 
-This is what paperless-ngx looks like. You shouldn't use paperless to index
-research papers though, its a horrible tool for that job.
+This is what Paperless-ngx looks like.
 
 The dashboard shows customizable views on your document and allows document uploads:
 
 .. image:: _static/screenshots/dashboard.png
+    :target: _static/screenshots/dashboard.png
 
 The document list provides three different styles to scroll through your documents:
 
 .. image:: _static/screenshots/documents-table.png
+    :target: _static/screenshots/documents-table.png
 .. image:: _static/screenshots/documents-smallcards.png
+    :target: _static/screenshots/documents-smallcards.png
 .. image:: _static/screenshots/documents-largecards.png
+    :target: _static/screenshots/documents-largecards.png
+
+Paperless-ngx also supports "dark mode":
+
+.. image:: _static/screenshots/documents-smallcards-dark.png
+    :target: _static/screenshots/documents-smallcards-dark.png
 
 Extensive filtering mechanisms:
 
 .. image:: _static/screenshots/documents-filter.png
+    :target: _static/screenshots/documents-filter.png
 
-Side-by-side editing of documents. Optimized for 1080p.
+Bulk editing of document tags, correspondents, etc.:
+
+.. image:: _static/screenshots/bulk-edit.png
+    :target: _static/screenshots/bulk-edit.png
+
+Side-by-side editing of documents:
 
 .. image:: _static/screenshots/editing.png
+    :target: _static/screenshots/editing.png
 
 Tag editing. This looks about the same for correspondents and document types.
 
 .. image:: _static/screenshots/new-tag.png
+    :target: _static/screenshots/new-tag.png
 
 Searching provides auto complete and highlights the results.
 
 .. image:: _static/screenshots/search-preview.png
+    :target: _static/screenshots/search-preview.png
 .. image:: _static/screenshots/search-results.png
+    :target: _static/screenshots/search-results.png
 
 Fancy mail filters!
 
 .. image:: _static/screenshots/mail-rules-edited.png
+    :target: _static/screenshots/mail-rules-edited.png
 
-Mobile support in the future? This kinda works, however some layouts are still
-too wide.
+Mobile devices are supported.
 
 .. image:: _static/screenshots/mobile.png
+    :target: _static/screenshots/mobile.png

docs/setup.rst

@@ -184,6 +184,25 @@ Install Paperless from Docker Hub
     port 8000. Modifying the part before the colon will map requests on another
     port to the webserver running on the default port.
 
+    **Rootless**
+
+    If you want to run Paperless as a rootless container, you will need to do the
+    following in your ``docker-compose.yml``:
+
+    - set the ``user`` running the container to map to the ``paperless`` user in the
+      container.
+      This value (``user_id`` below), should be the same id that ``USERMAP_UID`` and
+      ``USERMAP_GID`` are set to in the next step.
+      See ``USERMAP_UID`` and ``USERMAP_GID`` :ref:`here <configuration-docker>`.
+
+    Your entry for Paperless should contain something like:
+
+     .. code::
+
+        webserver:
+          image: ghcr.io/paperless-ngx/paperless-ngx:latest
+          user: <user_id>
+
 5.  Modify ``docker-compose.env``, following the comments in the file. The
     most important change is to set ``USERMAP_UID`` and ``USERMAP_GID``
     to the uid and gid of your user on the host system. Use ``id -u`` and
@@ -200,6 +219,19 @@ Install Paperless from Docker Hub
         You can copy any setting from the file ``paperless.conf.example`` and paste it here.
         Have a look at :ref:`configuration` to see what's available.
 
+    .. note::
+
+        You can utilize Docker secrets for some configuration settings by
+        appending `_FILE` to some configuration values.  This is supported currently
+        only by:
+          * PAPERLESS_DBUSER
+          * PAPERLESS_DBPASS
+          * PAPERLESS_SECRET_KEY
+          * PAPERLESS_AUTO_LOGIN_USERNAME
+          * PAPERLESS_ADMIN_USER
+          * PAPERLESS_ADMIN_MAIL
+          * PAPERLESS_ADMIN_PASSWORD
+
     .. caution::
 
         Some file systems such as NFS network shares don't support file system
@@ -286,17 +318,18 @@ writing. Windows is not and will never be supported.
 
     *   ``fonts-liberation`` for generating thumbnails for plain text files
     *   ``imagemagick`` >= 6 for PDF conversion
-    *   ``optipng`` for optimizing thumbnails
     *   ``gnupg`` for handling encrypted documents
     *   ``libpq-dev`` for PostgreSQL
     *   ``libmagic-dev`` for mime type detection
     *   ``mime-support`` for mime type detection
+    *   ``libzbar0`` for barcode detection
+    *   ``poppler-utils`` for barcode detection
 
     Use this list for your preferred package management:
 
     .. code::
 
-        python3 python3-pip python3-dev imagemagick fonts-liberation optipng gnupg libpq-dev libmagic-dev mime-support
+        python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev libmagic-dev mime-support libzbar0 poppler-utils
 
     These dependencies are required for OCRmyPDF, which is used for text recognition.
 
@@ -330,6 +363,12 @@ writing. Windows is not and will never be supported.
 3.  Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish
     to use PostgreSQL, SQLite is available as well.
 
+    .. note::
+
+        On bare-metal installations using SQLite, ensure the
+        `JSON1 extension <https://code.djangoproject.com/wiki/JSON1Extension>`_ is enabled. This is
+        usually the case, but not always.
+
 4.  Get the release archive from `<https://github.com/paperless-ngx/paperless-ngx/releases>`_.
     If you clone the git repo as it is, you also have to compile the front end by yourself.
     Extract the archive to a place from where you wish to execute it, such as ``/opt/paperless``.
@@ -345,6 +384,8 @@ writing. Windows is not and will never be supported.
         paperless stores its data. If you like, you can point both to the same directory.
     *   ``PAPERLESS_SECRET_KEY`` should be a random sequence of characters. It's used for authentication. Failure
         to do so allows third parties to forge authentication credentials.
+    *   ``PAPERLESS_URL`` if you are behind a reverse proxy. This should point to your domain. Please see
+        :ref:`configuration` for more information.
 
     Many more adjustments can be made to paperless, especially the OCR part. The following options are recommended
     for everyone:
@@ -509,7 +550,7 @@ how you installed paperless.
 This setup describes how to update an existing paperless Docker installation.
 The important things to keep in mind are as follows:
 
-* Read the :ref:`changelog <paperless_changelog>` and take note of breaking changes.
+* Read the :doc:`changelog </changelog>` and take note of breaking changes.
 * You should decide if you want to stick with SQLite or want to migrate your database
   to PostgreSQL. See :ref:`setup-sqlite_to_psql` for details on how to move your data from
   SQLite to PostgreSQL. Both work fine with paperless. However, if you already have a
@@ -720,12 +761,8 @@ configuring some options in paperless can help improve performance immensely:
 *   If you want to perform OCR on the device, consider using ``PAPERLESS_OCR_CLEAN=none``.
     This will speed up OCR times and use less memory at the expense of slightly worse
     OCR results.
-*   Set ``PAPERLESS_OPTIMIZE_THUMBNAILS`` to 'false' if you want faster consumption
-    times. Thumbnails will be about 20% larger.
 *   If using docker, consider setting ``PAPERLESS_WEBSERVER_WORKERS`` to
     1. This will save some memory.
-*   Use the arm compatible docker-compose if you're wanting to use Tika on something like
-		a raspberry pi. The official apache/tika image does not support the arm architecture.
 
 For details, refer to :ref:`configuration`.
 
@@ -784,4 +821,6 @@ the following configuration is required for paperless to operate:
         }
     }
 
+The ``PAPERLESS_URL`` configuration variable is also required when using a reverse proxy. Please refer to the :ref:`hosting-and-security` docs.
+
 Also read `this <https://channels.readthedocs.io/en/stable/deploying.html#nginx-supervisor-ubuntu>`__, towards the end of the section.

docs/troubleshooting.rst

@@ -125,7 +125,7 @@ If using docker-compose, this is achieved by the following configuration change
 .. code:: yaml
 
     gotenberg:
-        image: gotenberg/gotenberg:7
+        image: gotenberg/gotenberg:7.4
         restart: unless-stopped
         command:
             - "gotenberg"
@@ -235,3 +235,69 @@ You might find messages like these in your log files:
 This indicates that paperless failed to read PDF metadata from one of your documents. This happens when you
 open the affected documents in paperless for editing. Paperless will continue to work, and will simply not
 show the invalid metadata.
+
+Consumer fails with a FileNotFoundError
+#######################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.consumer] Error while consuming document SCN_0001.pdf: FileNotFoundError: [Errno 2] No such file or directory: '/tmp/ocrmypdf.io.yhk3zbv0/origin.pdf'
+    Traceback (most recent call last):
+      File "/app/paperless/src/paperless_tesseract/parsers.py", line 261, in parse
+        ocrmypdf.ocr(**args)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/api.py", line 337, in ocr
+        return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 385, in run_pipeline
+        exec_concurrent(context, executor)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 302, in exec_concurrent
+        pdf = post_process(pdf, context, executor)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_sync.py", line 235, in post_process
+        pdf_out = metadata_fixup(pdf_out, context)
+      File "/usr/local/lib/python3.8/dist-packages/ocrmypdf/_pipeline.py", line 798, in metadata_fixup
+        with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
+      File "/usr/local/lib/python3.8/dist-packages/pikepdf/_methods.py", line 923, in open
+        pdf = Pdf._open(
+    FileNotFoundError: [Errno 2] No such file or directory: '/tmp/ocrmypdf.io.yhk3zbv0/origin.pdf'
+
+This probably indicates paperless tried to consume the same file twice.  This can happen for a number of reasons,
+depending on how documents are placed into the consume folder.  If paperless is using inotify (the default) to
+check for documents, try adjusting the :ref:`inotify configuration <configuration-inotify>`.  If polling is enabled,
+try adjusting the :ref:`polling configuration <configuration-polling>`.
+
+Consumer fails waiting for file to remain unmodified.
+#####################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [ERROR] [paperless.management.consumer] Timeout while waiting on file /usr/src/paperless/src/../consume/SCN_0001.pdf to remain unmodified.
+
+This indicates paperless timed out while waiting for the file to be completely written to the consume folder.
+Adjusting :ref:`polling configuration <configuration-polling>` values should resolve the issue.
+
+.. note::
+
+    The user will need to manually move the file out of the consume folder and
+    back in, for the initial failing file to be consumed.
+
+Consumer fails reporting "OS reports file as busy still".
+#########################################################
+
+You might find messages like these in your log files:
+
+.. code::
+
+    [WARNING] [paperless.management.consumer] Not consuming file /usr/src/paperless/src/../consume/SCN_0001.pdf: OS reports file as busy still
+
+This indicates paperless was unable to open the file, as the OS reported the file as still being in use.  To prevent a
+crash, paperless did not try to consume the file.  If paperless is using inotify (the default) to
+check for documents, try adjusting the :ref:`inotify configuration <configuration-inotify>`.  If polling is enabled,
+try adjusting the :ref:`polling configuration <configuration-polling>`.
+
+.. note::
+
+    The user will need to manually move the file out of the consume folder and
+    back in, for the initial failing file to be consumed.

docs/usage_overview.rst

@@ -62,7 +62,7 @@ your documents:
 
 1.  OCR the document, if it has no text. Digital documents usually have text,
     and this step will be skipped for those documents.
-2.  Paperless will create an archiveable PDF/A document from your document.
+2.  Paperless will create an archivable PDF/A document from your document.
     If this document is coming from your scanner, it will have embedded selectable text.
 3.  Paperless performs automatic matching of tags, correspondents and types on the
     document before storing it in the database.
@@ -102,12 +102,14 @@ files from the scanner.  Typically, you're looking at an FTP server like
 
 .. TODO: hyperref to configuration of the location of this magic folder.
 
-Dashboard upload
-================
+Web UI Upload
+=============
 
 The dashboard has a file drop field to upload documents to paperless. Simply drag a file
 onto this field or select a file with the file dialog. Multiple files are supported.
 
+You can also upload documents on any other page of the web UI by dragging-and-dropping
+files into your browser window.
 
 .. _usage-mobile_upload:
 
@@ -159,6 +161,9 @@ These are as follows:
     will not consume flagged mails.
 *   **Move to folder:** Moves consumed mails out of the way so that paperless wont
     consume them again.
+*   **Add custom Tag:** Adds a custom tag to mails with consumed documents (the IMAP
+    standard calls these "keywords"). Paperless will not consume mails already tagged.
+    Not all mail servers support this feature!
 
 .. caution::
 
@@ -178,6 +183,15 @@ These are as follows:
     automatically or manually and tell paperless to move them to yet another folder
     after consumption. It's up to you.
 
+.. note::
+
+    When defining a mail rule with a folder, you may need to try different characters to
+    define how the sub-folders are separated.  Common values include ".", "/" or "|", but
+    this varies by the mail server.  Check the documentation for your mail server.  In the
+    event of an error fetching mail from a certain folder, check the Paperless logs.  When
+    a folder is not located, Paperless will attempt to list all folders found in the account
+    to the Paperless logs.
+
 .. note::
 
     Paperless will process the rules in the order defined in the admin page.

gunicorn.conf.py

@@ -1,7 +1,7 @@
 import os
 
-bind = f'0.0.0.0:{os.getenv("PAPERLESS_PORT", 8000)}'
-workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 2))
+bind = f'[::]:{os.getenv("PAPERLESS_PORT", 8000)}'
+workers = int(os.getenv("PAPERLESS_WEBSERVER_WORKERS", 1))
 worker_class = "paperless.workers.ConfigurableWorker"
 timeout = 120
 
@@ -24,7 +24,7 @@ def worker_int(worker):
     ## get traceback info
     import threading, sys, traceback
 
-    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+    id2name = {th.ident: th.name for th in threading.enumerate()}
     code = []
     for threadId, stack in sys._current_frames().items():
         code.append("\n# Thread: %s(%d)" % (id2name.get(threadId, ""), threadId))

install-paperless-ngx.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ask() {
 	while true ; do
@@ -47,24 +47,29 @@ if [[ $(id -u) == "0" ]] ; then
 	exit 1
 fi
 
-if [[ -z $(which wget) ]] ; then
+if ! command -v wget &> /dev/null ; then
 	echo "wget executable not found. Is wget installed?"
 	exit 1
 fi
 
-if [[ -z $(which docker) ]] ; then
+if ! command -v docker &> /dev/null ; then
 	echo "docker executable not found. Is docker installed?"
 	exit 1
 fi
 
-if [[ -z $(which docker-compose) ]] ; then
-	echo "docker-compose executable not found. Is docker-compose installed?"
-	exit 1
+DOCKER_COMPOSE_CMD="docker-compose"
+if ! command -v ${DOCKER_COMPOSE_CMD} ; then
+	if docker compose version &> /dev/null ; then
+		DOCKER_COMPOSE_CMD="docker compose"
+	else
+		echo "docker-compose executable not found. Is docker-compose installed?"
+		exit 1
+	fi
 fi
 
 # Check if user has permissions to run Docker by trying to get the status of Docker (docker status).
 # If this fails, the user probably does not have permissions for Docker.
-if [ ! "$(docker stats --no-stream 2>/dev/null 1>&2)" ] ; then
+if ! docker stats --no-stream &> /dev/null ; then
 	echo ""
 	echo "WARN: It look like the current user does not have Docker permissions."
 	echo "WARN: Use 'sudo usermod -aG docker $USER' to assign Docker permissions to the user."
@@ -87,6 +92,14 @@ echo ""
 echo "1. Application configuration"
 echo "============================"
 
+echo ""
+echo "The URL paperless will be available at. This is required if the"
+echo "installation will be accessible via the web, otherwise can be left blank."
+echo ""
+
+ask "URL" ""
+URL=$ask_result
+
 echo ""
 echo "The port on which the paperless webserver will listen for incoming"
 echo "connections."
@@ -273,6 +286,7 @@ if [[ "$DATABASE_BACKEND" == "postgres" ]] ; then
 	fi
 fi
 echo ""
+echo "URL: $URL"
 echo "Port: $PORT"
 echo "Database: $DATABASE_BACKEND"
 echo "Tika enabled: $TIKA_ENABLED"
@@ -305,9 +319,15 @@ wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/
 
 SECRET_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 64 | head -n 1)
 
-DEFAULT_LANGUAGES="deu eng fra ita spa"
+DEFAULT_LANGUAGES=("deu eng fra ita spa")
+
+_split_langs="${OCR_LANGUAGE//+/ }"
+read -r -a OCR_LANGUAGES_ARRAY <<< "${_split_langs}"
 
 {
+	if [[ ! $URL == "" ]] ; then
+		echo "PAPERLESS_URL=$URL"
+	fi
 	if [[ ! $USERMAP_UID == "1000" ]] ; then
 		echo "USERMAP_UID=$USERMAP_UID"
 	fi
@@ -317,8 +337,8 @@ DEFAULT_LANGUAGES="deu eng fra ita spa"
 	echo "PAPERLESS_TIME_ZONE=$TIME_ZONE"
 	echo "PAPERLESS_OCR_LANGUAGE=$OCR_LANGUAGE"
 	echo "PAPERLESS_SECRET_KEY=$SECRET_KEY"
-	if [[ ! " ${DEFAULT_LANGUAGES[*]} " =~ ${OCR_LANGUAGE} ]] ; then
-		echo "PAPERLESS_OCR_LANGUAGES=$OCR_LANGUAGE"
+	if [[ ! ${DEFAULT_LANGUAGES[*]} =~ ${OCR_LANGUAGES_ARRAY[*]} ]] ; then
+		echo "PAPERLESS_OCR_LANGUAGES=${OCR_LANGUAGES_ARRAY[*]}"
 	fi
 } > docker-compose.env
 
@@ -351,8 +371,8 @@ if [ "$l1" -eq "$l2" ] ; then
 fi
 
 
-docker-compose pull
+${DOCKER_COMPOSE_CMD} pull
 
-docker-compose run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
+${DOCKER_COMPOSE_CMD} run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
 
-docker-compose up -d
+${DOCKER_COMPOSE_CMD} up -d

paperless.conf.example

@@ -23,12 +23,15 @@
 #PAPERLESS_MEDIA_ROOT=../media
 #PAPERLESS_STATICDIR=../static
 #PAPERLESS_FILENAME_FORMAT=
+#PAPERLESS_FILENAME_FORMAT_REMOVE_NONE=
 
 # Security and hosting
 
 #PAPERLESS_SECRET_KEY=change-me
-#PAPERLESS_ALLOWED_HOSTS=example.com,www.example.com
-#PAPERLESS_CORS_ALLOWED_HOSTS=http://example.com,http://localhost:8000
+#PAPERLESS_URL=https://example.com
+#PAPERLESS_CSRF_TRUSTED_ORIGINS=https://example.com # can be set using PAPERLESS_URL
+#PAPERLESS_ALLOWED_HOSTS=example.com,www.example.com # can be set using PAPERLESS_URL
+#PAPERLESS_CORS_ALLOWED_HOSTS=https://localhost:8080,https://example.com # can be set using PAPERLESS_URL
 #PAPERLESS_FORCE_SCRIPT_NAME=
 #PAPERLESS_STATIC_URL=/static/
 #PAPERLESS_AUTO_LOGIN_USERNAME=
@@ -58,15 +61,17 @@
 #PAPERLESS_CONSUMER_POLLING=10
 #PAPERLESS_CONSUMER_DELETE_DUPLICATES=false
 #PAPERLESS_CONSUMER_RECURSIVE=false
-#PAPERLESS_CONSUMER_IGNORE_PATTERNS=[".DS_STORE/*", "._*", ".stfolder/*"]
+#PAPERLESS_CONSUMER_IGNORE_PATTERNS=[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini"]
 #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
-#PAPERLESS_OPTIMIZE_THUMBNAILS=true
+#PAPERLESS_CONSUMER_ENABLE_BARCODES=false
+#PAPERLESS_CONSUMER_ENABLE_BARCODES=PATCHT
 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_FILENAME_DATE_ORDER=YMD
 #PAPERLESS_FILENAME_PARSE_TRANSFORMS=[]
 #PAPERLESS_THUMBNAIL_FONT_NAME=
 #PAPERLESS_IGNORE_DATES=
+#PAPERLESS_ENABLE_UPDATE_CHECK=
 
 # Tika settings
 
@@ -78,4 +83,3 @@
 
 #PAPERLESS_CONVERT_BINARY=/usr/bin/convert
 #PAPERLESS_GS_BINARY=/usr/bin/gs
-#PAPERLESS_OPTIPNG_BINARY=/usr/bin/optipng

requirements.txt

@@ -1,43 +1,37 @@
-#
-# These requirements were autogenerated by pipenv
-# To regenerate from the project's Pipfile, run:
-#
-#    pipenv lock --requirements
-#
-
 -i https://pypi.python.org/simple
 --extra-index-url https://www.piwheels.org/simple
 aioredis==1.3.1
-anyio==3.5.0; python_full_version >= '3.6.2'
+anyio==3.6.1; python_full_version >= '3.6.2'
 arrow==1.2.2; python_version >= '3.6'
-asgiref==3.5.0; python_version >= '3.7'
+asgiref==3.5.2; python_version >= '3.7'
 async-timeout==4.0.2; python_version >= '3.6'
 attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-autobahn==22.2.2; python_version >= '3.7'
+autobahn==22.6.1; python_version >= '3.7'
 automat==20.2.0
 backports.zoneinfo==0.2.1; python_version < '3.9'
 blessed==1.19.1; python_version >= '2.7'
-certifi==2021.10.8
-cffi==1.15.0
+certifi==2022.6.15; python_version >= '3.6'
+cffi==1.15.1
+channels==3.0.5
 channels-redis==3.4.0
-channels==3.0.4
-chardet==4.0.0; python_version >= '3.1'
-charset-normalizer==2.0.12; python_version >= '3'
-click==8.0.4; python_version >= '3.6'
+charset-normalizer==2.1.0; python_version >= '3.6'
+click==8.1.3; python_version >= '3.7'
 coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 concurrent-log-handler==0.9.20
 constantly==15.1.0
-cryptography==36.0.2; python_version >= '3.6'
+cryptography==37.0.4; python_version >= '3.6'
 daphne==3.0.2; python_version >= '3.6'
 dateparser==1.1.1
-django-cors-headers==3.11.0
-django-extensions==3.1.5
-django-filter==21.1
-django-picklefield==3.0.1; python_version >= '3'
-django-q==1.3.9
-django==4.0.3
+deprecated==1.2.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+deprecation==2.1.0
+django==4.0.6
+django-cors-headers==3.13.0
+django-extensions==3.2.0
+django-filter==22.1
+django-picklefield==3.1; python_version >= '3'
+-e git+https://github.com/paperless-ngx/django-q.git@bf20d57f859a7d872d5979cd8879fac9c9df981c#egg=django-q
 djangorestframework==3.13.1
-filelock==3.6.0
+filelock==3.7.1
 fuzzywuzzy[speedup]==0.18.0
 gunicorn==20.1.0
 h11==0.13.0; python_version >= '3.6'
@@ -46,66 +40,69 @@ httptools==0.4.0
 humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 hyperlink==21.0.0
 idna==3.3; python_version >= '3.5'
-imap-tools==0.53.0
-img2pdf==0.4.3
-importlib-resources==5.6.0; python_version < '3.9'
+imap-tools==0.56.0
+img2pdf==0.4.4
+importlib-resources==5.8.0; python_version < '3.9'
 incremental==21.3.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
 joblib==1.1.0; python_version >= '3.6'
 langdetect==1.0.9
-lxml==4.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-msgpack==1.0.3
-numpy==1.22.3; python_version >= '3.8'
-ocrmypdf==13.4.1
+lxml==4.9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+msgpack==1.0.4
+numpy==1.23.1; python_version >= '3.8'
+ocrmypdf==13.6.0
 packaging==21.3; python_version >= '3.6'
 pathvalidate==2.5.0
-pdfminer.six==20211012
-pikepdf==5.1.0
-pillow==9.0.1
+pdf2image==1.16.0
+pdfminer.six==20220524
+pikepdf==5.3.1
+pillow==9.2.0
 pluggy==1.0.0; python_version >= '3.6'
-portalocker==2.4.0; python_version >= '3'
+portalocker==2.5.1; python_version >= '3'
 psycopg2==2.9.3
-pyasn1-modules==0.2.8
 pyasn1==0.4.8
+pyasn1-modules==0.2.8
 pycparser==2.21
 pyopenssl==22.0.0
-pyparsing==3.0.7; python_version >= '3.6'
+pyparsing==3.0.9; python_full_version >= '3.6.8'
 python-dateutil==2.8.2
 python-dotenv==0.20.0
-python-gnupg==0.4.8
+python-gnupg==0.4.9
 python-levenshtein==0.12.2
-python-magic==0.4.25
-pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+python-magic==0.4.27
 pytz==2022.1
+pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 pyyaml==6.0
-redis==3.5.3
+pyzbar==0.1.9
+redis==4.3.4
 regex==2022.3.2; python_version >= '3.6'
-reportlab==3.6.9; python_version >= '3.7' and python_version < '4'
-requests==2.27.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-scikit-learn==1.0.2
-scipy==1.8.0; python_version < '3.11' and python_version >= '3.8'
+reportlab==3.6.11; python_version >= '3.7' and python_version < '4'
+requests==2.28.1; python_version >= '3.7' and python_version < '4'
+scikit-learn==1.1.1
+scipy==1.8.1; python_version < '3.11' and python_version >= '3.8'
 service-identity==21.1.0
-setuptools==61.1.1; python_version >= '3.7'
+setuptools==63.1.0; python_version >= '3.7'
 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sniffio==1.2.0; python_version >= '3.5'
 sqlparse==0.4.2; python_version >= '3.5'
 threadpoolctl==3.1.0; python_version >= '3.6'
 tika==1.24
-tqdm==4.63.1
-twisted[tls]==22.2.0; python_full_version >= '3.6.7'
+tqdm==4.64.0
+twisted[tls]==22.4.0; python_full_version >= '3.6.7'
 txaio==22.2.1; python_version >= '3.6'
-typing-extensions==4.1.1; python_version >= '3.6'
+typing-extensions==4.3.0; python_version >= '3.7'
 tzdata==2022.1; python_version >= '3.6'
-tzlocal==4.1; python_version >= '3.6'
-urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-uvicorn[standard]==0.17.6
+tzlocal==4.2; python_version >= '3.6'
+urllib3==1.26.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'
+uvicorn[standard]==0.18.2
 uvloop==0.16.0
-watchdog==2.1.7
-watchgod==0.8.1
+watchdog==2.1.9
+watchfiles==0.15.0
 wcwidth==0.2.5
-websockets==10.2
-whitenoise==6.0.0
+websockets==10.3
+whitenoise==6.2.0
 whoosh==2.7.4
-zipp==3.7.0; python_version < '3.9'
+wrapt==1.14.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+zipp==3.8.0; python_version < '3.9'
 zope.interface==5.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

scripts/post-consumption-example.sh

@@ -1,21 +1,16 @@
 #!/usr/bin/env bash
 
-DOCUMENT_ID=${1}
-DOCUMENT_FILE_NAME=${2}
-DOCUMENT_SOURCE_PATH=${3}
-DOCUMENT_THUMBNAIL_PATH=${4}
-DOCUMENT_DOWNLOAD_URL=${5}
-DOCUMENT_THUMBNAIL_URL=${6}
-DOCUMENT_CORRESPONDENT=${7}
-DOCUMENT_TAGS=${8}
-
 echo "
 
 A document with an id of ${DOCUMENT_ID} was just consumed.  I know the
 following additional information about it:
 
 * Generated File Name: ${DOCUMENT_FILE_NAME}
+* Archive Path: ${DOCUMENT_ARCHIVE_PATH}
 * Source Path: ${DOCUMENT_SOURCE_PATH}
+* Created: ${DOCUMENT_CREATED}
+* Added: ${DOCUMENT_ADDED}
+* Modified: ${DOCUMENT_MODIFIED}
 * Thumbnail Path: ${DOCUMENT_THUMBNAIL_PATH}
 * Download URL: ${DOCUMENT_DOWNLOAD_URL}
 * Thumbnail URL: ${DOCUMENT_THUMBNAIL_URL}

scripts/start_services.sh

@@ -2,5 +2,5 @@
 
 docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
 docker run -d -p 6379:6379 redis:latest
-docker run -p 3000:3000 -d gotenberg/gotenberg:7
-docker run -p 9998:9998 -d apache/tika
+docker run -p 3000:3000 -d gotenberg/gotenberg:7.4
+docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/cypress.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from 'cypress'
+
+export default defineConfig({
+  videosFolder: 'cypress/videos',
+  screenshotsFolder: 'cypress/screenshots',
+  fixturesFolder: 'cypress/fixtures',
+  e2e: {
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.ts')(on, config)
+    },
+    baseUrl: 'http://localhost:4200',
+  },
+})

src-ui/cypress.json

@@ -1,9 +0,0 @@
-{
-  "integrationFolder": "cypress/integration",
-  "supportFile": "cypress/support/index.ts",
-  "videosFolder": "cypress/videos",
-  "screenshotsFolder": "cypress/screenshots",
-  "pluginsFile": "cypress/plugins/index.ts",
-  "fixturesFolder": "cypress/fixtures",
-  "baseUrl": "http://localhost:4200"
-}

src-ui/cypress/e2e/documents/document-detail.cy.ts

@@ -1,5 +1,7 @@
 describe('document-detail', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.modifiedDocuments = []
 
     cy.fixture('documents/documents.json').then((documentsJson) => {
@@ -15,30 +17,6 @@ describe('document-detail', () => {
       req.reply({ result: 'OK' })
     }).as('saveDoc')
 
-    cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
-      fixture: 'documents/1/metadata.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
-      fixture: 'documents/1/suggestions.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/saved_views/*', {
-      fixture: 'saved_views/savedviews.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/document_types/*', {
-      fixture: 'document_types/doctypes.json',
-    })
-
     cy.viewport(1024, 1024)
     cy.visit('/documents/1/')
   })

src-ui/cypress/e2e/documents/documents-list.cy.ts

@@ -1,8 +1,9 @@
 describe('documents-list', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.bulkEdits = {}
 
-    // mock API methods
     cy.fixture('documents/documents.json').then((documentsJson) => {
       // bulk edit
       cy.intercept(
@@ -53,40 +54,25 @@ describe('documents-list', () => {
       })
     })
 
-    cy.intercept('http://localhost:8000/api/documents/1/thumb/', {
-      fixture: 'documents/lorem-ipsum.png',
-    })
-
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/document_types/*', {
-      fixture: 'document_types/doctypes.json',
-    })
-
+    cy.viewport(1280, 1024)
     cy.visit('/documents')
   })
 
   it('should show a list of documents rendered as cards with thumbnails', () => {
     cy.contains('3 documents')
-    cy.contains('lorem-ipsum')
+    cy.contains('lorem ipsum')
     cy.get('app-document-card-small:first-of-type img')
       .invoke('attr', 'src')
       .should('eq', 'http://localhost:8000/api/documents/1/thumb/')
   })
 
   it('should change to table "details" view', () => {
-    cy.get('div.btn-group-toggle input[value="details"]').parent().click()
+    cy.get('div.btn-group input[value="details"]').next().click()
     cy.get('table')
   })
 
   it('should change to large cards view', () => {
-    cy.get('div.btn-group-toggle input[value="largeCards"]').parent().click()
+    cy.get('div.btn-group input[value="largeCards"]').next().click()
     cy.get('app-document-card-large')
   })
 

src-ui/cypress/e2e/documents/query-params.cy.ts

@@ -0,0 +1,331 @@
+import { PaperlessDocument } from 'src/app/data/paperless-document'
+
+describe('documents query params', () => {
+  beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
+    cy.fixture('documents/documents.json').then((documentsJson) => {
+      // mock api filtering
+      cy.intercept('GET', 'http://localhost:8000/api/documents/*', (req) => {
+        let response = { ...documentsJson }
+
+        if (req.query.hasOwnProperty('ordering')) {
+          const sort_field = req.query['ordering'].toString().replace('-', '')
+          const reverse = req.query['ordering'].toString().indexOf('-') !== -1
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).sort((docA, docB) => {
+            let result = 0
+            switch (sort_field) {
+              case 'created':
+              case 'added':
+                result =
+                  new Date(docA[sort_field]) < new Date(docB[sort_field])
+                    ? -1
+                    : 1
+                break
+              case 'archive_serial_number':
+                result = docA[sort_field] < docB[sort_field] ? -1 : 1
+                break
+            }
+            if (reverse) result = -result
+            return result
+          })
+        }
+
+        if (req.query.hasOwnProperty('tags__id__in')) {
+          const tag_ids: Array<number> = req.query['tags__id__in']
+            .toString()
+            .split(',')
+            .map((v) => +v)
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) =>
+              d.tags.length > 0 &&
+              d.tags.filter((t) => tag_ids.includes(t)).length > 0
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('tags__id__none')) {
+          const tag_ids: Array<number> = req.query['tags__id__none']
+            .toString()
+            .split(',')
+            .map((v) => +v)
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.tags.filter((t) => tag_ids.includes(t)).length == 0)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('is_tagged') &&
+          req.query['is_tagged'] == '0'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.tags.length == 0)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('document_type__id')) {
+          const doctype_id = +req.query['document_type__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.document_type == doctype_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('document_type__isnull') &&
+          req.query['document_type__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.document_type == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('correspondent__id')) {
+          const correspondent_id = +req.query['correspondent__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.correspondent == correspondent_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('correspondent__isnull') &&
+          req.query['correspondent__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.correspondent == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('storage_path__id')) {
+          const storage_path_id = +req.query['storage_path__id']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.storage_path == storage_path_id)
+          response.count = response.results.length
+        } else if (
+          req.query.hasOwnProperty('storage_path__isnull') &&
+          req.query['storage_path__isnull'] == '1'
+        ) {
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.storage_path == undefined)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('created__date__gt')) {
+          const date = new Date(req.query['created__date__gt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.created) > date)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('created__date__lt')) {
+          const date = new Date(req.query['created__date__lt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.created) < date)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('added__date__gt')) {
+          const date = new Date(req.query['added__date__gt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.added) > date)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('added__date__lt')) {
+          const date = new Date(req.query['added__date__lt'])
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => new Date(d.added) < date)
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('title_content')) {
+          const title_content_regexp = new RegExp(
+            req.query['title_content'].toString(),
+            'i'
+          )
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) =>
+              title_content_regexp.test(d.title) ||
+              title_content_regexp.test(d.content)
+          )
+          response.count = response.results.length
+        }
+
+        if (req.query.hasOwnProperty('archive_serial_number')) {
+          const asn = +req.query['archive_serial_number']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) => d.archive_serial_number == asn)
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__isnull')) {
+          const isnull = req.query['storage_path__isnull'] == '1'
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter((d) =>
+            isnull
+              ? d.archive_serial_number == undefined
+              : d.archive_serial_number != undefined
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__gt')) {
+          const asn = +req.query['archive_serial_number__gt']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) => d.archive_serial_number > 0 && d.archive_serial_number > asn
+          )
+          response.count = response.results.length
+        } else if (req.query.hasOwnProperty('archive_serial_number__lt')) {
+          const asn = +req.query['archive_serial_number__lt']
+          response.results = (
+            documentsJson.results as Array<PaperlessDocument>
+          ).filter(
+            (d) => d.archive_serial_number > 0 && d.archive_serial_number < asn
+          )
+          response.count = response.results.length
+        }
+
+        req.reply(response)
+      })
+    })
+  })
+
+  it('should show a list of documents sorted by created', () => {
+    cy.visit('/documents?sort=created')
+    cy.get('app-document-card-small').first().contains('No latin title')
+  })
+
+  it('should show a list of documents reverse sorted by created', () => {
+    cy.visit('/documents?sort=created&reverse=true')
+    cy.get('app-document-card-small').first().contains('sit amet')
+  })
+
+  it('should show a list of documents sorted by added', () => {
+    cy.visit('/documents?sort=added')
+    cy.get('app-document-card-small').first().contains('No latin title')
+  })
+
+  it('should show a list of documents reverse sorted by added', () => {
+    cy.visit('/documents?sort=added&reverse=true')
+    cy.get('app-document-card-small').first().contains('sit amet')
+  })
+
+  it('should show a list of documents filtered by any tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&tags__id__in=2,4,5')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by excluded tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&tags__id__none=2,4')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by no tags', () => {
+    cy.visit('/documents?sort=created&reverse=true&is_tagged=0')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by document type', () => {
+    cy.visit('/documents?sort=created&reverse=true&document_type__id=1')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by no document type', () => {
+    cy.visit('/documents?sort=created&reverse=true&document_type__isnull=1')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by correspondent', () => {
+    cy.visit('/documents?sort=created&reverse=true&correspondent__id=9')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by no correspondent', () => {
+    cy.visit('/documents?sort=created&reverse=true&correspondent__isnull=1')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by storage path', () => {
+    cy.visit('/documents?sort=created&reverse=true&storage_path__id=2')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by no storage path', () => {
+    cy.visit('/documents?sort=created&reverse=true&storage_path__isnull=1')
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by title or content', () => {
+    cy.visit('/documents?sort=created&reverse=true&title_content=lorem')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by asn', () => {
+    cy.visit('/documents?sort=created&reverse=true&archive_serial_number=12345')
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by empty asn', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__isnull=1'
+    )
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by non-empty asn', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__isnull=0'
+    )
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by asn greater than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__gt=12346'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by asn less than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&archive_serial_number__lt=12346'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by created date greater than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&created__date__gt=2022-03-23'
+    )
+    cy.contains('3 documents')
+  })
+
+  it('should show a list of documents filtered by created date less than', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&created__date__lt=2022-03-23'
+    )
+    cy.contains('One document')
+  })
+
+  it('should show a list of documents filtered by added date greater than', () => {
+    cy.visit('/documents?sort=created&reverse=true&added__date__gt=2022-03-24')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by added date less than', () => {
+    cy.visit('/documents?sort=created&reverse=true&added__date__lt=2022-03-24')
+    cy.contains('2 documents')
+  })
+
+  it('should show a list of documents filtered by multiple filters', () => {
+    cy.visit(
+      '/documents?sort=created&reverse=true&document_type__id=1&correspondent__id=9&tags__id__in=4,5'
+    )
+    cy.contains('2 documents')
+  })
+})

src-ui/cypress/e2e/manage/manage.cy.ts

@@ -1,12 +1,5 @@
 describe('manage', () => {
-  beforeEach(() => {
-    cy.intercept('http://localhost:8000/api/correspondents/*', {
-      fixture: 'correspondents/correspondents.json',
-    })
-    cy.intercept('http://localhost:8000/api/tags/*', {
-      fixture: 'tags/tags.json',
-    })
-  })
+  // also uses global fixtures from cypress/support/e2e.ts
 
   it('should show a list of correspondents with bottom pagination as well', () => {
     cy.visit('/correspondents')
@@ -26,7 +19,7 @@ describe('manage', () => {
         req.reply({ count: 3, next: null, previous: null, results: [] })
     })
     cy.visit('/tags')
-    cy.get('tbody').find('button').contains('Documents').first().click() // id = 4
+    cy.get('tbody').find('button:visible').contains('Documents').first().click() // id = 4
     cy.contains('3 documents')
   })
 })

src-ui/cypress/e2e/settings/settings.cy.ts

@@ -1,47 +1,49 @@
 describe('settings', () => {
   beforeEach(() => {
+    // also uses global fixtures from cypress/support/e2e.ts
+
     this.modifiedViews = []
 
     // mock API methods
-    cy.fixture('saved_views/savedviews.json').then((savedViewsJson) => {
-      // saved views PATCH
-      cy.intercept(
-        'PATCH',
-        'http://localhost:8000/api/saved_views/*',
-        (req) => {
-          this.modifiedViews.push(req.body) // store this for later
-          req.reply({ result: 'OK' })
-        }
-      )
+    cy.intercept('http://localhost:8000/api/ui_settings/', {
+      fixture: 'ui_settings/settings.json',
+    }).then(() => {
+      cy.fixture('saved_views/savedviews.json').then((savedViewsJson) => {
+        // saved views PATCH
+        cy.intercept(
+          'PATCH',
+          'http://localhost:8000/api/saved_views/*',
+          (req) => {
+            this.modifiedViews.push(req.body) // store this for later
+            req.reply({ result: 'OK' })
+          }
+        )
 
-      cy.intercept('GET', 'http://localhost:8000/api/saved_views/*', (req) => {
-        let response = { ...savedViewsJson }
-        if (this.modifiedViews.length) {
-          response.results = response.results.map((v) => {
-            if (this.modifiedViews.find((mv) => mv.id == v.id))
-              v = this.modifiedViews.find((mv) => mv.id == v.id)
-            return v
-          })
-        }
+        cy.intercept(
+          'GET',
+          'http://localhost:8000/api/saved_views/*',
+          (req) => {
+            let response = { ...savedViewsJson }
+            if (this.modifiedViews.length) {
+              response.results = response.results.map((v) => {
+                if (this.modifiedViews.find((mv) => mv.id == v.id))
+                  v = this.modifiedViews.find((mv) => mv.id == v.id)
+                return v
+              })
+            }
 
-        req.reply(response)
-      }).as('savedViews')
-    })
-
-    cy.fixture('documents/documents.json').then((documentsJson) => {
-      cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
-        let response = { ...documentsJson }
-        response = response.results.find((d) => d.id == 1)
-        req.reply(response)
+            req.reply(response)
+          }
+        ).as('savedViews')
       })
-    })
 
-    cy.intercept('http://localhost:8000/api/documents/1/metadata/', {
-      fixture: 'documents/1/metadata.json',
-    })
-
-    cy.intercept('http://localhost:8000/api/documents/1/suggestions/', {
-      fixture: 'documents/1/suggestions.json',
+      cy.fixture('documents/documents.json').then((documentsJson) => {
+        cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
+          let response = { ...documentsJson }
+          response = response.results.find((d) => d.id == 1)
+          req.reply(response)
+        })
+      })
     })
 
     cy.viewport(1024, 1024)

src-ui/cypress/e2e/tasks/tasks.cy.ts

@@ -0,0 +1,60 @@
+describe('tasks', () => {
+  beforeEach(() => {
+    this.dismissedTasks = new Set<number>()
+
+    cy.fixture('tasks/tasks.json').then((tasksViewsJson) => {
+      // acknowledge tasks POST
+      cy.intercept(
+        'POST',
+        'http://localhost:8000/api/acknowledge_tasks/',
+        (req) => {
+          req.body['tasks'].forEach((t) => this.dismissedTasks.add(t)) // store this for later
+          req.reply({ result: 'OK' })
+        }
+      )
+
+      cy.intercept('GET', 'http://localhost:8000/api/tasks/', (req) => {
+        let response = [...tasksViewsJson]
+        if (this.dismissedTasks.size) {
+          response = response.filter((t) => {
+            return !this.dismissedTasks.has(t.id)
+          })
+        }
+
+        req.reply(response)
+      }).as('tasks')
+    })
+
+    cy.visit('/tasks')
+    cy.wait('@tasks')
+  })
+
+  it('should show a list of dismissable tasks in tabs', () => {
+    cy.get('tbody').find('tr:visible').its('length').should('eq', 10) // double because collapsible result tr
+    cy.wait(500) // stabilizes the test, for some reason...
+    cy.get('tbody')
+      .find('button:visible')
+      .contains('Dismiss')
+      .first()
+      .click()
+      .wait('@tasks')
+      .wait(2000)
+      .then(() => {
+        cy.get('tbody').find('tr:visible').its('length').should('eq', 8) // double because collapsible result tr
+      })
+  })
+
+  it('should allow toggling all tasks in list and warn on dismiss', () => {
+    cy.get('thead').find('input[type="checkbox"]').first().click()
+    cy.get('body').find('button').contains('Dismiss selected').first().click()
+    cy.contains('Confirm')
+    cy.get('.modal')
+      .contains('button', 'Dismiss')
+      .click()
+      .wait('@tasks')
+      .wait(2000)
+      .then(() => {
+        cy.get('tbody').find('tr:visible').should('not.exist')
+      })
+  })
+})

src-ui/cypress/fixtures/remote_version/remote_version.json

@@ -0,0 +1 @@
+{"version":"v1.7.1","update_available":false,"feature_is_set":true}

src-ui/cypress/fixtures/storage_paths/storage_paths.json

@@ -0,0 +1,17 @@
+{
+    "count": 1,
+    "next": null,
+    "previous": null,
+    "results": [
+        {
+            "id": 2,
+            "slug": "year-title",
+            "name": "Year - Title",
+            "path": "{created_year}/{title}",
+            "match": "",
+            "matching_algorithm": 6,
+            "is_insensitive": true,
+            "document_count": 1
+        }
+    ]
+}