v2.1.1

Co-authored-by: Crowdin Bot <support+bot@crowdin.com>

.build-config.json

@@ -1,9 +0,0 @@
-{
-  "qpdf": {
-      "version": "11.3.0"
-    },
-  "jbig2enc": {
-      "version": "0.29",
-      "git_tag": "0.29"
-    }
-}

.codecov.yml

@@ -1,3 +1,15 @@
+codecov:
+  require_ci_to_pass: true
+# https://docs.codecov.com/docs/flags#recommended-automatic-flag-management
+# Require each flag to have 1 upload before notification
+flag_management:
+  individual_flags:
+    - name: backend
+      paths:
+        - src/
+    - name: frontend
+      paths:
+        - src-ui/
 # https://docs.codecov.com/docs/pull-request-comments
 # codecov will only comment if coverage changes
 comment:
@@ -8,12 +20,9 @@ coverage:
       default:
         # https://docs.codecov.com/docs/commit-status#threshold
         threshold: 1%
-        # https://docs.codecov.com/docs/commit-status#only_pulls
-        only_pulls: true
     patch:
       default:
         # For the changed lines only, target 75% covered, but
         # allow as low as 50%
         target: 75%
         threshold: 25%
-        only_pulls: true

.dockerignore

@@ -1,21 +1,28 @@
+# Tool caches
 **/__pycache__
-/src-ui/.vscode
-/src-ui/node_modules
-/src-ui/dist
+**/.ruff_cache/
+**/.mypy_cache/
+# Virtual environment & similar
+.venv/
+./src-ui/node_modules
+./src-ui/dist
+# IDE folders
+.idea/
+.vscode/
+./src-ui/.vscode
+# VCS
 .git
-/export
-/consume
-/media
-/data
-/docs
-.pytest_cache
-/dist
-/scripts
-/resources
+# Test related
+**/.pytest_cache
 **/tests
 **/*.spec.ts
 **/htmlcov
-/src/.pytest_cache
-.idea
-.venv/
-.vscode/
+# Local folders
+./export
+./consume
+./media
+./data
+./docs
+./dist
+./scripts
+./resources

.env

@@ -1,2 +1 @@
 COMPOSE_PROJECT_NAME=paperless
-export PROMPT="(pipenv-projectname)$P$G"

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -6,14 +6,21 @@ body:
   - type: markdown
     attributes:
       value: |
-        Have a question? 👉 [Start a new discussion](https://github.com/paperless-ngx/paperless-ngx/discussions/new) or [ask in chat](https://matrix.to/#/#paperlessngx:matrix.org).
+        ### ⚠️ Please remember: issues are for *bugs*
+        That is, something you believe affects every single user of Paperless-ngx, not just you. If you're not sure, start with one of the other options below.
 
-        Before opening an issue, please double check:
+        Also, note that **Paperless-ngx does not perform OCR itself**, that is handled by other tools. Problems with OCR of specific files should likely be raised 'upstream', see https://github.com/ocrmypdf/OCRmyPDF/issues or https://github.com/tesseract-ocr/tesseract/issues
+  - type: markdown
+    attributes:
+      value: |
+        #### Have a question? 👉 [Start a new discussion](https://github.com/paperless-ngx/paperless-ngx/discussions/new) or [ask in chat](https://matrix.to/#/#paperlessngx:matrix.org).
+
+        #### Before opening an issue, please double check:
 
         - [The troubleshooting documentation](https://docs.paperless-ngx.com/troubleshooting/).
         - [The installation instructions](https://docs.paperless-ngx.com/setup/#installation).
         - [Existing issues and discussions](https://github.com/paperless-ngx/paperless-ngx/search?q=&type=issues).
-        - Disable any customer container initialization scripts, if using any
+        - Disable any customer container initialization scripts, if using
 
         If you encounter issues while installing or configuring Paperless-ngx, please post in the ["Support" section of the discussions](https://github.com/paperless-ngx/paperless-ngx/discussions/new?category=support).
   - type: textarea

.github/PULL_REQUEST_TEMPLATE.md

@@ -20,11 +20,16 @@ NOTE: Please check only one box!
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
-- [ ] Other (please explain)
+- [ ] Other (please explain):
 
 ## Checklist:
 
+<!--
+NOTE: PRs that do not address the following will not be merged, please do not skip any relevant items.
+-->
+
 - [ ] I have read & agree with the [contributing guidelines](https://github.com/paperless-ngx/paperless-ngx/blob/main/CONTRIBUTING.md).
+- [ ] If applicable, I have included testing coverage for new code in this PR, for [backend](https://docs.paperless-ngx.com/development/#testing) and / or [front-end](https://docs.paperless-ngx.com/development/#testing-and-code-style) changes.
 - [ ] If applicable, I have tested my code for new features & regressions on both mobile & desktop devices, using the latest version of major browsers.
 - [ ] If applicable, I have checked that all tests pass, see [documentation](https://docs.paperless-ngx.com/development/#back-end-development).
 - [ ] I have run all `pre-commit` hooks, see [documentation](https://docs.paperless-ngx.com/development/#code-formatting-with-pre-commit-hooks).

.github/dependabot.yml

@@ -8,7 +8,7 @@ updates:
     target-branch: "dev"
     # Look for `package.json` and `lock` files in the `/src-ui` directory
     directory: "/src-ui"
-    # Check the npm registry for updates every month
+    open-pull-requests-limit: 10
     schedule:
       interval: "monthly"
     labels:
@@ -17,6 +17,21 @@ updates:
     # Add reviewers
     reviewers:
       - "paperless-ngx/frontend"
+    groups:
+      frontend-angular-dependencies:
+        patterns:
+          - "@angular*"
+          - "@ng-*"
+          - "ngx-*"
+          - "ng2-pdf-viewer"
+      frontend-jest-dependencies:
+        patterns:
+          - "@types/jest"
+          - "jest*"
+      frontend-eslint-dependencies:
+        patterns:
+          - "@typescript-eslint*"
+          - "eslint"
 
   # Enable version updates for Python
   - package-ecosystem: "pip"
@@ -32,8 +47,25 @@ updates:
     # Add reviewers
     reviewers:
       - "paperless-ngx/backend"
+    groups:
+      development:
+        patterns:
+          - "*pytest*"
+          - "black"
+          - "ruff"
+          - "mkdocs-material"
+      django:
+        patterns:
+          - "*django*"
+      major-versions:
+        update-types:
+          - "major"
+      small-changes:
+        update-types:
+          - "minor"
+          - "patch"
 
-  # Enable updates for Github Actions
+  # Enable updates for GitHub Actions
   - package-ecosystem: "github-actions"
     target-branch: "dev"
     directory: "/"
@@ -46,3 +78,9 @@ updates:
     # Add reviewers
     reviewers:
       - "paperless-ngx/ci-cd"
+    groups:
+      actions:
+        update-types:
+          - "major"
+          - "minor"
+          - "patch"

.github/scripts/cleanup-tags.py

@@ -1,488 +0,0 @@
-import json
-import logging
-import os
-import shutil
-import subprocess
-from argparse import ArgumentParser
-from typing import Dict
-from typing import Final
-from typing import Iterator
-from typing import List
-from typing import Optional
-
-from common import get_log_level
-from github import ContainerPackage
-from github import GithubBranchApi
-from github import GithubContainerRegistryApi
-
-logger = logging.getLogger("cleanup-tags")
-
-
-class ImageProperties:
-    """
-    Data class wrapping the properties of an entry in the image index
-    manifests list.  It is NOT an actual image with layers, etc
-
-    https://docs.docker.com/registry/spec/manifest-v2-2/
-    https://github.com/opencontainers/image-spec/blob/main/manifest.md
-    https://github.com/opencontainers/image-spec/blob/main/descriptor.md
-    """
-
-    def __init__(self, data: Dict) -> None:
-        self._data = data
-        # This is the sha256: digest string.  Corresponds to GitHub API name
-        # if the package is an untagged package
-        self.digest = self._data["digest"]
-        platform_data_os = self._data["platform"]["os"]
-        platform_arch = self._data["platform"]["architecture"]
-        platform_variant = self._data["platform"].get(
-            "variant",
-            "",
-        )
-        self.platform = f"{platform_data_os}/{platform_arch}{platform_variant}"
-
-
-class ImageIndex:
-    """
-    Data class wrapping up logic for an OCI Image Index
-    JSON data.  Primary use is to access the manifests listing
-
-    See https://github.com/opencontainers/image-spec/blob/main/image-index.md
-    """
-
-    def __init__(self, package_url: str, tag: str) -> None:
-        self.qualified_name = f"{package_url}:{tag}"
-        logger.info(f"Getting image index for {self.qualified_name}")
-        try:
-            proc = subprocess.run(
-                [
-                    shutil.which("docker"),
-                    "buildx",
-                    "imagetools",
-                    "inspect",
-                    "--raw",
-                    self.qualified_name,
-                ],
-                capture_output=True,
-                check=True,
-            )
-
-            self._data = json.loads(proc.stdout)
-
-        except subprocess.CalledProcessError as e:
-            logger.error(
-                f"Failed to get image index for {self.qualified_name}: {e.stderr}",
-            )
-            raise e
-
-    @property
-    def image_pointers(self) -> Iterator[ImageProperties]:
-        for manifest_data in self._data["manifests"]:
-            yield ImageProperties(manifest_data)
-
-
-class RegistryTagsCleaner:
-    """
-    This is the base class for the image registry cleaning.  Given a package
-    name, it will keep all images which are tagged and all untagged images
-    referred to by a manifest.  This results in only images which have been untagged
-    and cannot be referenced except by their SHA in being removed.  None of these
-    images should be referenced, so it is fine to delete them.
-    """
-
-    def __init__(
-        self,
-        package_name: str,
-        repo_owner: str,
-        repo_name: str,
-        package_api: GithubContainerRegistryApi,
-        branch_api: Optional[GithubBranchApi],
-    ):
-        self.actually_delete = False
-        self.package_api = package_api
-        self.branch_api = branch_api
-        self.package_name = package_name
-        self.repo_owner = repo_owner
-        self.repo_name = repo_name
-        self.tags_to_delete: List[str] = []
-        self.tags_to_keep: List[str] = []
-
-        # Get the information about all versions of the given package
-        # These are active, not deleted, the default returned from the API
-        self.all_package_versions = self.package_api.get_active_package_versions(
-            self.package_name,
-        )
-
-        # Get a mapping from a tag like "1.7.0" or "feature-xyz" to the ContainerPackage
-        # tagged with it.  It makes certain lookups easy
-        self.all_pkgs_tags_to_version: Dict[str, ContainerPackage] = {}
-        for pkg in self.all_package_versions:
-            for tag in pkg.tags:
-                self.all_pkgs_tags_to_version[tag] = pkg
-        logger.info(
-            f"Located {len(self.all_package_versions)} versions of package {self.package_name}",
-        )
-
-        self.decide_what_tags_to_keep()
-
-    def clean(self):
-        """
-        This method will delete image versions, based on the selected tags to delete.
-        It behaves more like an unlinking than actual deletion.  Removing the tag
-        simply removes a pointer to an image, but the actual image data remains accessible
-        if one has the sha256 digest of it.
-        """
-        for tag_to_delete in self.tags_to_delete:
-            package_version_info = self.all_pkgs_tags_to_version[tag_to_delete]
-
-            if self.actually_delete:
-                logger.info(
-                    f"Deleting {tag_to_delete} (id {package_version_info.id})",
-                )
-                self.package_api.delete_package_version(
-                    package_version_info,
-                )
-
-            else:
-                logger.info(
-                    f"Would delete {tag_to_delete} (id {package_version_info.id})",
-                )
-        else:
-            logger.info("No tags to delete")
-
-    def clean_untagged(self, is_manifest_image: bool):
-        """
-        This method will delete untagged images, that is those which are not named.  It
-        handles if the image tag is actually a manifest, which points to images that look otherwise
-        untagged.
-        """
-
-        def _clean_untagged_manifest():
-            """
-
-            Handles the deletion of untagged images, but where the package is a manifest, ie a multi
-            arch image, which means some "untagged" images need to exist still.
-
-            Ok, bear with me, these are annoying.
-
-            Our images are multi-arch, so the manifest is more like a pointer to a sha256 digest.
-            These images are untagged, but pointed to, and so should not be removed (or every pull fails).
-
-            So for each image getting kept, parse the manifest to find the digest(s) it points to.  Then
-            remove those from the list of untagged images.  The final result is the untagged, not pointed to
-            version which should be safe to remove.
-
-            Example:
-                Tag: ghcr.io/paperless-ngx/paperless-ngx:1.7.1 refers to
-                    amd64: sha256:b9ed4f8753bbf5146547671052d7e91f68cdfc9ef049d06690b2bc866fec2690
-                    armv7: sha256:81605222df4ba4605a2ba4893276e5d08c511231ead1d5da061410e1bbec05c3
-                    arm64: sha256:374cd68db40734b844705bfc38faae84cc4182371de4bebd533a9a365d5e8f3b
-                each of which appears as untagged image, but isn't really.
-
-                So from the list of untagged packages, remove those digests.  Once all tags which
-                are being kept are checked, the remaining untagged packages are actually untagged
-                with no referrals in a manifest to them.
-            """
-            # Simplify the untagged data, mapping name (which is a digest) to the version
-            # At the moment, these are the images which APPEAR untagged.
-            untagged_versions = {}
-            for x in self.all_package_versions:
-                if x.untagged:
-                    untagged_versions[x.name] = x
-
-            skips = 0
-
-            # Parse manifests to locate digests pointed to
-            for tag in sorted(self.tags_to_keep):
-                try:
-                    image_index = ImageIndex(
-                        f"ghcr.io/{self.repo_owner}/{self.package_name}",
-                        tag,
-                    )
-                    for manifest in image_index.image_pointers:
-
-                        if manifest.digest in untagged_versions:
-                            logger.info(
-                                f"Skipping deletion of {manifest.digest},"
-                                f" referred to by {image_index.qualified_name}"
-                                f" for {manifest.platform}",
-                            )
-                            del untagged_versions[manifest.digest]
-                            skips += 1
-
-                except Exception as err:
-                    self.actually_delete = False
-                    logger.exception(err)
-                    return
-
-            logger.info(
-                f"Skipping deletion of {skips} packages referred to by a manifest",
-            )
-
-            # Delete the untagged and not pointed at packages
-            logger.info(f"Deleting untagged packages of {self.package_name}")
-            for to_delete_name in untagged_versions:
-                to_delete_version = untagged_versions[to_delete_name]
-
-                if self.actually_delete:
-                    logger.info(
-                        f"Deleting id {to_delete_version.id} named {to_delete_version.name}",
-                    )
-                    self.package_api.delete_package_version(
-                        to_delete_version,
-                    )
-                else:
-                    logger.info(
-                        f"Would delete {to_delete_name} (id {to_delete_version.id})",
-                    )
-
-        def _clean_untagged_non_manifest():
-            """
-            If the package is not a multi-arch manifest, images without tags are safe to delete.
-            """
-
-            for package in self.all_package_versions:
-                if package.untagged:
-                    if self.actually_delete:
-                        logger.info(
-                            f"Deleting id {package.id} named {package.name}",
-                        )
-                        self.package_api.delete_package_version(
-                            package,
-                        )
-                    else:
-                        logger.info(
-                            f"Would delete {package.name} (id {package.id})",
-                        )
-                else:
-                    logger.info(
-                        f"Not deleting tag {package.tags[0]} of package {self.package_name}",
-                    )
-
-        logger.info("Beginning untagged image cleaning")
-
-        if is_manifest_image:
-            _clean_untagged_manifest()
-        else:
-            _clean_untagged_non_manifest()
-
-    def decide_what_tags_to_keep(self):
-        """
-        This method holds the logic to delete what tags to keep and there fore
-        what tags to delete.
-
-        By default, any image with at least 1 tag will be kept
-        """
-        # By default, keep anything which is tagged
-        self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))
-
-    def check_remaining_tags_valid(self):
-        """
-        Checks the non-deleted tags are still valid.  The assumption is if the
-        manifest is can be inspected and each image manifest if points to can be
-        inspected, the image will still pull.
-
-        https://github.com/opencontainers/image-spec/blob/main/image-index.md
-        """
-        logger.info("Beginning confirmation step")
-        a_tag_failed = False
-        for tag in sorted(self.tags_to_keep):
-
-            try:
-                image_index = ImageIndex(
-                    f"ghcr.io/{self.repo_owner}/{self.package_name}",
-                    tag,
-                )
-                for manifest in image_index.image_pointers:
-                    logger.info(f"Checking {manifest.digest} for {manifest.platform}")
-
-                    # This follows the pointer from the index to an actual image, layers and all
-                    # Note the format is @
-                    digest_name = f"ghcr.io/{self.repo_owner}/{self.package_name}@{manifest.digest}"
-
-                    try:
-
-                        subprocess.run(
-                            [
-                                shutil.which("docker"),
-                                "buildx",
-                                "imagetools",
-                                "inspect",
-                                "--raw",
-                                digest_name,
-                            ],
-                            capture_output=True,
-                            check=True,
-                        )
-                    except subprocess.CalledProcessError as e:
-                        logger.error(f"Failed to inspect digest: {e.stderr}")
-                        a_tag_failed = True
-            except subprocess.CalledProcessError as e:
-                a_tag_failed = True
-                logger.error(f"Failed to inspect: {e.stderr}")
-                continue
-
-        if a_tag_failed:
-            raise Exception("At least one image tag failed to inspect")
-
-
-class MainImageTagsCleaner(RegistryTagsCleaner):
-    def decide_what_tags_to_keep(self):
-        """
-        Overrides the default logic for deciding what images to keep.  Images tagged as "feature-"
-        will be removed, if the corresponding branch no longer exists.
-        """
-
-        # Default to everything gets kept still
-        super().decide_what_tags_to_keep()
-
-        # Locate the feature branches
-        feature_branches = {}
-        for branch in self.branch_api.get_branches(
-            repo=self.repo_name,
-        ):
-            if branch.name.startswith("feature-"):
-                logger.debug(f"Found feature branch {branch.name}")
-                feature_branches[branch.name] = branch
-
-        logger.info(f"Located {len(feature_branches)} feature branches")
-
-        if not len(feature_branches):
-            # Our work here is done, delete nothing
-            return
-
-        # Filter to packages which are tagged with feature-*
-        packages_tagged_feature: List[ContainerPackage] = []
-        for package in self.all_package_versions:
-            if package.tag_matches("feature-"):
-                packages_tagged_feature.append(package)
-
-        # Map tags like "feature-xyz" to a ContainerPackage
-        feature_pkgs_tags_to_versions: Dict[str, ContainerPackage] = {}
-        for pkg in packages_tagged_feature:
-            for tag in pkg.tags:
-                feature_pkgs_tags_to_versions[tag] = pkg
-
-        logger.info(
-            f'Located {len(feature_pkgs_tags_to_versions)} versions of package {self.package_name} tagged "feature-"',
-        )
-
-        # All the feature tags minus all the feature branches leaves us feature tags
-        # with no corresponding branch
-        self.tags_to_delete = list(
-            set(feature_pkgs_tags_to_versions.keys()) - set(feature_branches.keys()),
-        )
-
-        # All the tags minus the set of going to be deleted tags leaves us the
-        # tags which will be kept around
-        self.tags_to_keep = list(
-            set(self.all_pkgs_tags_to_version.keys()) - set(self.tags_to_delete),
-        )
-        logger.info(
-            f"Located {len(self.tags_to_delete)} versions of package {self.package_name} to delete",
-        )
-
-
-class LibraryTagsCleaner(RegistryTagsCleaner):
-    """
-    Exists for the off chance that someday, the installer library images
-    will need their own logic
-    """
-
-
-def _main():
-    parser = ArgumentParser(
-        description="Using the GitHub API locate and optionally delete container"
-        " tags which no longer have an associated feature branch",
-    )
-
-    # Requires an affirmative command to actually do a delete
-    parser.add_argument(
-        "--delete",
-        action="store_true",
-        default=False,
-        help="If provided, actually delete the container tags",
-    )
-
-    # When a tagged image is updated, the previous version remains, but it no longer tagged
-    # Add this option to remove them as well
-    parser.add_argument(
-        "--untagged",
-        action="store_true",
-        default=False,
-        help="If provided, delete untagged containers as well",
-    )
-
-    # If given, the package is assumed to be a multi-arch manifest.  Cache packages are
-    # not multi-arch, all other types are
-    parser.add_argument(
-        "--is-manifest",
-        action="store_true",
-        default=False,
-        help="If provided, the package is assumed to be a multi-arch manifest following schema v2",
-    )
-
-    # Allows configuration of log level for debugging
-    parser.add_argument(
-        "--loglevel",
-        default="info",
-        help="Configures the logging level",
-    )
-
-    # Get the name of the package being processed this round
-    parser.add_argument(
-        "package",
-        help="The package to process",
-    )
-
-    args = parser.parse_args()
-
-    logging.basicConfig(
-        level=get_log_level(args),
-        datefmt="%Y-%m-%d %H:%M:%S",
-        format="%(asctime)s %(levelname)-8s %(message)s",
-    )
-
-    # Must be provided in the environment
-    repo_owner: Final[str] = os.environ["GITHUB_REPOSITORY_OWNER"]
-    repo: Final[str] = os.environ["GITHUB_REPOSITORY"]
-    gh_token: Final[str] = os.environ["TOKEN"]
-
-    # Find all branches named feature-*
-    # Note: Only relevant to the main application, but simpler to
-    # leave in for all packages
-    with GithubBranchApi(gh_token) as branch_api:
-        with GithubContainerRegistryApi(gh_token, repo_owner) as container_api:
-            if args.package in {"paperless-ngx", "paperless-ngx/builder/cache/app"}:
-                cleaner = MainImageTagsCleaner(
-                    args.package,
-                    repo_owner,
-                    repo,
-                    container_api,
-                    branch_api,
-                )
-            else:
-                cleaner = LibraryTagsCleaner(
-                    args.package,
-                    repo_owner,
-                    repo,
-                    container_api,
-                    None,
-                )
-
-            # Set if actually doing a delete vs dry run
-            cleaner.actually_delete = args.delete
-
-            # Clean images with tags
-            cleaner.clean()
-
-            # Clean images which are untagged
-            cleaner.clean_untagged(args.is_manifest)
-
-            # Verify remaining tags still pull
-            if args.is_manifest:
-                cleaner.check_remaining_tags_valid()
-
-
-if __name__ == "__main__":
-    _main()

.github/scripts/common.py

@@ -1,47 +0,0 @@
-import logging
-
-
-def get_image_tag(
-    repo_name: str,
-    pkg_name: str,
-    pkg_version: str,
-) -> str:
-    """
-    Returns a string representing the normal image for a given package
-    """
-    return f"ghcr.io/{repo_name.lower()}/builder/{pkg_name}:{pkg_version}"
-
-
-def get_cache_image_tag(
-    repo_name: str,
-    pkg_name: str,
-    pkg_version: str,
-    branch_name: str,
-) -> str:
-    """
-    Returns a string representing the expected image cache tag for a given package
-
-    Registry type caching is utilized for the builder images, to allow fast
-    rebuilds, generally almost instant for the same version
-    """
-    return f"ghcr.io/{repo_name.lower()}/builder/cache/{pkg_name}:{pkg_version}"
-
-
-def get_log_level(args) -> int:
-    """
-    Returns a logging level, based
-    :param args:
-    :return:
-    """
-    levels = {
-        "critical": logging.CRITICAL,
-        "error": logging.ERROR,
-        "warn": logging.WARNING,
-        "warning": logging.WARNING,
-        "info": logging.INFO,
-        "debug": logging.DEBUG,
-    }
-    level = levels.get(args.loglevel.lower())
-    if level is None:
-        level = logging.INFO
-    return level

.github/scripts/get-build-json.py

@@ -1,91 +0,0 @@
-"""
-This is a helper script for the mutli-stage Docker image builder.
-It provides a single point of configuration for package version control.
-The output JSON object is used by the CI workflow to determine what versions
-to build and pull into the final Docker image.
-
-Python package information is obtained from the Pipfile.lock.  As this is
-kept updated by dependabot, it usually will need no further configuration.
-The sole exception currently is pikepdf, which has a dependency on qpdf,
-and is configured here to use the latest version of qpdf built by the workflow.
-
-Other package version information is configured directly below, generally by
-setting the version and Git information, if any.
-
-"""
-import argparse
-import json
-import os
-from pathlib import Path
-from typing import Final
-
-from common import get_cache_image_tag
-from common import get_image_tag
-
-
-def _main():
-    parser = argparse.ArgumentParser(
-        description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
-    )
-    parser.add_argument(
-        "package",
-        help="The name of the package to generate JSON for",
-    )
-
-    PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock")
-    BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json")
-
-    # Read the main config file
-    build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text())
-
-    # Read Pipfile.lock file
-    pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text())
-
-    args: Final = parser.parse_args()
-
-    # Read from environment variables set by GitHub Actions
-    repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"]
-    branch_name: Final[str] = os.environ["GITHUB_REF_NAME"]
-
-    # Default output values
-    version = None
-    extra_config = {}
-
-    if args.package in pipfile_data["default"]:
-        # Read the version from Pipfile.lock
-        pkg_data = pipfile_data["default"][args.package]
-        pkg_version = pkg_data["version"].split("==")[-1]
-        version = pkg_version
-
-        # Any extra/special values needed
-        if args.package == "pikepdf":
-            extra_config["qpdf_version"] = build_json["qpdf"]["version"]
-
-    elif args.package in build_json:
-        version = build_json[args.package]["version"]
-
-    else:
-        raise NotImplementedError(args.package)
-
-    # The JSON object we'll output
-    output = {
-        "name": args.package,
-        "version": version,
-        "image_tag": get_image_tag(repo_name, args.package, version),
-        "cache_tag": get_cache_image_tag(
-            repo_name,
-            args.package,
-            version,
-            branch_name,
-        ),
-    }
-
-    # Add anything special a package may need
-    output.update(extra_config)
-
-    # Output the JSON info to stdout
-    print(json.dumps(output))
-
-
-if __name__ == "__main__":
-    _main()

.github/scripts/github.py

@@ -1,270 +0,0 @@
-"""
-This module contains some useful classes for interacting with the Github API.
-The full documentation for the API can be found here: https://docs.github.com/en/rest
-
-Mostly, this focusses on two areas, repo branches and repo packages, as the use case
-is cleaning up container images which are no longer referred to.
-
-"""
-import functools
-import logging
-import re
-import urllib.parse
-from typing import Dict
-from typing import List
-from typing import Optional
-
-import httpx
-
-logger = logging.getLogger("github-api")
-
-
-class _GithubApiBase:
-    """
-    A base class for interacting with the Github API.  It
-    will handle the session and setting authorization headers.
-    """
-
-    def __init__(self, token: str) -> None:
-        self._token = token
-        self._client: Optional[httpx.Client] = None
-
-    def __enter__(self) -> "_GithubApiBase":
-        """
-        Sets up the required headers for auth and response
-        type from the API
-        """
-        self._client = httpx.Client()
-        self._client.headers.update(
-            {
-                "Accept": "application/vnd.github.v3+json",
-                "Authorization": f"token {self._token}",
-            },
-        )
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        """
-        Ensures the authorization token is cleaned up no matter
-        the reason for the exit
-        """
-        if "Accept" in self._client.headers:
-            del self._client.headers["Accept"]
-        if "Authorization" in self._client.headers:
-            del self._client.headers["Authorization"]
-
-        # Close the session as well
-        self._client.close()
-        self._client = None
-
-    def _read_all_pages(self, endpoint):
-        """
-        Helper function to read all pages of an endpoint, utilizing the
-        next.url until exhausted.  Assumes the endpoint returns a list
-        """
-        internal_data = []
-
-        while True:
-            resp = self._client.get(endpoint)
-            if resp.status_code == 200:
-                internal_data += resp.json()
-                if "next" in resp.links:
-                    endpoint = resp.links["next"]["url"]
-                else:
-                    logger.debug("Exiting pagination loop")
-                    break
-            else:
-                logger.warning(f"Request to {endpoint} return HTTP {resp.status_code}")
-                resp.raise_for_status()
-
-        return internal_data
-
-
-class _EndpointResponse:
-    """
-    For all endpoint JSON responses, store the full
-    response data, for ease of extending later, if need be.
-    """
-
-    def __init__(self, data: Dict) -> None:
-        self._data = data
-
-
-class GithubBranch(_EndpointResponse):
-    """
-    Simple wrapper for a repository branch, only extracts name information
-    for now.
-    """
-
-    def __init__(self, data: Dict) -> None:
-        super().__init__(data)
-        self.name = self._data["name"]
-
-
-class GithubBranchApi(_GithubApiBase):
-    """
-    Wrapper around branch API.
-
-    See https://docs.github.com/en/rest/branches/branches
-
-    """
-
-    def __init__(self, token: str) -> None:
-        super().__init__(token)
-
-        self._ENDPOINT = "https://api.github.com/repos/{REPO}/branches"
-
-    def get_branches(self, repo: str) -> List[GithubBranch]:
-        """
-        Returns all current branches of the given repository owned by the given
-        owner or organization.
-        """
-        # The environment GITHUB_REPOSITORY already contains the owner in the correct location
-        endpoint = self._ENDPOINT.format(REPO=repo)
-        internal_data = self._read_all_pages(endpoint)
-        return [GithubBranch(branch) for branch in internal_data]
-
-
-class ContainerPackage(_EndpointResponse):
-    """
-    Data class wrapping the JSON response from the package related
-    endpoints
-    """
-
-    def __init__(self, data: Dict):
-        super().__init__(data)
-        # This is a numerical ID, required for interactions with this
-        # specific package, including deletion of it or restoration
-        self.id: int = self._data["id"]
-
-        # A string name.  This might be an actual name or it could be a
-        # digest string like "sha256:"
-        self.name: str = self._data["name"]
-
-        # URL to the package, including its ID, can be used for deletion
-        # or restoration without needing to build up a URL ourselves
-        self.url: str = self._data["url"]
-
-        # The list of tags applied to this image. Maybe an empty list
-        self.tags: List[str] = self._data["metadata"]["container"]["tags"]
-
-    @functools.cached_property
-    def untagged(self) -> bool:
-        """
-        Returns True if the image has no tags applied to it, False otherwise
-        """
-        return len(self.tags) == 0
-
-    @functools.cache
-    def tag_matches(self, pattern: str) -> bool:
-        """
-        Returns True if the image has at least one tag which matches the given regex,
-        False otherwise
-        """
-        return any(re.match(pattern, tag) is not None for tag in self.tags)
-
-    def __repr__(self):
-        return f"Package {self.name}"
-
-
-class GithubContainerRegistryApi(_GithubApiBase):
-    """
-    Class wrapper to deal with the Github packages API.  This class only deals with
-    container type packages, the only type published by paperless-ngx.
-    """
-
-    def __init__(self, token: str, owner_or_org: str) -> None:
-        super().__init__(token)
-        self._owner_or_org = owner_or_org
-        if self._owner_or_org == "paperless-ngx":
-            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-an-organization
-            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
-            # https://docs.github.com/en/rest/packages#delete-package-version-for-an-organization
-            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/orgs/{ORG}/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
-        else:
-            # https://docs.github.com/en/rest/packages#get-all-package-versions-for-a-package-owned-by-the-authenticated-user
-            self._PACKAGES_VERSIONS_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions"
-            # https://docs.github.com/en/rest/packages#delete-a-package-version-for-the-authenticated-user
-            self._PACKAGE_VERSION_DELETE_ENDPOINT = "https://api.github.com/user/packages/{PACKAGE_TYPE}/{PACKAGE_NAME}/versions/{PACKAGE_VERSION_ID}"
-        self._PACKAGE_VERSION_RESTORE_ENDPOINT = (
-            f"{self._PACKAGE_VERSION_DELETE_ENDPOINT}/restore"
-        )
-
-    def get_active_package_versions(
-        self,
-        package_name: str,
-    ) -> List[ContainerPackage]:
-        """
-        Returns all the versions of a given package (container images) from
-        the API
-        """
-
-        package_type: str = "container"
-        # Need to quote this for slashes in the name
-        package_name = urllib.parse.quote(package_name, safe="")
-
-        endpoint = self._PACKAGES_VERSIONS_ENDPOINT.format(
-            ORG=self._owner_or_org,
-            PACKAGE_TYPE=package_type,
-            PACKAGE_NAME=package_name,
-        )
-
-        pkgs = []
-
-        for data in self._read_all_pages(endpoint):
-            pkgs.append(ContainerPackage(data))
-
-        return pkgs
-
-    def get_deleted_package_versions(
-        self,
-        package_name: str,
-    ) -> List[ContainerPackage]:
-        package_type: str = "container"
-        # Need to quote this for slashes in the name
-        package_name = urllib.parse.quote(package_name, safe="")
-
-        endpoint = (
-            self._PACKAGES_VERSIONS_ENDPOINT.format(
-                ORG=self._owner_or_org,
-                PACKAGE_TYPE=package_type,
-                PACKAGE_NAME=package_name,
-            )
-            + "?state=deleted"
-        )
-
-        pkgs = []
-
-        for data in self._read_all_pages(endpoint):
-            pkgs.append(ContainerPackage(data))
-
-        return pkgs
-
-    def delete_package_version(self, package_data: ContainerPackage):
-        """
-        Deletes the given package version from the GHCR
-        """
-        resp = self._client.delete(package_data.url)
-        if resp.status_code != 204:
-            logger.warning(
-                f"Request to delete {package_data.url} returned HTTP {resp.status_code}",
-            )
-
-    def restore_package_version(
-        self,
-        package_name: str,
-        package_data: ContainerPackage,
-    ):
-        package_type: str = "container"
-        endpoint = self._PACKAGE_VERSION_RESTORE_ENDPOINT.format(
-            ORG=self._owner_or_org,
-            PACKAGE_TYPE=package_type,
-            PACKAGE_NAME=package_name,
-            PACKAGE_VERSION_ID=package_data.id,
-        )
-
-        resp = self._client.post(endpoint)
-        if resp.status_code != 204:
-            logger.warning(
-                f"Request to delete {endpoint} returned HTTP {resp.status_code}",
-            )

.github/stale.yml

@@ -1,23 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
-onlyLabels: [cant-reproduce]
-
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
-
-# See https://github.com/marketplace/stale for more info on the app
-# and https://github.com/probot/stale for the configuration docs

.github/workflows/ci.yml

@@ -16,19 +16,25 @@ on:
 env:
   # This is the version of pipenv all the steps will use
   # If changing this, change Dockerfile
-  DEFAULT_PIP_ENV_VERSION: "2023.3.20"
-  # This is the default version of Python to use in most steps
-  # If changing this, change Dockerfile
-  DEFAULT_PYTHON_VERSION: "3.9"
+  DEFAULT_PIP_ENV_VERSION: "2023.10.24"
+  # This is the default version of Python to use in most steps which aren't specific
+  DEFAULT_PYTHON_VERSION: "3.10"
 
 jobs:
   pre-commit:
+    # We want to run on external PRs, but not on our own internal PRs as they'll be run
+    # by the push to the branch. Without this if check, checks are duplicated since
+    # internal PRs match both the push and pull_request events.
+    if:
+      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
+      github.repository
+
     name: Linting Checks
     runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Install python
         uses: actions/setup-python@v4
@@ -39,14 +45,14 @@ jobs:
         uses: pre-commit/action@v3.0.0
 
   documentation:
-    name: "Build Documentation"
+    name: "Build & Deploy Documentation"
     runs-on: ubuntu-22.04
     needs:
       - pre-commit
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Set up Python
         id: setup-python
@@ -58,7 +64,7 @@ jobs:
       -
         name: Install pipenv
         run: |
-          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
+          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
       -
         name: Install dependencies
         run: |
@@ -71,59 +77,40 @@ jobs:
         name: Make documentation
         run: |
           pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs build --config-file ./mkdocs.yml
+      -
+        name: Deploy documentation
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          echo "docs.paperless-ngx.com" > "${{ github.workspace }}/docs/CNAME"
+          git config --global user.name "${{ github.actor }}"
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs gh-deploy --force --no-history
       -
         name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
           name: documentation
           path: site/
-
-  documentation-deploy:
-    name: "Deploy Documentation"
-    runs-on: ubuntu-22.04
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    needs:
-      - documentation
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Deploy docs
-        uses: mhausenblas/mkdocs-deploy-gh-pages@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CUSTOM_DOMAIN: docs.paperless-ngx.com
-          CONFIG_FILE: mkdocs.yml
-          EXTRA_PACKAGES: build-base
+          retention-days: 7
 
   tests-backend:
-    name: "Tests (${{ matrix.python-version }})"
+    name: "Backend Tests (Python ${{ matrix.python-version }})"
     runs-on: ubuntu-22.04
     needs:
       - pre-commit
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
-    env:
-      # Enable Tika end to end testing
-      TIKA_LIVE: 1
-      # Enable paperless_mail testing against real server
-      PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
-      PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
-      PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
-      # Enable Gotenberg end to end testing
-      GOTENBERG_LIVE: 1
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Start containers
         run: |
-          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml pull --quiet
-          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml up --detach
+          docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml pull --quiet
+          docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml up --detach
       -
         name: Set up Python
         id: setup-python
@@ -135,7 +122,7 @@ jobs:
       -
         name: Install pipenv
         run: |
-          pip install --user pipenv==${DEFAULT_PIP_ENV_VERSION}
+          pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }}
       -
         name: Install system dependencies
         run: |
@@ -156,131 +143,176 @@ jobs:
           pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list
       -
         name: Tests
+        env:
+          PAPERLESS_CI_TEST: 1
+          # Enable paperless_mail testing against real server
+          PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }}
+          PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }}
+          PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }}
         run: |
           cd src/
           pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra
       -
-        name: Upload coverage to Codecov
+        name: Upload coverage
         if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: backend-coverage-report
+          path: src/coverage.xml
+          retention-days: 7
+          if-no-files-found: warn
+      -
+        name: Stop containers
+        if: always()
+        run: |
+          docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml logs
+          docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml down
+
+  install-frontend-depedendencies:
+    name: "Install Frontend Dependendencies"
+    runs-on: ubuntu-22.04
+    needs:
+      - pre-commit
+    steps:
+      - uses: actions/checkout@v4
+      -
+        name: Use Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.x
+          cache: 'npm'
+          cache-dependency-path: 'src-ui/package-lock.json'
+      - name: Cache frontend depdendencies
+        id: cache-frontend-deps
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.npm
+            ~/.cache
+          key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/package-lock.json') }}
+      -
+        name: Install dependencies
+        if: steps.cache-frontend-deps.outputs.cache-hit != 'true'
+        run: cd src-ui && npm ci
+      -
+        name: Install Playwright
+        if: steps.cache-frontend-deps.outputs.cache-hit != 'true'
+        run: cd src-ui && npx playwright install --with-deps
+
+  tests-frontend:
+    name: "Frontend Tests (Node ${{ matrix.node-version }} - ${{ matrix.shard-index }}/${{ matrix.shard-count }})"
+    runs-on: ubuntu-22.04
+    needs:
+      - install-frontend-depedendencies
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [20.x]
+        shard-index: [1, 2, 3, 4]
+        shard-count: [4]
+    steps:
+      - uses: actions/checkout@v4
+      -
+        name: Use Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.x
+          cache: 'npm'
+          cache-dependency-path: 'src-ui/package-lock.json'
+      - name: Cache frontend depdendencies
+        id: cache-frontend-deps
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.npm
+            ~/.cache
+          key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/package-lock.json') }}
+      - name: Re-link Angular cli
+        run: cd src-ui && npm link @angular/cli
+      -
+        name: Linting checks
+        run: cd src-ui && npm run lint
+      -
+        name: Run Jest unit tests
+        run: cd src-ui && npm run test -- --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
+      -
+        name: Upload Jest coverage
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: jest-coverage-report-${{ matrix.shard-index }}
+          path: |
+            src-ui/coverage/coverage-final.json
+            src-ui/coverage/lcov.info
+            src-ui/coverage/clover.xml
+          retention-days: 7
+          if-no-files-found: warn
+      -
+        name: Run Playwright e2e tests
+        run: cd src-ui && npx playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }}
+      -
+        name: Upload Playwright test results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: playwright-report
+          path: src-ui/playwright-report
+          retention-days: 7
+
+  tests-coverage-upload:
+    name: "Upload Coverage"
+    runs-on: ubuntu-22.04
+    needs:
+      - tests-backend
+      - tests-frontend
+    steps:
+      -
+        uses: actions/checkout@v4
+      -
+        name: Download frontend coverage
+        uses: actions/download-artifact@v3
+        with:
+          path: src-ui/coverage/
+      -
+        name: Upload frontend coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          # not required for public repos, but intermittently fails otherwise
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: frontend
+          directory: src-ui/coverage/
+          # dont include backend coverage files here
+          files: '!coverage.xml'
+      -
+        name: Download backend coverage
+        uses: actions/download-artifact@v3
+        with:
+          name: backend-coverage-report
+          path: src/
+      -
+        name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
           # not required for public repos, but intermittently fails otherwise
           token: ${{ secrets.CODECOV_TOKEN }}
           # future expansion
           flags: backend
-      -
-        name: Stop containers
-        if: always()
-        run: |
-          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml logs
-          docker compose --file ${GITHUB_WORKSPACE}/docker/compose/docker-compose.ci-test.yml down
+          directory: src/
 
-  tests-frontend:
-    name: "Tests Frontend"
-    runs-on: ubuntu-22.04
-    needs:
-      - pre-commit
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      -
-        name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: 'npm'
-          cache-dependency-path: 'src-ui/package-lock.json'
-      - run: cd src-ui && npm ci
-      - run: cd src-ui && npm run lint
-      - run: cd src-ui && npm run test
-      - run: cd src-ui && npm run e2e:ci
-
-  prepare-docker-build:
-    name: Prepare Docker Pipeline Data
-    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
-    runs-on: ubuntu-22.04
-    needs:
-      - documentation
-      - tests-backend
-      - tests-frontend
-    steps:
-      -
-        name: Set ghcr repository name
-        id: set-ghcr-repository
-        run: |
-          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
-          echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-      -
-        name: Setup qpdf image
-        id: qpdf-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
-
-          echo ${build_json}
-
-          echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
-      -
-        name: Setup psycopg2 image
-        id: psycopg2-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
-
-          echo ${build_json}
-
-          echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
-      -
-        name: Setup pikepdf image
-        id: pikepdf-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
-
-          echo ${build_json}
-
-          echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
-      -
-        name: Setup jbig2enc image
-        id: jbig2enc-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
-
-          echo ${build_json}
-
-          echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
-
-    outputs:
-
-      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
-
-      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
-
-      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
-
-      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
-
-      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
-
-  # build and push image to docker hub.
   build-docker-image:
+    name: Build Docker image for ${{ github.ref_name }}
     runs-on: ubuntu-22.04
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
     concurrency:
       group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
       cancel-in-progress: true
     needs:
-      - prepare-docker-build
+      - tests-backend
+      - tests-frontend
     steps:
       -
         name: Check pushing to Docker Hub
-        id: docker-hub
+        id: push-other-places
         # Only push to Dockerhub from the main repo AND the ref is either:
         #  main
         #  dev
@@ -288,22 +320,29 @@ jobs:
         #  a tag
         # Otherwise forks would require a Docker Hub account and secrets setup
         run: |
-          if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
+          if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
             echo "Enabling DockerHub image push"
             echo "enable=true" >> $GITHUB_OUTPUT
           else
             echo "Not pushing to DockerHub"
             echo "enable=false" >> $GITHUB_OUTPUT
           fi
+      -
+        name: Set ghcr repository name
+        id: set-ghcr-repository
+        run: |
+          ghcr_name=$(echo "${{ github.repository }}" | awk '{ print tolower($0) }')
+          echo "Name is ${ghcr_name}"
+          echo "ghcr-repository=${ghcr_name}" >> $GITHUB_OUTPUT
       -
         name: Gather Docker metadata
         id: docker-meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: |
-            ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}
-            name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
-            name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
+            ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}
+            name=paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
+            name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
           tags: |
             # Tag branches with branch name
             type=ref,event=branch
@@ -313,60 +352,59 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      # If https://github.com/docker/buildx/issues/1044 is resolved,
+      # the append input with a native arm64 arch could be used to
+      # significantly speed up building
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
       -
-        name: Login to Github Container Registry
-        uses: docker/login-action@v2
+        name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         # Don't attempt to login is not pushing to Docker Hub
-        if: steps.docker-hub.outputs.enable == 'true'
+        if: steps.push-other-places.outputs.enable == 'true'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Login to Quay.io
-        uses: docker/login-action@v2
-        # Don't attempt to login is not pushing to Docker Hub
-        if: steps.docker-hub.outputs.enable == 'true'
+        uses: docker/login-action@v3
+        # Don't attempt to login is not pushing to Quay.io
+        if: steps.push-other-places.outputs.enable == 'true'
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_ROBOT_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
-          build-args: |
-            JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
-            QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-            PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
-            PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
-          # Get cache layers from this branch, then dev, then main
+          # Get cache layers from this branch, then dev
           # This allows new branches to get at least some cache benefits, generally from dev
           cache-from: |
-            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
-            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev
-            type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main
+            type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:dev
           cache-to: |
-            type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,mode=max,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
       -
         name: Inspect image
         run: |
@@ -382,15 +420,18 @@ jobs:
         with:
           name: frontend-compiled
           path: src/documents/static/frontend/
+          retention-days: 7
 
   build-release:
+    name: "Build Release"
     needs:
       - build-docker-image
+      - documentation
     runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       -
         name: Set up Python
         id: setup-python
@@ -402,11 +443,17 @@ jobs:
       -
         name: Install pipenv + tools
         run: |
-          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
+          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
       -
         name: Install Python dependencies
         run: |
           pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev
+      -
+        name: Patch whitenoise
+        run: |
+          curl --fail --silent --show-error --location --output 484.patch https://github.com/evansd/whitenoise/pull/484.patch
+          patch -d $(pipenv --venv)/lib/python3.10/site-packages --verbose -p2 < 484.patch
+          rm 484.patch
       -
         name: Install system dependencies
         run: |
@@ -490,8 +537,10 @@ jobs:
         with:
           name: release
           path: dist/paperless-ngx.tar.xz
+          retention-days: 7
 
   publish-release:
+    name: "Publish Release"
     runs-on: ubuntu-22.04
     outputs:
       prerelease: ${{ steps.get_version.outputs.prerelease }}
@@ -541,6 +590,7 @@ jobs:
           asset_content_type: application/x-xz
 
   append-changelog:
+    name: "Append Changelog"
     runs-on: ubuntu-22.04
     needs:
       - publish-release
@@ -548,7 +598,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: main
       -
@@ -561,7 +611,7 @@ jobs:
       -
         name: Install pipenv + tools
         run: |
-          pip install --upgrade --user pipenv==${DEFAULT_PIP_ENV_VERSION} setuptools wheel
+          pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel
       -
         name: Append Changelog to docs
         id: append-Changelog
@@ -582,7 +632,7 @@ jobs:
           git push origin ${{ needs.publish-release.outputs.version }}-changelog
       -
         name: Create Pull Request
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           script: |
             const { repo, owner } = context.repo;

.github/workflows/cleanup-tags.yml

@@ -12,9 +12,6 @@ on:
   push:
     paths:
       - ".github/workflows/cleanup-tags.yml"
-      - ".github/scripts/cleanup-tags.py"
-      - ".github/scripts/github.py"
-      - ".github/scripts/common.py"
 
 concurrency:
   group: registry-tags-cleanup
@@ -26,58 +23,48 @@ jobs:
     if: github.repository_owner == 'paperless-ngx'
     runs-on: ubuntu-22.04
     strategy:
+      fail-fast: false
       matrix:
-        include:
-          - primary-name: "paperless-ngx"
-            cache-name: "paperless-ngx/builder/cache/app"
-
-          - primary-name: "paperless-ngx/builder/qpdf"
-            cache-name: "paperless-ngx/builder/cache/qpdf"
-
-          - primary-name: "paperless-ngx/builder/pikepdf"
-            cache-name: "paperless-ngx/builder/cache/pikepdf"
-
-          - primary-name: "paperless-ngx/builder/jbig2enc"
-            cache-name: "paperless-ngx/builder/cache/jbig2enc"
-
-          - primary-name: "paperless-ngx/builder/psycopg2"
-            cache-name: "paperless-ngx/builder/cache/psycopg2"
+        primary-name: ["paperless-ngx", "paperless-ngx/builder/cache/app"]
     env:
       # Requires a personal access token with the OAuth scope delete:packages
       TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
     steps:
       -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Login to Github Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      -
-        name: Install Python libraries
-        run: |
-          python -m pip install httpx docker
-      #
-      # Clean up primary package
-      #
-      -
-        name: Cleanup for package "${{ matrix.primary-name }}"
+        name: Clean temporary images
         if: "${{ env.TOKEN != '' }}"
-        run: |
-          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --is-manifest --delete "${{ matrix.primary-name }}"
-      #
-      # Clean up registry cache package
-      #
+        uses: stumpylog/image-cleaner-action/ephemeral@v0.4.0
+        with:
+          token: "${{ env.TOKEN }}"
+          owner: "${{ github.repository_owner }}"
+          is_org: "true"
+          package_name: "${{ matrix.primary-name }}"
+          scheme: "branch"
+          repo_name: "paperless-ngx"
+          match_regex: "feature-"
+          do_delete: "true"
+
+  cleanup-untagged-images:
+    name: Cleanup Untagged Images Tags for ${{ matrix.primary-name }}
+    if: github.repository_owner == 'paperless-ngx'
+    runs-on: ubuntu-22.04
+    needs:
+      - cleanup-images
+    strategy:
+      fail-fast: false
+      matrix:
+        primary-name: ["paperless-ngx", "paperless-ngx/builder/cache/app"]
+    env:
+      # Requires a personal access token with the OAuth scope delete:packages
+      TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}
+    steps:
       -
-        name: Cleanup for package "${{ matrix.cache-name }}"
+        name: Clean untagged images
         if: "${{ env.TOKEN != '' }}"
-        run: |
-          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
+        uses: stumpylog/image-cleaner-action/untagged@v0.4.0
+        with:
+          token: "${{ env.TOKEN }}"
+          owner: "${{ github.repository_owner }}"
+          is_org: "true"
+          package_name: "${{ matrix.primary-name }}"
+          do_delete: "true"

.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/crowdin.yml

@@ -0,0 +1,33 @@
+name: Crowdin Action
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '2 */12 * * *'
+  push:
+    paths: [
+      'src/locale/**',
+      'src-ui/src/locale/**'
+    ]
+    branches: [ dev ]
+
+jobs:
+  synchronize-with-crowdin:
+    name: Crowdin Sync
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: crowdin action
+      uses: crowdin/github-action@v1
+      with:
+        upload_translations: false
+        download_translations: true
+        crowdin_branch_name: 'dev'
+        localization_branch_name: l10n_dev
+        pull_request_labels: 'skip-changelog, translation'
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+        CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}

.github/workflows/installer-library.yml

@@ -1,310 +0,0 @@
-# This workflow will run to update the installer library of
-# Docker images.  These are the images which provide updated wheels
-# .deb installation packages or maybe just some compiled library
-
-name: Build Image Library
-
-on:
-  push:
-    # Must match one of these branches AND one of the paths
-    # to be triggered
-    branches:
-      - "main"
-      - "dev"
-      - "library-*"
-      - "feature-*"
-    paths:
-      # Trigger the workflow if a Dockerfile changed
-      - "docker-builders/**"
-      # Trigger if a package was updated
-      - ".build-config.json"
-      - "Pipfile.lock"
-      # Also trigger on workflow changes related to the library
-      - ".github/workflows/installer-library.yml"
-      - ".github/workflows/reusable-workflow-builder.yml"
-      - ".github/scripts/**"
-
-# Set a workflow level concurrency group so primary workflow
-# can wait for this to complete if needed
-# DO NOT CHANGE without updating main workflow group
-concurrency:
-  group: build-installer-library
-  cancel-in-progress: false
-
-jobs:
-  prepare-docker-build:
-    name: Prepare Docker Image Version Data
-    runs-on: ubuntu-22.04
-    steps:
-      -
-        name: Set ghcr repository name
-        id: set-ghcr-repository
-        run: |
-          ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
-          echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-      -
-        name: Install jq
-        run: |
-          sudo apt-get update
-          sudo apt-get install jq
-      -
-        name: Setup qpdf image
-        id: qpdf-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
-
-          echo ${build_json}
-
-          echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
-      -
-        name: Setup psycopg2 image
-        id: psycopg2-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
-
-          echo ${build_json}
-
-          echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
-      -
-        name: Setup pikepdf image
-        id: pikepdf-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
-
-          echo ${build_json}
-
-          echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
-      -
-        name: Setup jbig2enc image
-        id: jbig2enc-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
-
-          echo ${build_json}
-
-          echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
-      -
-        name: Setup other versions
-        id: cache-bust-setup
-        run: |
-          pillow_version=$(jq -r '.default.pillow.version | gsub("=";"")' Pipfile.lock)
-          lxml_version=$(jq -r '.default.lxml.version | gsub("=";"")' Pipfile.lock)
-
-          echo "Pillow is ${pillow_version}"
-          echo "lxml is ${lxml_version}"
-
-          echo "pillow-version=${pillow_version}" >> $GITHUB_OUTPUT
-          echo "lxml-version=${lxml_version}" >> $GITHUB_OUTPUT
-
-    outputs:
-
-      ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
-
-      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
-
-      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
-
-      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
-
-      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json }}
-
-      pillow-version: ${{ steps.cache-bust-setup.outputs.pillow-version }}
-
-      lxml-version: ${{ steps.cache-bust-setup.outputs.lxml-version }}
-
-  build-qpdf-debs:
-    name: qpdf
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.qpdf
-      build-platforms: linux/amd64
-      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
-      build-args: |
-        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-
-  build-jbig2enc:
-    name: jbig2enc
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.jbig2enc
-      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
-      build-args: |
-        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
-
-  build-psycopg2-wheel:
-    name: psycopg2
-    needs:
-      - prepare-docker-build
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.psycopg2
-      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
-      build-args: |
-        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
-
-  build-pikepdf-wheel:
-    name: pikepdf
-    needs:
-      - prepare-docker-build
-      - build-qpdf-debs
-    uses: ./.github/workflows/reusable-workflow-builder.yml
-    with:
-      dockerfile: ./docker-builders/Dockerfile.pikepdf
-      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
-      build-args: |
-        REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }}
-        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
-        PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
-        LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}
-
-  commit-binary-files:
-    name: Store installers
-    needs:
-      - prepare-docker-build
-      - build-qpdf-debs
-      - build-jbig2enc
-      - build-psycopg2-wheel
-      - build-pikepdf-wheel
-    runs-on: ubuntu-22.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-        with:
-          ref: binary-library
-      -
-        name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-      -
-        name: Install system dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends tree
-      -
-        name: Extract qpdf files
-        run: |
-          version=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-          tag=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
-
-          docker pull --quiet ${tag}
-          docker create --name qpdf-extract ${tag}
-
-          mkdir --parents qpdf/${version}/amd64
-          docker cp qpdf-extract:/usr/src/qpdf/${version}/amd64 qpdf/${version}
-
-          mkdir --parents qpdf/${version}/arm64
-          docker cp qpdf-extract:/usr/src/qpdf/${version}/arm64 qpdf/${version}
-
-          mkdir --parents qpdf/${version}/armv7
-          docker cp qpdf-extract:/usr/src/qpdf/${version}/armv7 qpdf/${version}
-      -
-        name: Extract psycopg2 files
-        run: |
-          version=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
-          tag=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
-
-          docker pull --quiet --platform linux/amd64 ${tag}
-          docker create --platform linux/amd64 --name psycopg2-extract ${tag}
-          mkdir --parents psycopg2/${version}/amd64
-          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/amd64
-          mv psycopg2/${version}/amd64/wheels/* psycopg2/${version}/amd64
-          rm -r psycopg2/${version}/amd64/wheels/
-          docker rm psycopg2-extract
-
-          docker pull --quiet --platform linux/arm64 ${tag}
-          docker create --platform linux/arm64 --name psycopg2-extract ${tag}
-          mkdir --parents psycopg2/${version}/arm64
-          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/arm64
-          mv psycopg2/${version}/arm64/wheels/* psycopg2/${version}/arm64
-          rm -r psycopg2/${version}/arm64/wheels/
-          docker rm psycopg2-extract
-
-          docker pull --quiet --platform linux/arm/v7 ${tag}
-          docker create --platform linux/arm/v7 --name psycopg2-extract ${tag}
-          mkdir --parents psycopg2/${version}/armv7
-          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/armv7
-          mv psycopg2/${version}/armv7/wheels/* psycopg2/${version}/armv7
-          rm -r psycopg2/${version}/armv7/wheels/
-          docker rm psycopg2-extract
-      -
-        name: Extract pikepdf files
-        run: |
-          version=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
-          tag=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
-
-          docker pull --quiet --platform linux/amd64 ${tag}
-          docker create --platform linux/amd64 --name pikepdf-extract ${tag}
-          mkdir --parents pikepdf/${version}/amd64
-          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/amd64
-          mv pikepdf/${version}/amd64/wheels/* pikepdf/${version}/amd64
-          rm -r pikepdf/${version}/amd64/wheels/
-          docker rm pikepdf-extract
-
-          docker pull --quiet --platform linux/arm64 ${tag}
-          docker create --platform linux/arm64 --name pikepdf-extract ${tag}
-          mkdir --parents pikepdf/${version}/arm64
-          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/arm64
-          mv pikepdf/${version}/arm64/wheels/* pikepdf/${version}/arm64
-          rm -r pikepdf/${version}/arm64/wheels/
-          docker rm pikepdf-extract
-
-          docker pull --quiet --platform linux/arm/v7 ${tag}
-          docker create --platform linux/arm/v7 --name pikepdf-extract ${tag}
-          mkdir --parents pikepdf/${version}/armv7
-          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/armv7
-          mv pikepdf/${version}/armv7/wheels/* pikepdf/${version}/armv7
-          rm -r pikepdf/${version}/armv7/wheels/
-          docker rm pikepdf-extract
-      -
-        name: Extract jbig2enc files
-        run: |
-          version=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
-          tag=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
-
-          docker pull --quiet --platform linux/amd64 ${tag}
-          docker create --platform linux/amd64 --name jbig2enc-extract ${tag}
-          mkdir --parents jbig2enc/${version}/amd64
-          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/amd64/
-          mv jbig2enc/${version}/amd64/build/* jbig2enc/${version}/amd64/
-          docker rm jbig2enc-extract
-
-          docker pull --quiet --platform linux/arm64 ${tag}
-          docker create --platform linux/arm64 --name jbig2enc-extract ${tag}
-          mkdir --parents jbig2enc/${version}/arm64
-          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/arm64
-          mv jbig2enc/${version}/arm64/build/* jbig2enc/${version}/arm64/
-          docker rm jbig2enc-extract
-
-          docker pull --quiet --platform linux/arm/v7 ${tag}
-          docker create --platform linux/arm/v7 --name jbig2enc-extract ${tag}
-          mkdir --parents jbig2enc/${version}/armv7
-          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/armv7
-          mv jbig2enc/${version}/armv7/build/* jbig2enc/${version}/armv7/
-          docker rm jbig2enc-extract
-      -
-        name: Show file structure
-        run: |
-          tree .
-      -
-        name: Commit files
-        run: |
-          git config --global user.name "github-actions"
-          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add pikepdf/ qpdf/ psycopg2/ jbig2enc/
-          git commit -m "Updating installer packages" || true
-          git push origin || true

.github/workflows/project-actions.yml

@@ -1,10 +1,6 @@
 name: Project Automations
 
 on:
-  issues:
-    types:
-      - opened
-      - reopened
   pull_request_target: #_target allows access to secrets
     types:
       - opened
@@ -16,25 +12,7 @@ on:
 permissions:
   contents: read
 
-env:
-  todo: Todo
-  done: Done
-  in_progress: In Progress
-
 jobs:
-  issue_opened_or_reopened:
-    name: issue_opened_or_reopened
-    runs-on: ubuntu-22.04
-    if: github.event_name == 'issues' && (github.event.action == 'opened' || github.event.action == 'reopened')
-    steps:
-      - name: Add issue to project and set status to ${{ env.todo }}
-        uses: leonsteinhaeuser/project-beta-automations@v2.1.0
-        with:
-          gh_token: ${{ secrets.GH_TOKEN }}
-          organization: paperless-ngx
-          project_id: 2
-          resource_node_id: ${{ github.event.issue.node_id }}
-          status_value: ${{ env.todo }} # Target status
   pr_opened_or_reopened:
     name: pr_opened_or_reopened
     runs-on: ubuntu-22.04
@@ -43,14 +21,6 @@ jobs:
       pull-requests: write
     if: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'reopened') && github.event.pull_request.user.login != 'dependabot'
     steps:
-      - name: Add PR to project and set status to "Needs Review"
-        uses: leonsteinhaeuser/project-beta-automations@v2.1.0
-        with:
-          gh_token: ${{ secrets.GH_TOKEN }}
-          organization: paperless-ngx
-          project_id: 2
-          resource_node_id: ${{ github.event.pull_request.node_id }}
-          status_value: "Needs Review" # Target status
       - name: Label PR with release-drafter
         uses: release-drafter/release-drafter@v5
         env:

.github/workflows/repo-maintenance.yml

@@ -8,6 +8,7 @@ on:
 permissions:
   issues: write
   pull-requests: write
+  discussions: write
 
 concurrency:
   group: lock
@@ -19,9 +20,9 @@ jobs:
     steps:
       - uses: actions/stale@v8
         with:
-          days-before-stale: 30
-          days-before-close: 7
-          only-labels: 'cant-reproduce'
+          days-before-stale: 7
+          days-before-close: 14
+          any-of-labels: 'cant-reproduce,not a bug'
           stale-issue-label: stale
           stale-pr-label: stale
           stale-issue-message: >
@@ -32,10 +33,11 @@ jobs:
     name: 'Lock Old Threads'
     runs-on: ubuntu-latest
     steps:
-      - uses: dessant/lock-threads@v4
+      - uses: dessant/lock-threads@v5
         with:
           issue-inactive-days: '30'
           pr-inactive-days: '30'
+          discussion-inactive-days: '30'
           log-output: true
           issue-comment: >
             This issue has been automatically locked since there
@@ -45,3 +47,63 @@ jobs:
             This pull request has been automatically locked since there
             has not been any recent activity after it was closed.
             Please open a new discussion or issue for related concerns.
+          discussion-comment: >
+            This discussion has been automatically locked since there
+            has not been any recent activity after it was closed.
+            Please open a new discussion for related concerns.
+  close-answered-discussions:
+    name: 'Close Answered Discussions'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            function sleep(ms) {
+              return new Promise(resolve => setTimeout(resolve, ms));
+            }
+
+            const query = `query($owner:String!, $name:String!) {
+              repository(owner:$owner, name:$name){
+                discussions(first:100, answered:true, states:[OPEN]) {
+                  nodes {
+                    id,
+                    number
+                  }
+                }
+              }
+            }`;
+            const variables = {
+              owner: context.repo.owner,
+              name: context.repo.repo,
+            }
+            const result = await github.graphql(query, variables)
+
+            console.log(`Found ${result.repository.discussions.nodes.length} open answered discussions`)
+
+            for (const discussion of result.repository.discussions.nodes) {
+              console.log(`Closing dicussion #${discussion.number} (${discussion.id})`)
+
+              const addCommentMutation = `mutation($discussion:ID!, $body:String!) {
+                addDiscussionComment(input:{discussionId:$discussion, body:$body}) {
+                  clientMutationId
+                }
+              }`;
+              const commentVariables = {
+                discussion: discussion.id,
+                body: 'This discussion has been automatically closed because it was marked as answered.',
+              }
+              await github.graphql(addCommentMutation, commentVariables)
+
+              const closeDiscussionMutation = `mutation($discussion:ID!, $reason:DiscussionCloseReason!) {
+                closeDiscussion(input:{discussionId:$discussion, reason:$reason}) {
+                  clientMutationId
+                }
+              }`;
+              const closeVariables = {
+                discussion: discussion.id,
+                reason: "RESOLVED",
+              }
+              await github.graphql(closeDiscussionMutation, closeVariables)
+
+              await sleep(1000)
+            }

.github/workflows/reusable-workflow-builder.yml

@@ -1,57 +0,0 @@
-name: Reusable Image Builder
-
-on:
-  workflow_call:
-    inputs:
-      dockerfile:
-        required: true
-        type: string
-      build-json:
-        required: true
-        type: string
-      build-args:
-        required: false
-        default: ""
-        type: string
-      build-platforms:
-        required: false
-        default: linux/amd64,linux/arm64,linux/arm/v7
-        type: string
-
-concurrency:
-  group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
-  cancel-in-progress: false
-
-jobs:
-  build-image:
-    name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
-    runs-on: ubuntu-22.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Login to Github Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      -
-        name: Build ${{ fromJSON(inputs.build-json).name }}
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ${{ inputs.dockerfile }}
-          tags: ${{ fromJSON(inputs.build-json).image_tag }}
-          platforms: ${{ inputs.build-platforms }}
-          build-args: ${{ inputs.build-args }}
-          push: true
-          cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}
-          cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }}

.pre-commit-config.yaml

@@ -5,7 +5,7 @@
 repos:
   # General hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-docstring-first
       - id: check-json
@@ -27,7 +27,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.7.1"
+    rev: 'v3.1.0'
     hooks:
       - id: prettier
         types_or:
@@ -36,17 +36,17 @@ repos:
           - markdown
         exclude: "(^Pipfile\\.lock$)"
   # Python hooks
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.259'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.1.5'
     hooks:
       - id: ruff
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 23.11.0
     hooks:
       - id: black
   # Dockerfile hooks
   - repo: https://github.com/AleksaC/hadolint-py
-    rev: v2.10.0
+    rev: v2.12.0.3
     hooks:
       - id: hadolint
   # Shell script hooks
@@ -57,6 +57,6 @@ repos:
         args:
           - "--tab"
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: "v0.9.0.2"
+    rev: "v0.9.0.6"
     hooks:
       - id: shellcheck

.prettierrc

@@ -1,4 +1,16 @@
-# https://prettier.io/docs/en/options.html#semicolons
-semi: false
-# https://prettier.io/docs/en/options.html#quotes
-singleQuote: true
+{
+    # https://prettier.io/docs/en/options.html#semicolons
+    "semi": false,
+    # https://prettier.io/docs/en/options.html#quotes
+    "singleQuote": true,
+    # https://prettier.io/docs/en/options.html#trailing-commas
+    "trailingComma": "es5",
+    "overrides": [
+        {
+            "files": ["index.md", "administration.md"],
+            "options": {
+                "tabWidth": 4
+            }
+        }
+    ]
+}

.python-version

@@ -1 +1 @@
-3.8.16
+3.9.18

.ruff.toml

@@ -1,14 +1,14 @@
 # https://beta.ruff.rs/docs/settings/
 # https://beta.ruff.rs/docs/rules/
-select = ["F", "E", "W", "UP", "COM", "DJ", "EXE", "ISC", "ICN", "G201", "INP", "PIE", "RSE", "SIM", "TID", "PLC", "PLE", "RUF"]
+extend-select = ["I", "W", "UP", "COM", "DJ", "EXE", "ISC", "ICN", "G201", "INP", "PIE", "RSE", "SIM", "TID", "PLC", "PLE", "RUF"]
 # TODO PTH
-ignore = ["DJ001", "SIM105"]
+ignore = ["DJ001", "SIM105", "RUF012"]
 fix = true
 line-length = 88
 respect-gitignore = true
 src = ["src"]
-target-version = "py38"
-format = "grouped"
+target-version = "py39"
+output-format = "grouped"
 show-fixes = true
 
 [per-file-ignores]

CONTRIBUTING.md

@@ -11,7 +11,7 @@ If you want to implement something big:
 
 ## Python
 
-Paperless supports python 3.8 and 3.9. We format Python code with [Black](https://github.com/psf/black).
+Paperless supports python 3.9 - 3.11. We format Python code with [Black](https://github.com/psf/black).
 
 ## Branches
 
@@ -45,7 +45,7 @@ Examples of `non-trivial` PRs might include:
 
 - Additional features
 - Large changes to many distinct files
-- Breaking or depreciation of existing features
+- Breaking or deprecation of existing features
 
 Our community review process for `non-trivial` PRs is the following:
 
@@ -58,6 +58,13 @@ Our community review process for `non-trivial` PRs is the following:
 
 This process might be slow as community members have different schedules and time to dedicate to the Paperless project. However it ensures community code reviews are as brilliantly thorough as they once were with @jonaswinkler.
 
+# AI-Generated Code
+
+This project does not specifically prohibit the use of AI-generated code _during the process_ of creating a PR, however:
+
+1. Any code present in the final PR that was generated using AI sources should be clearly attributed as such and must not violate copyright protections.
+2. We will not accept PRs that are entirely or mostly AI-derived.
+
 # Translating Paperless-ngx
 
 Some notes about translation:

Dockerfile

@@ -1,11 +1,11 @@
-# syntax=docker/dockerfile:1.4
+# syntax=docker/dockerfile:1
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md
 
 # Stage: compile-frontend
 # Purpose: Compiles the frontend
 # Notes:
 #  - Does NPM stuff with Typescript and such
-FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
+FROM --platform=$BUILDPLATFORM docker.io/node:20-bookworm-slim AS compile-frontend
 
 COPY ./src-ui /src/src-ui
 
@@ -21,7 +21,7 @@ RUN set -eux \
 # Comments:
 #  - pipenv dependencies are not left in the final image
 #  - pipenv can't touch the final image somehow
-FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base
+FROM --platform=$BUILDPLATFORM docker.io/python:3.11-alpine as pipenv-base
 
 WORKDIR /usr/src/pipenv
 
@@ -29,7 +29,7 @@ COPY Pipfile* ./
 
 RUN set -eux \
   && echo "Installing pipenv" \
-    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.3.20 \
+    && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.10.24 \
   && echo "Generating requirement.txt" \
     && pipenv requirements > requirements.txt
 
@@ -37,7 +37,7 @@ RUN set -eux \
 # Purpose: The final image
 # Comments:
 #  - Don't leave anything extra in here
-FROM python:3.9-slim-bullseye as main-app
+FROM docker.io/python:3.11-slim-bookworm as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
 LABEL org.opencontainers.image.documentation="https://docs.paperless-ngx.com/"
@@ -47,6 +47,14 @@ LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
+# Buildx provided, must be defined to use though
+ARG TARGETARCH
+
+# Can be workflow provided, defaults set for manual building
+ARG JBIG2ENC_VERSION=0.29
+ARG QPDF_VERSION=11.6.3
+ARG GS_VERSION=10.02.0
+
 #
 # Begin installation and configuration
 # Order the steps below from least often changed to most
@@ -67,23 +75,11 @@ ARG RUNTIME_PACKAGES="\
   gnupg \
   icc-profiles-free \
   imagemagick \
-  # Image processing
-  liblept5 \
-  liblcms2-2 \
-  libtiff5 \
-  libfreetype6 \
-  libwebp6 \
-  libopenjp2-7 \
-  libimagequant0 \
-  libraqm0 \
-  libjpeg62-turbo \
   # PostgreSQL
   libpq5 \
   postgresql-client \
   # MySQL / MariaDB
   mariadb-client \
-  # For Numpy
-  libatlas3-base \
   # OCRmyPDF dependencies
   tesseract-ocr \
   tesseract-ocr-eng \
@@ -93,11 +89,12 @@ ARG RUNTIME_PACKAGES="\
   tesseract-ocr-spa \
   unpaper \
   pngquant \
-  # pikepdf / qpdf
   jbig2dec \
+  # lxml
   libxml2 \
   libxslt1.1 \
-  libgnutls30 \
+  # itself
+  qpdf \
   # Mime type detection
   file \
   libmagic1 \
@@ -105,9 +102,7 @@ ARG RUNTIME_PACKAGES="\
   zlib1g \
   # Barcode splitter
   libzbar0 \
-  poppler-utils \
-  # RapidFuzz on armv7
-  libatomic1"
+  poppler-utils"
 
 # Install basic runtime packages.
 # These change very infrequently
@@ -115,7 +110,37 @@ RUN set -eux \
   echo "Installing system packages" \
     && apt-get update \
     && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} \
-    && rm -rf /var/lib/apt/lists/* \
+    && echo "Installing pre-built updates" \
+      && echo "Installing qpdf ${QPDF_VERSION}" \
+        && curl --fail --silent --show-error --location \
+          --output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+        && dpkg --install ./libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+        && dpkg --install ./qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
+      && echo "Installing Ghostscript ${GS_VERSION}" \
+        && curl --fail --silent --show-error --location \
+          --output libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && curl --fail --silent --show-error --location \
+          --output libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
+          https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
+        && dpkg --install ./libgs10-common_${GS_VERSION}.dfsg-2_all.deb \
+        && dpkg --install ./libgs10_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+        && dpkg --install ./ghostscript_${GS_VERSION}.dfsg-2_${TARGETARCH}.deb \
+      && echo "Installing jbig2enc" \
+        && curl --fail --silent --show-error --location \
+          --output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
+          https://github.com/paperless-ngx/builder/releases/download/jbig2enc-${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
+        && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
+      && echo "Cleaning up image layer" \
+        && rm --force --verbose *.deb \
+    && rm --recursive --force --verbose /var/lib/apt/lists/* \
   && echo "Installing supervisor" \
     && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor==4.2.5
 
@@ -166,43 +191,6 @@ RUN set -eux \
     && chmod +x install_management_commands.sh \
     && ./install_management_commands.sh
 
-# Buildx provided, must be defined to use though
-ARG TARGETARCH
-ARG TARGETVARIANT
-
-# Workflow provided, defaults set for manual building
-ARG JBIG2ENC_VERSION=0.29
-ARG QPDF_VERSION=11.3.0
-ARG PIKEPDF_VERSION=7.1.1
-ARG PSYCOPG2_VERSION=2.9.5
-
-# Install the built packages from the installer library images
-# These change sometimes
-RUN set -eux \
-  && echo "Getting binaries" \
-    && mkdir paperless-ngx \
-    && curl --fail --silent --show-error --output paperless-ngx.tar.gz --location https://github.com/paperless-ngx/paperless-ngx/archive/ba28a1e16c27d121b644b4f6bdb78855a2850561.tar.gz \
-    && tar -xf paperless-ngx.tar.gz --directory paperless-ngx --strip-components=1 \
-    && cd paperless-ngx \
-    # Setting a specific revision ensures we know what this installed
-    # and ensures cache breaking on changes
-  && echo "Installing jbig2enc" \
-    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/jbig2 /usr/local/bin/ \
-    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/libjbig2enc* /usr/local/lib/ \
-  && echo "Installing qpdf" \
-    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
-    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
-  && echo "Installing pikepdf and dependencies" \
-    && python3 -m pip install --no-cache-dir ./pikepdf/${PIKEPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.whl \
-    && python3 -m pip list \
-  && echo "Installing psycopg2" \
-    && python3 -m pip install --no-cache-dir ./psycopg2/${PSYCOPG2_VERSION}/${TARGETARCH}${TARGETVARIANT}/psycopg2*.whl \
-    && python3 -m pip list \
-  && echo "Cleaning up image layer" \
-    && cd ../ \
-    && rm -rf paperless-ngx \
-    && rm paperless-ngx.tar.gz
-
 WORKDIR /usr/src/paperless/src/
 
 # Python dependencies
@@ -214,44 +202,61 @@ COPY --from=pipenv-base /usr/src/pipenv/requirements.txt ./
 ARG BUILD_PACKAGES="\
   build-essential \
   git \
+  # https://www.psycopg.org/docs/install.html#prerequisites
+  libpq-dev \
+  # https://github.com/PyMySQL/mysqlclient#linux
   default-libmysqlclient-dev \
-  python3-dev"
+  pkg-config"
 
-RUN set -eux \
+# hadolint ignore=DL3042
+RUN --mount=type=cache,target=/root/.cache/pip/,id=pip-cache \
+  set -eux \
   && echo "Installing build system packages" \
     && apt-get update \
     && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
     && python3 -m pip install --no-cache-dir --upgrade wheel \
   && echo "Installing Python requirements" \
-    && python3 -m pip install --default-timeout=1000 --no-cache-dir --requirement requirements.txt \
+    && python3 -m pip install --default-timeout=1000 --requirement requirements.txt \
+  && echo "Patching whitenoise for compression speedup" \
+    && curl --fail --silent --show-error --location --output 484.patch https://github.com/evansd/whitenoise/pull/484.patch \
+    && patch -d /usr/local/lib/python3.11/site-packages --verbose -p2 < 484.patch \
+    && rm 484.patch \
   && echo "Installing NLTK data" \
     && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" snowball_data \
     && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" stopwords \
     && python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" punkt \
   && echo "Cleaning up image" \
-    && apt-get -y purge ${BUILD_PACKAGES} \
-    && apt-get -y autoremove --purge \
+    && apt-get --yes purge ${BUILD_PACKAGES} \
+    && apt-get --yes autoremove --purge \
     && apt-get clean --yes \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm -rf /tmp/* \
-    && rm -rf /var/tmp/* \
-    && rm -rf /var/cache/apt/archives/* \
-    && truncate -s 0 /var/log/*log
+    && rm --recursive --force --verbose /var/lib/apt/lists/* \
+    && rm --recursive --force --verbose /tmp/* \
+    && rm --recursive --force --verbose /var/tmp/* \
+    && rm --recursive --force --verbose /var/cache/apt/archives/* \
+    && truncate --size 0 /var/log/*log
 
 # copy backend
-COPY ./src ./
+COPY --chown=1000:1000 ./src ./
 
 # copy frontend
-COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
+COPY --from=compile-frontend --chown=1000:1000 /src/src/documents/static/frontend/ ./documents/static/frontend/
 
 # add users, setup scripts
 # Mount the compiled frontend to expected location
 RUN set -eux \
-  && addgroup --gid 1000 paperless \
-  && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
-  && chown -R paperless:paperless /usr/src/paperless \
-  && gosu paperless python3 manage.py collectstatic --clear --no-input --link \
-  && gosu paperless python3 manage.py compilemessages
+  && echo "Setting up user/group" \
+    && addgroup --gid 1000 paperless \
+    && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
+  && echo "Creating volume directories" \
+    && mkdir --parents --verbose /usr/src/paperless/data \
+    && mkdir --parents --verbose /usr/src/paperless/media \
+    && mkdir --parents --verbose /usr/src/paperless/consume \
+    && mkdir --parents --verbose /usr/src/paperless/export \
+  && echo "Adjusting all permissions" \
+    && chown --from root:root --changes --recursive paperless:paperless /usr/src/paperless \
+  && echo "Collecting static files" \
+    && gosu paperless python3 manage.py collectstatic --clear --no-input --link \
+    && gosu paperless python3 manage.py compilemessages
 
 VOLUME ["/usr/src/paperless/data", \
         "/usr/src/paperless/media", \

Pipfile

@@ -3,82 +3,78 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-[[source]]
-url = "https://www.piwheels.org/simple"
-verify_ssl = true
-name = "piwheels"
-
 [packages]
 dateparser = "~=1.1"
-django = "~=4.1"
-django-cors-headers = "*"
+# WARNING: django does not use semver.
+#          Only patch versions are guaranteed to not introduce breaking changes.
+django = "~=4.2.7"
+django-auditlog = "*"
 django-celery-results = "*"
 django-compression-middleware = "*"
-django-guardian = "*"
+django-cors-headers = "*"
 django-extensions = "*"
-django-filter = "~=22.1"
+django-filter = "~=23.3"
+django-guardian = "*"
+django-multiselectfield = "*"
 djangorestframework = "~=3.14"
 djangorestframework-guardian = "*"
-django-ipware = "*"
+drf-writable-nested = "*"
+bleach = "*"
+celery = {extras = ["redis"], version = "*"}
+channels = "~=4.0"
+channels-redis = "*"
+concurrent-log-handler = "*"
 filelock = "*"
+flower = "*"
+gotenberg-client = "*"
 gunicorn = "*"
 imap-tools = "*"
+inotifyrecursive = "~=0.3"
 langdetect = "*"
+mysqlclient = "*"
+nltk = "*"
+ocrmypdf = "~=15.0"
 pathvalidate = "*"
-pillow = "~=9.4"
-pikepdf = "*"
-python-gnupg = "*"
-python-dotenv = "*"
-python-dateutil = "*"
-python-magic = "*"
+pdf2image = "*"
 psycopg2 = "*"
+python-dateutil = "*"
+python-dotenv = "*"
+python-gnupg = "*"
+python-ipware = "*"
+python-magic = "*"
+pyzbar = "*"
 rapidfuzz = "*"
 redis = {extras = ["hiredis"], version = "*"}
-scikit-learn = "~=1.2"
-numpy = "*"
-whitenoise = "~=6.3"
-watchdog = "~=2.2"
-whoosh="~=2.7"
-inotifyrecursive = "~=0.3"
-ocrmypdf = "~=14.0"
-tqdm = "*"
-tika = "*"
-# TODO: This will sadly also install daphne+dependencies,
-#  which an ASGI server we don't need. Adds about 15MB image size.
-channels = "~=3.0"
-channels-redis = "*"
-uvicorn = {extras = ["standard"], version = "*"}
-concurrent-log-handler = "*"
-"pdfminer.six" = "*"
-pyzbar = "*"
-mysqlclient = "*"
-celery = {extras = ["redis"], version = "*"}
+scikit-learn = "~=1.3"
 setproctitle = "*"
-nltk = "*"
-pdf2image = "*"
-flower = "*"
-bleach = "*"
+tika-client = "*"
+tqdm = "*"
+uvicorn = {extras = ["standard"], version = "*"}
+watchdog = "~=3.0"
+whitenoise = "~=6.6"
+whoosh="~=2.7"
 zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"}
-#
-# Packages locked due to issues (try to check if these are fixed in a release every so often)
-#
-# Pin this until piwheels is building 1.9 (see https://www.piwheels.org/project/scipy/)
-scipy = "==1.8.1"
 
 [dev-packages]
-coveralls = "*"
+# Linting
+black = "*"
+pre-commit = "*"
+ruff = "*"
+# Testing
 factory-boy = "*"
 pytest = "*"
 pytest-cov = "*"
 pytest-django = "*"
+pytest-httpx = "*"
 pytest-env = "*"
 pytest-sugar = "*"
 pytest-xdist = "*"
-black = "*"
-pre-commit = "*"
+pytest-rerunfailures = "*"
 imagehash = "*"
+daphne = "*"
+# Documentation
 mkdocs-material = "*"
-ruff = "*"
+mkdocs-glightbox = "*"
 
 [typing-dev]
 mypy = "*"
@@ -90,12 +86,10 @@ celery-types = "*"
 django-stubs = {extras= ["compatible-mypy"], version="*"}
 types-dateparser = "*"
 types-bleach = "*"
-types-humanfriendly = "*"
 types-redis = "*"
 types-tqdm = "*"
 types-Markdown = "*"
 types-Pygments = "*"
-types-backports = "*"
 types-colorama = "*"
 types-psycopg2 = "*"
 types-setuptools = "*"

README.md

@@ -6,8 +6,11 @@
 [![demo](https://cronitor.io/badges/ve7ItY/production/W5E_B9jkelG9ZbDiNHUPQEVH3MY.svg)](https://demo.paperless-ngx.com)
 
 <p align="center">
-<img src="https://github.com/paperless-ngx/paperless-ngx/raw/main/resources/logo/web/png/Black%20logo%20-%20no%20background.png#gh-light-mode-only" width="50%" />
-<img src="https://github.com/paperless-ngx/paperless-ngx/raw/main/resources/logo/web/png/White%20logo%20-%20no%20background.png#gh-dark-mode-only" width="50%" />
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/paperless-ngx/paperless-ngx/blob/main/resources/logo/web/png/White%20logo%20-%20no%20background.png" width="50%">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/paperless-ngx/paperless-ngx/raw/main/resources/logo/web/png/Black%20logo%20-%20no%20background.png" width="50%">
+    <img src="https://github.com/paperless-ngx/paperless-ngx/raw/main/resources/logo/web/png/Black%20logo%20-%20no%20background.png" width="50%">
+  </picture>
 </p>
 
 <!-- omit in toc -->
@@ -16,8 +19,7 @@
 
 Paperless-ngx is a document management system that transforms your physical documents into a searchable online archive so you can keep, well, _less paper_.
 
-Paperless-ngx forked from [paperless-ng](https://github.com/jonaswinkler/paperless-ng) to continue the great work and distribute responsibility of supporting and advancing the project among a team of people. [Consider joining us!](#community-support) Discussion of this transition can be found in issues
-[#1599](https://github.com/jonaswinkler/paperless-ng/issues/1599) and [#1632](https://github.com/jonaswinkler/paperless-ng/issues/1632).
+Paperless-ngx is the official successor to the original [Paperless](https://github.com/the-paperless-project/paperless) & [Paperless-ng](https://github.com/jonaswinkler/paperless-ng) projects and is designed to distribute the responsibility of advancing and supporting the project among a team of people. [Consider joining us!](#community-support)
 
 A demo is available at [demo.paperless-ngx.com](https://demo.paperless-ngx.com) using login `demo` / `demo`. _Note: demo content is reset frequently and confidential information should not be uploaded._
 
@@ -33,37 +35,19 @@ A demo is available at [demo.paperless-ngx.com](https://demo.paperless-ngx.com)
 
 # Features
 
-![Dashboard](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docs/assets/screenshots/documents-smallcards.png#gh-light-mode-only)
-![Dashboard](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docs/assets/screenshots/documents-smallcards-dark.png#gh-dark-mode-only)
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docs/assets/screenshots/documents-smallcards-dark.png">
+  <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docs/assets/screenshots/documents-smallcards.png">
+  <img src="https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docs/assets/screenshots/documents-smallcards.png">
+</picture>
 
-- Organize and index your scanned documents with tags, correspondents, types, and more.
-- Performs OCR on your documents, adds selectable text to image only documents and adds tags, correspondents and document types to your documents.
-- Supports PDF documents, images, plain text files, and Office documents (Word, Excel, Powerpoint, and LibreOffice equivalents).
-  - Office document support is optional and provided by Apache Tika (see [configuration](https://docs.paperless-ngx.com/configuration/#tika))
-- Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and their format can be configured freely.
-- Single page application front end.
-  - Includes a dashboard that shows basic statistics and has document upload.
-  - Filtering by tags, correspondents, types, and more.
-  - Customizable views can be saved and displayed on the dashboard.
-- Full text search helps you find what you need.
-  - Auto completion suggests relevant words from your documents.
-  - Results are sorted by relevance to your search query.
-  - Highlighting shows you which parts of the document matched the query.
-  - Searching for similar documents ("More like this")
-- Email processing: Paperless adds documents from your email accounts.
-  - Configure multiple accounts and filters for each account.
-  - When adding documents from mail, paperless can move these mail to a new folder, mark them as read, flag them as important or delete them.
-- Machine learning powered document matching.
-  - Paperless-ngx learns from your documents and will be able to automatically assign tags, correspondents and types to documents once you've stored a few documents in paperless.
-- Optimized for multi core systems: Paperless-ngx consumes multiple documents in parallel.
-- The integrated sanity checker makes sure that your document archive is in good health.
-- [More screenshots are available in the documentation](https://docs.paperless-ngx.com/#screenshots).
+A full list of [features](https://docs.paperless-ngx.com/#features) and [screenshots](https://docs.paperless-ngx.com/#screenshots) are available in the [documentation](https://docs.paperless-ngx.com/).
 
 # Getting started
 
-The easiest way to deploy paperless is docker-compose. The files in the [`/docker/compose` directory](https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose) are configured to pull the image from Github Packages.
+The easiest way to deploy paperless is `docker compose`. The files in the [`/docker/compose` directory](https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose) are configured to pull the image from GitHub Packages.
 
-If you'd like to jump right in, you can configure a docker-compose environment with our install script:
+If you'd like to jump right in, you can configure a `docker compose` environment with our install script:
 
 ```bash
 bash -c "$(curl -L https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/install-paperless-ngx.sh)"
@@ -85,7 +69,7 @@ If you feel like contributing to the project, please do! Bug fixes, enhancements
 
 ## Community Support
 
-People interested in continuing the work on paperless-ngx are encouraged to reach out here on github and in the [Matrix Room](https://matrix.to/#/#paperless:adnidor.de). If you would like to contribute to the project on an ongoing basis there are multiple [teams](https://github.com/orgs/paperless-ngx/people) (frontend, ci/cd, etc) that could use your help so please reach out!
+People interested in continuing the work on paperless-ngx are encouraged to reach out here on github and in the [Matrix Room](https://matrix.to/#/#paperless:matrix.org). If you would like to contribute to the project on an ongoing basis there are multiple [teams](https://github.com/orgs/paperless-ngx/people) (frontend, ci/cd, etc) that could use your help so please reach out!
 
 ## Translation
 
@@ -101,14 +85,7 @@ For bugs please [open an issue](https://github.com/paperless-ngx/paperless-ngx/i
 
 # Affiliated Projects
 
-Paperless has been around for a while now, and people have built tools that interact with it. If you're one of them, please reach out and we can add your project to the list. Current projects include:
-
-- **Mobile**
-  - [Paperless App](https://github.com/bauerj/paperless_app): An Android/iOS application for Paperless-ngx.
-  - [Paperless Mobile](https://github.com/astubenbord/paperless-mobile): A modern, feature rich Android app for Paperless-ngx.
-  - [Paperless Share](https://github.com/qcasey/paperless_share): Share any files from your Android application with Paperless-ngx. Very simple, but works with all mobile scanning apps that allow you to share scanned documents.
-- **Desktop**
-  - [Scan to Paperless](https://github.com/sbrunner/scan-to-paperless): Scan and prepare (crop, deskew, OCR, ...) your documents for use in Paperless-ngx.
+Please see [the wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Affiliated-Projects) for a user-maintained list of affiliated projects and software that is compatible with Paperless-ngx.
 
 # Important Note
 

build-docker-image.sh

@@ -1,81 +0,0 @@
-#!/usr/bin/env bash
-
-# Helper script for building the Docker image locally.
-# Parses and provides the nessecary versions of other images to Docker
-# before passing in the rest of script args.
-
-# First Argument: The Dockerfile to build
-# Other Arguments: Additional arguments to docker build
-
-# Example Usage:
-#	./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature
-
-set -eu
-
-if ! command -v jq &> /dev/null ;  then
-	echo "jq required"
-	exit 1
-elif [ ! -f "$1" ]; then
-	echo "$1 is not a file, please provide the Dockerfile"
-	exit 1
-fi
-
-# Get the branch name (used for caching)
-branch_name=$(git rev-parse --abbrev-ref HEAD)
-
-# Parse eithe Pipfile.lock or the .build-config.json
-jbig2enc_version=$(jq -r '.jbig2enc.version' .build-config.json)
-qpdf_version=$(jq -r '.qpdf.version' .build-config.json)
-psycopg2_version=$(jq -r '.default.psycopg2.version | gsub("=";"")' Pipfile.lock)
-pikepdf_version=$(jq -r '.default.pikepdf.version | gsub("=";"")' Pipfile.lock)
-pillow_version=$(jq -r '.default.pillow.version | gsub("=";"")' Pipfile.lock)
-lxml_version=$(jq -r '.default.lxml.version | gsub("=";"")' Pipfile.lock)
-
-base_filename="$(basename -- "${1}")"
-build_args_str=""
-cache_from_str=""
-
-case "${base_filename}" in
-
-	*.jbig2enc)
-		build_args_str="--build-arg JBIG2ENC_VERSION=${jbig2enc_version}"
-		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/jbig2enc:${jbig2enc_version}"
-		;;
-
-	*.psycopg2)
-		build_args_str="--build-arg PSYCOPG2_VERSION=${psycopg2_version}"
-		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/psycopg2:${psycopg2_version}"
-		;;
-
-	*.qpdf)
-		build_args_str="--build-arg QPDF_VERSION=${qpdf_version}"
-		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/qpdf:${qpdf_version}"
-		;;
-
-	*.pikepdf)
-		build_args_str="--build-arg QPDF_VERSION=${qpdf_version} --build-arg PIKEPDF_VERSION=${pikepdf_version} --build-arg PILLOW_VERSION=${pillow_version} --build-arg LXML_VERSION=${lxml_version}"
-		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/pikepdf:${pikepdf_version}"
-		;;
-
-	Dockerfile)
-		build_args_str="--build-arg QPDF_VERSION=${qpdf_version} --build-arg PIKEPDF_VERSION=${pikepdf_version} --build-arg PSYCOPG2_VERSION=${psycopg2_version} --build-arg JBIG2ENC_VERSION=${jbig2enc_version}"
-		cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:${branch_name} --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev"
-		;;
-
-	*)
-		echo "Unable to match ${base_filename}"
-		exit 1
-		;;
-esac
-
-read -r -a build_args_arr <<< "${build_args_str}"
-read -r -a cache_from_arr <<< "${cache_from_str}"
-
-set -eux
-
-docker buildx build --file "${1}" \
-	--progress=plain \
-	--output=type=docker \
-	"${cache_from_arr[@]}" \
-	"${build_args_arr[@]}" \
-	"${@:2}" .

crowdin.yml

@@ -1,4 +1,6 @@
-commit_message: '[ci skip]'
+project_id_env: CROWDIN_PROJECT_ID
+api_token_env: CROWDIN_PERSONAL_TOKEN
+preserve_hierarchy: true
 files:
   - source: /src/locale/en_US/LC_MESSAGES/django.po
     translation: /src/locale/%locale_with_underscore%/LC_MESSAGES/django.po

docker-builders/Dockerfile.jbig2enc

@@ -1,48 +0,0 @@
-# This Dockerfile compiles the jbig2enc library
-# Inputs:
-#    - JBIG2ENC_VERSION - the Git tag to checkout and build
-
-FROM debian:bullseye-slim as main
-
-LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
-
-ARG DEBIAN_FRONTEND=noninteractive
-ARG JBIG2ENC_VERSION
-
-ARG BUILD_PACKAGES="\
-  build-essential \
-  automake \
-  libtool \
-  libleptonica-dev \
-  zlib1g-dev \
-  git \
-  ca-certificates"
-
-WORKDIR /usr/src/jbig2enc
-
-RUN set -eux \
-  && echo "Installing build tools" \
-    && apt-get update --quiet \
-    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
-  && echo "Building jbig2enc" \
-    && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc . \
-    && ./autogen.sh \
-    && ./configure \
-    && make \
-  && echo "Gathering package data" \
-    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./pkg-list.txt \
-  && echo "Cleaning up image" \
-    && apt-get -y purge ${BUILD_PACKAGES} \
-    && apt-get -y autoremove --purge \
-    && rm -rf /var/lib/apt/lists/* \
-  && echo "Moving files around" \
-    && mkdir build \
-    # Unlink a symlink that causes problems
-    && unlink ./src/.libs/libjbig2enc.la \
-    # Move what the link pointed to
-    && mv ./src/libjbig2enc.la ./build/ \
-    # Move the shared library .so files
-    && mv ./src/.libs/libjbig2enc* ./build/ \
-    # And move the cli binary
-    && mv ./src/jbig2 ./build/ \
-    && mv ./pkg-list.txt ./build/

docker-builders/Dockerfile.pikepdf

@@ -1,118 +0,0 @@
-# This Dockerfile builds the pikepdf wheel
-# Inputs:
-#    - REPO - Docker repository to pull qpdf from
-#    - QPDF_VERSION - The image qpdf version to copy .deb files from
-#    - PIKEPDF_VERSION - Version of pikepdf to build wheel for
-
-# Default to pulling from the main repo registry when manually building
-ARG REPO="paperless-ngx/paperless-ngx"
-
-# This does nothing, except provide a name for a copy below
-ARG QPDF_VERSION
-FROM --platform=$BUILDPLATFORM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
-
-#
-# Stage: builder
-# Purpose:
-#  - Build the pikepdf wheel
-#  - Build any dependent wheels which can't be found
-#
-FROM python:3.9-slim-bullseye as builder
-
-LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
-
-# Buildx provided
-ARG TARGETARCH
-ARG TARGETVARIANT
-
-ARG DEBIAN_FRONTEND=noninteractive
-# Workflow provided
-ARG QPDF_VERSION
-ARG PIKEPDF_VERSION
-# These are not used, but will still bust the cache if one changes
-# Otherwise, the main image will try to build thing (and fail)
-ARG PILLOW_VERSION
-ARG LXML_VERSION
-
-ARG BUILD_PACKAGES="\
-  build-essential \
-  python3-dev \
-  python3-pip \
-  # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers
-  libgnutls28-dev \
-  # lxml requrements - https://lxml.de/installation.html
-  libxml2-dev \
-  libxslt1-dev \
-  # Pillow requirements - https://pillow.readthedocs.io/en/stable/installation.html#external-libraries
-  # JPEG functionality
-  libjpeg62-turbo-dev \
-  # conpressed PNG
-  zlib1g-dev \
-  # compressed TIFF
-  libtiff-dev \
-  # type related services
-  libfreetype-dev \
-  # color management
-  liblcms2-dev \
-  # WebP format
-  libwebp-dev \
-  # JPEG 2000
-  libopenjp2-7-dev \
-  # improved color quantization
-  libimagequant-dev \
-  # complex text layout support
-  libraqm-dev"
-
-WORKDIR /usr/src
-
-COPY --from=qpdf-builder /usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.deb ./
-
-# As this is an base image for a multi-stage final image
-# the added size of the install is basically irrelevant
-
-RUN set -eux \
-  && echo "Installing build tools" \
-    && apt-get update --quiet \
-    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
-  && echo "Installing qpdf" \
-    && dpkg --install libqpdf29_*.deb \
-    && dpkg --install libqpdf-dev_*.deb \
-  && echo "Installing Python tools" \
-    && python3 -m pip install --no-cache-dir --upgrade \
-      pip \
-      wheel \
-      # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements
-      pybind11 \
-  && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \
-    && mkdir wheels \
-    && python3 -m pip wheel \
-      # Build the package at the required version
-      pikepdf==${PIKEPDF_VERSION} \
-      # Look to piwheels for additional pre-built wheels
-      --extra-index-url https://www.piwheels.org/simple \
-      # Output the *.whl into this directory
-      --wheel-dir wheels \
-      # Do not use a binary packge for the package being built
-      --no-binary=pikepdf \
-      # Do use binary packages for dependencies
-      --prefer-binary \
-      # Don't cache build files
-      --no-cache-dir \
-    && ls -ahl wheels \
-  && echo "Gathering package data" \
-    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./wheels/pkg-list.txt \
-  && echo "Cleaning up image" \
-    && apt-get -y purge ${BUILD_PACKAGES} \
-    && apt-get -y autoremove --purge \
-    && rm -rf /var/lib/apt/lists/*
-
-#
-# Stage: package
-# Purpose: Holds the compiled .whl files in a tiny image to pull
-#
-FROM alpine:3.17 as package
-
-WORKDIR /usr/src/wheels/
-
-COPY --from=builder /usr/src/wheels/*.whl ./
-COPY --from=builder /usr/src/wheels/pkg-list.txt ./

docker-builders/Dockerfile.psycopg2

@@ -1,66 +0,0 @@
-# This Dockerfile builds the psycopg2 wheel
-# Inputs:
-#    - PSYCOPG2_VERSION - Version to build
-
-#
-# Stage: builder
-# Purpose:
-#  - Build the psycopg2 wheel
-#
-FROM python:3.9-slim-bullseye as builder
-
-LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
-
-ARG PSYCOPG2_VERSION
-ARG DEBIAN_FRONTEND=noninteractive
-
-ARG BUILD_PACKAGES="\
-  build-essential \
-  python3-dev \
-  python3-pip \
-  # https://www.psycopg.org/docs/install.html#prerequisites
-  libpq-dev"
-
-WORKDIR /usr/src
-
-# As this is an base image for a multi-stage final image
-# the added size of the install is basically irrelevant
-
-RUN set -eux \
-  && echo "Installing build tools" \
-    && apt-get update --quiet \
-    && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
-  && echo "Installing Python tools" \
-    && python3 -m pip install --no-cache-dir --upgrade pip wheel \
-  && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \
-    && cd /usr/src \
-    && mkdir wheels \
-    && python3 -m pip wheel \
-      # Build the package at the required version
-      psycopg2==${PSYCOPG2_VERSION} \
-      # Output the *.whl into this directory
-      --wheel-dir wheels \
-      # Do not use a binary packge for the package being built
-      --no-binary=psycopg2 \
-      # Do use binary packages for dependencies
-      --prefer-binary \
-      # Don't cache build files
-      --no-cache-dir \
-    && ls -ahl wheels/ \
-  && echo "Gathering package data" \
-    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./wheels/pkg-list.txt \
-  && echo "Cleaning up image" \
-    && apt-get -y purge ${BUILD_PACKAGES} \
-    && apt-get -y autoremove --purge \
-    && rm -rf /var/lib/apt/lists/*
-
-#
-# Stage: package
-# Purpose: Holds the compiled .whl files in a tiny image to pull
-#
-FROM alpine:3.17 as package
-
-WORKDIR /usr/src/wheels/
-
-COPY --from=builder /usr/src/wheels/*.whl ./
-COPY --from=builder /usr/src/wheels/pkg-list.txt ./

docker-builders/Dockerfile.qpdf

@@ -1,156 +0,0 @@
-#
-# Stage: pre-build
-# Purpose:
-#  - Installs common packages
-#  - Sets common environment variables related to dpkg
-#  - Aquires the qpdf source from bookwork
-# Useful Links:
-#  - https://qpdf.readthedocs.io/en/stable/installation.html#system-requirements
-#  - https://wiki.debian.org/Multiarch/HOWTO
-#  - https://wiki.debian.org/CrossCompiling
-#
-
-FROM debian:bullseye-slim as pre-build
-
-ARG QPDF_VERSION
-
-ARG COMMON_BUILD_PACKAGES="\
-  cmake \
-  debhelper\
-  debian-keyring \
-  devscripts \
-  dpkg-dev \
-  equivs \
-  packaging-dev \
-  libtool"
-
-ENV DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2"
-
-WORKDIR /usr/src
-
-RUN set -eux \
-  && echo "Installing common packages" \
-    && apt-get update --quiet \
-    && apt-get install --yes --quiet --no-install-recommends ${COMMON_BUILD_PACKAGES} \
-  && echo "Getting qpdf source" \
-    && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
-    && apt-get update --quiet \
-    && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm
-
-#
-# Stage: amd64-builder
-# Purpose: Builds qpdf for x86_64 (native build)
-#
-FROM pre-build as amd64-builder
-
-ARG AMD64_BUILD_PACKAGES="\
-  build-essential \
-  libjpeg62-turbo-dev:amd64 \
-  libgnutls28-dev:amd64 \
-  zlib1g-dev:amd64"
-
-WORKDIR /usr/src/qpdf-${QPDF_VERSION}
-
-RUN set -eux \
-  && echo "Beginning amd64" \
-    && echo "Install amd64 packages" \
-      && apt-get update --quiet \
-      && apt-get install --yes --quiet --no-install-recommends ${AMD64_BUILD_PACKAGES} \
-    && echo "Building amd64" \
-      && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean \
-    && echo "Removing debug files" \
-      && rm -f ../libqpdf29-dbgsym* \
-      && rm -f ../qpdf-dbgsym* \
-    && echo "Gathering package data" \
-      && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt
-#
-# Stage: armhf-builder
-# Purpose:
-#  - Sets armhf specific environment
-#  - Builds qpdf for armhf (cross compile)
-#
-FROM pre-build as armhf-builder
-
-ARG ARMHF_PACKAGES="\
-  crossbuild-essential-armhf \
-  libjpeg62-turbo-dev:armhf \
-  libgnutls28-dev:armhf \
-  zlib1g-dev:armhf"
-
-WORKDIR /usr/src/qpdf-${QPDF_VERSION}
-
-ENV CXX="/usr/bin/arm-linux-gnueabihf-g++" \
-    CC="/usr/bin/arm-linux-gnueabihf-gcc"
-
-RUN set -eux \
-  && echo "Beginning armhf" \
-    && echo "Install armhf packages" \
-      && dpkg --add-architecture armhf \
-      && apt-get update --quiet \
-      && apt-get install --yes --quiet --no-install-recommends ${ARMHF_PACKAGES} \
-    && echo "Building armhf" \
-      && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean --host-arch armhf \
-    && echo "Removing debug files" \
-      && rm -f ../libqpdf29-dbgsym* \
-      && rm -f ../qpdf-dbgsym* \
-    && echo "Gathering package data" \
-      && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt
-
-#
-# Stage: aarch64-builder
-# Purpose:
-#  - Sets aarch64 specific environment
-#  - Builds qpdf for aarch64 (cross compile)
-#
-FROM pre-build as aarch64-builder
-
-ARG ARM64_PACKAGES="\
-  crossbuild-essential-arm64 \
-  libjpeg62-turbo-dev:arm64 \
-  libgnutls28-dev:arm64 \
-  zlib1g-dev:arm64"
-
-ENV CXX="/usr/bin/aarch64-linux-gnu-g++" \
-    CC="/usr/bin/aarch64-linux-gnu-gcc"
-
-WORKDIR /usr/src/qpdf-${QPDF_VERSION}
-
-RUN set -eux \
-  && echo "Beginning arm64" \
-    && echo "Install arm64 packages" \
-      && dpkg --add-architecture arm64 \
-      && apt-get update --quiet \
-      && apt-get install --yes --quiet --no-install-recommends ${ARM64_PACKAGES} \
-    && echo "Building arm64" \
-      && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean --host-arch arm64 \
-    && echo "Removing debug files" \
-      && rm -f ../libqpdf29-dbgsym* \
-      && rm -f ../qpdf-dbgsym* \
-    && echo "Gathering package data" \
-      && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt
-
-#
-# Stage: package
-# Purpose: Holds the compiled .deb files in arch/variant specific folders
-#
-FROM alpine:3.17 as package
-
-LABEL org.opencontainers.image.description="A image with qpdf installers stored in architecture & version specific folders"
-
-ARG QPDF_VERSION
-
-WORKDIR /usr/src/qpdf/${QPDF_VERSION}/amd64
-
-COPY --from=amd64-builder /usr/src/*.deb ./
-COPY --from=amd64-builder /usr/src/pkg-list.txt ./
-
-# Note this is ${TARGETARCH}${TARGETVARIANT} for armv7
-WORKDIR /usr/src/qpdf/${QPDF_VERSION}/armv7
-
-COPY --from=armhf-builder /usr/src/*.deb ./
-COPY --from=armhf-builder /usr/src/pkg-list.txt ./
-
-WORKDIR /usr/src/qpdf/${QPDF_VERSION}/arm64
-
-COPY --from=aarch64-builder /usr/src/*.deb ./
-COPY --from=aarch64-builder /usr/src/pkg-list.txt ./

docker-builders/README.md

@@ -1,57 +0,0 @@
-# Installer Library
-
-This folder contains the Dockerfiles for building certain installers or libraries, which are then pulled into the main image.
-
-## [jbig2enc](https://github.com/agl/jbig2enc)
-
-### Why
-
-JBIG is an image coding which can achieve better compression of images for PDFs.
-
-### What
-
-The Docker image builds a shared library file and utility, which is copied into the correct location in the final image.
-
-### Updating
-
-1. Ensure the given qpdf version is present in [Debian bookworm](https://packages.debian.org/bookworm/qpdf)
-2. Update `.build-config.json` to the given version
-3. If the Debian specific version has incremented, update `Dockerfile.qpdf`
-
-See Also:
-
-- [OCRMyPDF Documentation](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html)
-
-## [psycopg2](https://www.psycopg.org/)
-
-### Why
-
-The pre-built wheels of psycopg2 are built on Debian 9, which provides a quite old version of libpq-dev. This causes issue with authentication methods.
-
-### What
-
-The image builds psycopg2 wheels on Debian 10 and places the produced wheels into `/usr/src/wheels/`.
-
-See Also:
-
-- [Issue 266](https://github.com/paperless-ngx/paperless-ngx/issues/266)
-
-## [qpdf](https://qpdf.readthedocs.io/en/stable/index.html)
-
-### Why
-
-qpdf and it's library provide tools to read, manipulate and fix up PDFs. Version 11 is also required by `pikepdf` 6+ and Debian 9 does not provide above version 10.
-
-### What
-
-The Docker image cross compiles .deb installers for each supported architecture of the main image. The installers are placed in `/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/`
-
-## [pikepdf](https://pikepdf.readthedocs.io/en/latest/)
-
-### Why
-
-Required by OCRMyPdf, this is a general purpose library for PDF manipulation in Python via the qpdf libraries.
-
-### What
-
-The built wheels are placed into `/usr/src/wheels/`

docker/compose/docker-compose.ci-test.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless testing with actual gotenberg
+# Docker Compose file for running paperless testing with actual gotenberg
 # and Tika containers for a more end to end test of the Tika related functionality
 # Can be used locally or by the CI to start the nessecary containers with the
 # correct networking for the tests
@@ -6,7 +6,7 @@
 version: "3.7"
 services:
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.8
+    image: docker.io/gotenberg/gotenberg:7.10
     hostname: gotenberg
     container_name: gotenberg
     network_mode: host
@@ -17,6 +17,8 @@ services:
       - "gotenberg"
       - "--chromium-disable-javascript=true"
       - "--chromium-allow-list=file:///tmp/.*"
+      - "--log-level=warn"
+      - "--log-format=text"
   tika:
     image: ghcr.io/paperless-ngx/tika:latest
     hostname: tika

docker/compose/docker-compose.mariadb-tika.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless from the Docker Hub.
+# docker compose file for running paperless from the Docker Hub.
 # This file contains everything paperless needs to run.
 # Paperless supports amd64, arm and arm64 hardware.
 #
@@ -10,7 +10,7 @@
 #   as this file and mounted to the correct folders inside the container.
 # - Paperless listens on port 8000.
 #
-# In addition to that, this docker-compose file adds the following optional
+# In addition to that, this Docker Compose file adds the following optional
 # configurations:
 #
 # - Instead of SQLite (default), MariaDB is used as the database server.
@@ -23,9 +23,9 @@
 #
 # - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
 #   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
+# - Run 'docker compose pull'.
+# - Run 'docker compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker compose up -d'.
 #
 # For more extensive installation and update instructions, refer to the
 # documentation.
@@ -83,7 +83,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.8
+    image: docker.io/gotenberg/gotenberg:7.10
     restart: unless-stopped
     # The gotenberg chromium route is used to convert .eml files. We do not
     # want to allow external content like tracking pixels or even javascript.

docker/compose/docker-compose.mariadb.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless from the Docker Hub.
+# Docker Compose file for running paperless from the Docker Hub.
 # This file contains everything paperless needs to run.
 # Paperless supports amd64, arm and arm64 hardware.
 #
@@ -10,7 +10,7 @@
 #   as this file and mounted to the correct folders inside the container.
 # - Paperless listens on port 8000.
 #
-# In addition to that, this docker-compose file adds the following optional
+# In addition to that, this Docker Compose file adds the following optional
 # configurations:
 #
 # - Instead of SQLite (default), MariaDB is used as the database server.
@@ -19,9 +19,9 @@
 #
 # - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
 #   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
+# - Run 'docker compose pull'.
+# - Run 'docker compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker compose up -d'.
 #
 # For more extensive installation and update instructions, refer to the
 # documentation.

docker/compose/docker-compose.portainer.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless from the Docker Hub.
+# Docker Compose file for running paperless from the Docker Hub.
 # This file contains everything paperless needs to run.
 # Paperless supports amd64, arm and arm64 hardware.
 #
@@ -10,7 +10,7 @@
 #   as this file and mounted to the correct folders inside the container.
 # - Paperless listens on port 8010.
 #
-# In addition to that, this docker-compose file adds the following optional
+# In addition to that, this Docker Compose file adds the following optional
 # configurations:
 #
 # - Instead of SQLite (default), PostgreSQL is used as the database server.
@@ -37,7 +37,7 @@ services:
       - redisdata:/data
 
   db:
-    image: docker.io/library/postgres:13
+    image: docker.io/library/postgres:15
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.postgres-tika.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless from the docker container registry.
+# Docker Compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
 # Paperless supports amd64, arm and arm64 hardware.
 #
@@ -10,7 +10,7 @@
 #   as this file and mounted to the correct folders inside the container.
 # - Paperless listens on port 8000.
 #
-# In addition to that, this docker-compose file adds the following optional
+# In addition to that, this Docker Compose file adds the following optional
 # configurations:
 #
 # - Instead of SQLite (default), PostgreSQL is used as the database server.
@@ -23,9 +23,9 @@
 #
 # - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
 #   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
+# - Run 'docker compose pull'.
+# - Run 'docker compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker compose up -d'.
 #
 # For more extensive installation and update instructions, refer to the
 # documentation.
@@ -39,7 +39,7 @@ services:
       - redisdata:/data
 
   db:
-    image: docker.io/library/postgres:13
+    image: docker.io/library/postgres:15
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
@@ -77,7 +77,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.8
+    image: docker.io/gotenberg/gotenberg:7.10
     restart: unless-stopped
 
     # The gotenberg chromium route is used to convert .eml files. We do not

docker/compose/docker-compose.postgres.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless from the Docker Hub.
+# Docker Compose file for running paperless from the Docker Hub.
 # This file contains everything paperless needs to run.
 # Paperless supports amd64, arm and arm64 hardware.
 #
@@ -10,7 +10,7 @@
 #   as this file and mounted to the correct folders inside the container.
 # - Paperless listens on port 8000.
 #
-# In addition to that, this docker-compose file adds the following optional
+# In addition to that, this Docker Compose file adds the following optional
 # configurations:
 #
 # - Instead of SQLite (default), PostgreSQL is used as the database server.
@@ -19,9 +19,9 @@
 #
 # - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
 #   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
+# - Run 'docker compose pull'.
+# - Run 'docker compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker compose up -d'.
 #
 # For more extensive installation and update instructions, refer to the
 # documentation.
@@ -35,7 +35,7 @@ services:
       - redisdata:/data
 
   db:
-    image: docker.io/library/postgres:13
+    image: docker.io/library/postgres:15
     restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data

docker/compose/docker-compose.sqlite-tika.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless from the docker container registry.
+# Docker Compose file for running paperless from the docker container registry.
 # This file contains everything paperless needs to run.
 # Paperless supports amd64, arm and arm64 hardware.
 # All compose files of paperless configure paperless in the following way:
@@ -11,7 +11,7 @@
 #
 # SQLite is used as the database. The SQLite file is stored in the data volume.
 #
-# In addition to that, this docker-compose file adds the following optional
+# In addition to that, this Docker Compose file adds the following optional
 # configurations:
 #
 # - Apache Tika and Gotenberg servers are started with paperless and paperless
@@ -23,9 +23,9 @@
 #
 # - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
 #   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
+# - Run 'docker compose pull'.
+# - Run 'docker compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker compose up -d'.
 #
 # For more extensive installation and update instructions, refer to the
 # documentation.
@@ -65,7 +65,7 @@ services:
       PAPERLESS_TIKA_ENDPOINT: http://tika:9998
 
   gotenberg:
-    image: docker.io/gotenberg/gotenberg:7.8
+    image: docker.io/gotenberg/gotenberg:7.10
     restart: unless-stopped
 
     # The gotenberg chromium route is used to convert .eml files. We do not

docker/compose/docker-compose.sqlite.yml

@@ -1,4 +1,4 @@
-# docker-compose file for running paperless from the Docker Hub.
+# Docker Compose file for running paperless from the Docker Hub.
 # This file contains everything paperless needs to run.
 # Paperless supports amd64, arm and arm64 hardware.
 #
@@ -16,9 +16,9 @@
 #
 # - Copy this file as 'docker-compose.yml' and the files 'docker-compose.env'
 #   and '.env' into a folder.
-# - Run 'docker-compose pull'.
-# - Run 'docker-compose run --rm webserver createsuperuser' to create a user.
-# - Run 'docker-compose up -d'.
+# - Run 'docker compose pull'.
+# - Run 'docker compose run --rm webserver createsuperuser' to create a user.
+# - Run 'docker compose up -d'.
 #
 # For more extensive installation and update instructions, refer to the
 # documentation.

docker/docker-prepare.sh

@@ -80,7 +80,7 @@ django_checks() {
 
 search_index() {
 
-	local -r index_version=4
+	local -r index_version=7
 	local -r index_version_file=${DATA_DIR}/.index_version
 
 	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then

docker/env-from-file.sh

@@ -15,6 +15,10 @@ do
 	env_name=${line%%=*}
 	# Check if it starts with "PAPERLESS_" and ends in "_FILE"
 	if [[ ${env_name} == PAPERLESS_*_FILE ]]; then
+		# This should have been named different..
+		if [[ ${env_name} == "PAPERLESS_OCR_SKIP_ARCHIVE_FILE" ]]; then
+			continue
+		fi
 		# Extract the value of the environment
 		env_value=${line#*=}
 

docker/install_management_commands.sh

@@ -13,6 +13,7 @@ for command in decrypt_documents \
 	document_retagger \
 	document_thumbnails \
 	document_sanity_checker \
+	document_fuzzy_match \
 	manage_superuser;
 do
 	echo "installing $command..."

docker/supervisord.conf

@@ -15,6 +15,7 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
+environment = HOME="/usr/src/paperless",USER="paperless"
 
 [program:consumer]
 command=python3 manage.py document_consumer
@@ -25,6 +26,7 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
+environment = HOME="/usr/src/paperless",USER="paperless"
 
 [program:celery]
 
@@ -37,6 +39,7 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
+environment = HOME="/usr/src/paperless",USER="paperless"
 
 [program:celery-beat]
 
@@ -48,6 +51,7 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
+environment = HOME="/usr/src/paperless",USER="paperless"
 
 [program:celery-flower]
 command = /usr/local/bin/flower-conditional.sh
@@ -58,3 +62,4 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
+environment = HOME="/usr/src/paperless",USER="paperless"

docker/wait-for-redis.py

@@ -12,7 +12,6 @@ from typing import Final
 from redis import Redis
 
 if __name__ == "__main__":
-
     MAX_RETRY_COUNT: Final[int] = 5
     RETRY_SLEEP_SECONDS: Final[int] = 5
 
@@ -29,7 +28,7 @@ if __name__ == "__main__":
             except Exception as e:
                 print(
                     f"Redis ping #{attempt} failed.\n"
-                    f"Error: {str(e)}.\n"
+                    f"Error: {e!s}.\n"
                     f"Waiting {RETRY_SLEEP_SECONDS}s",
                     flush=True,
                 )

docs/administration.md

@@ -5,17 +5,19 @@
 Multiple options exist for making backups of your paperless instance,
 depending on how you installed paperless.
 
-Before making backups, make sure that paperless is not running.
+Before making a backup, it's probably best to make sure that paperless is not actively
+consuming documents at that time.
 
 Options available to any installation of paperless:
 
-- Use the [document exporter](#exporter). The document exporter exports all your documents,
-  thumbnails and metadata to a specific folder. You may import your
-  documents into a fresh instance of paperless again or store your
-  documents in another DMS with this export.
-- The document exporter is also able to update an already existing
-  export. Therefore, incremental backups with `rsync` are entirely
-  possible.
+-   Use the [document exporter](#exporter). The document exporter exports all your documents,
+    thumbnails, metadata, and database contents to a specific folder. You may import your
+    documents and settings into a fresh instance of paperless again or store your
+    documents in another DMS with this export.
+
+    The document exporter is also able to update an already existing
+    export. Therefore, incremental backups with `rsync` are entirely
+    possible.
 
 !!! caution
 
@@ -25,31 +27,37 @@ Options available to any installation of paperless:
 
 Options available to docker installations:
 
-- Backup the docker volumes. These usually reside within
-  `/var/lib/docker/volumes` on the host and you need to be root in
-  order to access them.
+-   Backup the docker volumes. These usually reside within
+    `/var/lib/docker/volumes` on the host and you need to be root in
+    order to access them.
 
-  Paperless uses 4 volumes:
+    Paperless uses 4 volumes:
 
-  - `paperless_media`: This is where your documents are stored.
-  - `paperless_data`: This is where auxillary data is stored. This
-    folder also contains the SQLite database, if you use it.
-  - `paperless_pgdata`: Exists only if you use PostgreSQL and
-    contains the database.
-  - `paperless_dbdata`: Exists only if you use MariaDB and contains
-    the database.
+    -   `paperless_media`: This is where your documents are stored.
+    -   `paperless_data`: This is where auxiliary data is stored. This
+        folder also contains the SQLite database, if you use it.
+    -   `paperless_pgdata`: Exists only if you use PostgreSQL and
+        contains the database.
+    -   `paperless_dbdata`: Exists only if you use MariaDB and contains
+        the database.
 
 Options available to bare-metal and non-docker installations:
 
-- Backup the entire paperless folder. This ensures that if your
-  paperless instance crashes at some point or your disk fails, you can
-  simply copy the folder back into place and it works.
+-   Backup the entire paperless folder. This ensures that if your
+    paperless instance crashes at some point or your disk fails, you can
+    simply copy the folder back into place and it works.
 
-  When using PostgreSQL or MariaDB, you'll also have to backup the
-  database.
+    When using PostgreSQL or MariaDB, you'll also have to backup the
+    database.
 
 ### Restoring {#migrating-restoring}
 
+If you've backed-up Paperless-ngx using the [document exporter](#exporter),
+restoring can simply be done with the [document importer](#importer).
+
+Of course, other backup strategies require restoring any volumes, folders and database
+copies you created in the steps above.
+
 ## Updating Paperless {#updating}
 
 ### Docker Route {#docker-updating}
@@ -63,30 +71,30 @@ First of all, ensure that paperless is stopped.
 
 ```shell-session
 $ cd /path/to/paperless
-$ docker-compose down
+$ docker compose down
 ```
 
 After that, [make a backup](#backup).
 
-1. If you pull the image from the docker hub, all you need to do is:
+1.  If you pull the image from the docker hub, all you need to do is:
 
-   ```shell-session
-   $ docker-compose pull
-   $ docker-compose up
-   ```
+    ```shell-session
+    $ docker compose pull
+    $ docker compose up
+    ```
 
-   The docker-compose files refer to the `latest` version, which is
-   always the latest stable release.
+    The Docker Compose files refer to the `latest` version, which is
+    always the latest stable release.
 
-2. If you built the image yourself, do the following:
+1.  If you built the image yourself, do the following:
 
-   ```shell-session
-   $ git pull
-   $ docker-compose build
-   $ docker-compose up
-   ```
+    ```shell-session
+    $ git pull
+    $ docker compose build
+    $ docker compose up
+    ```
 
-Running `docker-compose up` will also apply any new database migrations.
+Running `docker compose up` will also apply any new database migrations.
 If you see everything working, press CTRL+C once to gracefully stop
 paperless. Then you can start paperless-ngx with `-d` to have it run in
 the background.
@@ -94,7 +102,7 @@ the background.
 !!! note
 
     In version 0.9.14, the update process was changed. In 0.9.13 and
-    earlier, the docker-compose files specified exact versions and pull
+    earlier, the Docker Compose files specified exact versions and pull
     won't automatically update to newer versions. In order to enable
     updates as described above, either get the new `docker-compose.yml`
     file from
@@ -139,7 +147,7 @@ following:
 1.  Update dependencies. New paperless version may require additional
     dependencies. The dependencies required are listed in the section
     about
-    [bare metal installations](/setup#bare_metal).
+    [bare metal installations](setup.md#bare_metal).
 
 2.  Update python requirements. Keep in mind to activate your virtual
     environment before that, if you use one.
@@ -148,6 +156,13 @@ following:
     $ pip install -r requirements.txt
     ```
 
+    !!! note
+
+        At times, some dependencies will be removed from requirements.txt.
+        Comparing the versions and removing no longer needed dependencies
+        will keep your system or virtual environment clean and prevent
+        possible conflicts.
+
 3.  Migrate the database.
 
     ```shell-session
@@ -160,6 +175,16 @@ following:
     This might not actually do anything. Not every new paperless version
     comes with new database migrations.
 
+### Database Upgrades
+
+In general, paperless does not require a specific version of PostgreSQL or MariaDB and it is
+safe to update them to newer versions. However, you should always take a backup and follow
+the instructions from your database's documentation for how to upgrade between major versions.
+
+For PostgreSQL, refer to [Upgrading a PostgreSQL Cluster](https://www.postgresql.org/docs/current/upgrading.html).
+
+For MariaDB, refer to [Upgrading MariaDB](https://mariadb.com/kb/en/upgrading/)
+
 ## Downgrading Paperless {#downgrade-paperless}
 
 Downgrades are possible. However, some updates also contain database
@@ -195,11 +220,11 @@ Paperless comes with some management commands that perform various
 maintenance tasks on your paperless instance. You can invoke these
 commands in the following way:
 
-With docker-compose, while paperless is running:
+With Docker Compose, while paperless is running:
 
 ```shell-session
 $ cd /path/to/paperless
-$ docker-compose exec webserver <command> <arguments>
+$ docker compose exec webserver <command> <arguments>
 ```
 
 With docker, while paperless is running:
@@ -222,26 +247,28 @@ with the argument `--help`.
 
 ### Document exporter {#exporter}
 
-The document exporter exports all your data from paperless into a folder
-for backup or migration to another DMS.
+The document exporter exports all your data (including your settings
+and database contents) from paperless into a folder for backup or
+migration to another DMS.
 
 If you use the document exporter within a cronjob to backup your data
 you might use the `-T` flag behind exec to suppress "The input device
 is not a TTY" errors. For example:
-`docker-compose exec -T webserver document_exporter ../export`
+`docker compose exec -T webserver document_exporter ../export`
 
 ```
 document_exporter target [-c] [-d] [-f] [-na] [-nt] [-p] [-sm] [-z]
 
 optional arguments:
--c, --compare-checksums
--d, --delete
--f, --use-filename-format
+-c,  --compare-checksums
+-d,  --delete
+-f,  --use-filename-format
 -na, --no-archive
 -nt, --no-thumbnail
--p, --use-folder-prefix
+-p,  --use-folder-prefix
 -sm, --split-manifest
--z  --zip
+-z,  --zip
+-zn, --zip-name
 ```
 
 `target` is a folder to which the data gets written. This includes
@@ -269,7 +296,7 @@ other files.
 
 The filenames generated by this command follow the format
 `[date created] [correspondent] [title].[extension]`. If you want
-paperless to use `PAPERLESS_FILENAME_FORMAT` for exported filenames
+paperless to use [`PAPERLESS_FILENAME_FORMAT`](configuration.md#PAPERLESS_FILENAME_FORMAT) for exported filenames
 instead, specify `-f` or `--use-filename-format`.
 
 If `-na` or `--no-archive` is provided, no archive files will be exported,
@@ -296,8 +323,9 @@ will be placed in individual json files, instead of a single JSON file. The main
 manifest.json will still contain application wide information (e.g. tags, correspondent,
 documenttype, etc)
 
-If `-z` or `--zip` is provided, the export will be a zipfile
-in the target directory, named according to the current date.
+If `-z` or `--zip` is provided, the export will be a zip file
+in the target directory, named according to the current local date or the
+value set in `-zn` or `--zip-name`.
 
 !!! warning
 
@@ -334,7 +362,7 @@ currently-imported docs. This problem is common enough that there are
 tools for it.
 
 ```
-document_retagger [-h] [-c] [-T] [-t] [-i] [--use-first] [-f]
+document_retagger [-h] [-c] [-T] [-t] [-i] [--id-range] [--use-first] [-f]
 
 optional arguments:
 -c, --correspondent
@@ -342,6 +370,7 @@ optional arguments:
 -t, --document_type
 -s, --storage_path
 -i, --inbox-only
+--id-range
 --use-first
 -f, --overwrite
 ```
@@ -358,6 +387,11 @@ Specify `-i` to have the document retagger work on documents tagged with
 inbox tags only. This is useful when you don't want to mess with your
 already processed documents.
 
+Specify `--id-range 1 100` to have the document retagger work only on a
+specific range of document id´s. This can be useful if you have a lot of
+documents and want to test the matching rules only on a subset of
+documents.
+
 When multiple document types or correspondents match a single document,
 the retagger won't assign these to the document. Specify `--use-first`
 to override this behavior and just use the first correspondent or type
@@ -374,7 +408,7 @@ that don't match a document anymore get removed as well.
 ### Managing the Automatic matching algorithm
 
 The _Auto_ matching algorithm requires a trained neural network to work.
-This network needs to be updated whenever somethings in your data
+This network needs to be updated whenever something in your data
 changes. The docker image takes care of that automatically with the task
 scheduler. You can manually renew the classifier by invoking the
 following management command:
@@ -389,6 +423,9 @@ This command takes no arguments.
 
 Use this command to re-create document thumbnails. Optionally include the ` --document {id}` option to generate thumbnails for a specific document only.
 
+You may also specify `--processes` to control the number of processes used to generate new thumbnails. The default is to utilize
+a quarter of the available processors.
+
 ```
 document_thumbnails
 ```
@@ -416,7 +453,7 @@ task scheduler.
 ### Managing filenames {#renamer}
 
 If you use paperless' feature to
-[assign custom filenames to your documents](/advanced_usage#file-name-handling), you can use this command to move all your files after
+[assign custom filenames to your documents](advanced_usage.md#file-name-handling), you can use this command to move all your files after
 changing the naming scheme.
 
 !!! warning
@@ -441,19 +478,19 @@ collection for issues.
 
 The issues detected by the sanity checker are as follows:
 
-- Missing original files.
-- Missing archive files.
-- Inaccessible original files due to improper permissions.
-- Inaccessible archive files due to improper permissions.
-- Corrupted original documents by comparing their checksum against
-  what is stored in the database.
-- Corrupted archive documents by comparing their checksum against what
-  is stored in the database.
-- Missing thumbnails.
-- Inaccessible thumbnails due to improper permissions.
-- Documents without any content (warning).
-- Orphaned files in the media directory (warning). These are files
-  that are not referenced by any document im paperless.
+-   Missing original files.
+-   Missing archive files.
+-   Inaccessible original files due to improper permissions.
+-   Inaccessible archive files due to improper permissions.
+-   Corrupted original documents by comparing their checksum against
+    what is stored in the database.
+-   Corrupted archive documents by comparing their checksum against what
+    is stored in the database.
+-   Missing thumbnails.
+-   Inaccessible thumbnails due to improper permissions.
+-   Documents without any content (warning).
+-   Orphaned files in the media directory (warning). These are files
+    that are not referenced by any document in paperless.
 
 ```
 document_sanity_checker
@@ -522,7 +559,7 @@ Documents can be stored in Paperless using GnuPG encryption.
 
 !!! warning
 
-    Encryption is deprecated since [paperless-ng 0.9](/changelog#paperless-ng-090) and doesn't really
+    Encryption is deprecated since [paperless-ng 0.9](changelog.md#paperless-ng-090) and doesn't really
     provide any additional security, since you have to store the passphrase
     in a configuration file on the same system as the encrypted documents
     for paperless to work. Furthermore, the entire text content of the
@@ -543,9 +580,30 @@ Enabling encryption is no longer supported.
 
 Basic usage to disable encryption of your document store:
 
-(Note: If `PAPERLESS_PASSPHRASE` isn't set already, you need to specify
+(Note: If [`PAPERLESS_PASSPHRASE`](configuration.md#PAPERLESS_PASSPHRASE) isn't set already, you need to specify
 it here)
 
 ```
 decrypt_documents [--passphrase SECR3TP4SSPHRA$E]
 ```
+
+### Detecting duplicates {#fuzzy_duplicate}
+
+Paperless already catches and prevents upload of exactly matching documents,
+however a new scan of an existing document may not produce an exact bit for bit
+duplicate. But the content should be exact or close, allowing detection.
+
+This tool does a fuzzy match over document content, looking for
+those which look close according to a given ratio.
+
+At this time, other metadata (such as correspondent or type) is not
+taken into account by the detection.
+
+```
+document_fuzzy_match [--ratio] [--processes N]
+```
+
+| Option      | Required | Default             | Description                                                                                                                    |
+| ----------- | -------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
+| --ratio     | No       | 85.0                | a number between 0 and 100, setting how similar a document must be for it to be reported. Higher numbers mean more similarity. |
+| --processes | No       | 1/4 of system cores | Number of processes to use for matching. Setting 1 disables multiple processes                                                 |

docs/advanced_usage.md

@@ -1,6 +1,6 @@
 # Advanced Topics
 
-Paperless offers a couple features that automate certain tasks and make
+Paperless offers a couple of features that automate certain tasks and make
 your life easier.
 
 ## Matching tags, correspondents, document types, and storage paths {#matching}
@@ -35,9 +35,10 @@ The following algorithms are available:
   (i.e. preserve ordering) in the PDF.
 - **Regular expression:** Parses the match as a regular expression and
   tries to find a match within the document.
-- **Fuzzy match:** I don't know. Look at the source.
+- **Fuzzy match:** Uses a partial matching based on locating the tag text
+  inside the document, using a [partial ratio](https://maxbachmann.github.io/RapidFuzz/Usage/fuzz.html#partial-ratio)
 - **Auto:** Tries to automatically match new documents. This does not
-  require you to set a match. See the notes below.
+  require you to set a match. See the [notes below](#automatic-matching).
 
 When using the _any_ or _all_ matching algorithms, you can search for
 terms that consist of multiple words by enclosing them in double quotes.
@@ -92,7 +93,7 @@ when using this feature:
   decide when not to assign a certain tag, correspondent, document
   type, or storage path. This will usually be the case as you start
   filling up paperless with documents. Example: If all your documents
-  are either from "Webshop" and "Bank", paperless will assign one
+  are either from "Webshop" or "Bank", paperless will assign one
   of these correspondents to ANY new document, if both are set to
   automatic matching.
 
@@ -101,12 +102,12 @@ when using this feature:
 Sometimes you may want to do something arbitrary whenever a document is
 consumed. Rather than try to predict what you may want to do, Paperless
 lets you execute scripts of your own choosing just before or after a
-document is consumed using a couple simple hooks.
+document is consumed using a couple of simple hooks.
 
 Just write a script, put it somewhere that Paperless can read & execute,
 and then put the path to that script in `paperless.conf` or
 `docker-compose.env` with the variable name of either
-`PAPERLESS_PRE_CONSUME_SCRIPT` or `PAPERLESS_POST_CONSUME_SCRIPT`.
+[`PAPERLESS_PRE_CONSUME_SCRIPT`](configuration.md#PAPERLESS_PRE_CONSUME_SCRIPT) or [`PAPERLESS_POST_CONSUME_SCRIPT`](configuration.md#PAPERLESS_POST_CONSUME_SCRIPT).
 
 !!! info
 
@@ -126,6 +127,7 @@ script can access the following relevant environment variables set:
 | ----------------------- | ------------------------------------------------------------ |
 | `DOCUMENT_SOURCE_PATH`  | Original path of the consumed document                       |
 | `DOCUMENT_WORKING_PATH` | Path to a copy of the original that consumption will work on |
+| `TASK_ID`               | UUID of the task used to process the new document (if any)   |
 
 !!! note
 
@@ -168,21 +170,22 @@ Executed after the consumer has successfully processed a document and
 has moved it into paperless. It receives the following environment
 variables:
 
-| Environment Variable         | Description                                   |
-| ---------------------------- | --------------------------------------------- |
-| `DOCUMENT_ID`                | Database primary key of the document          |
-| `DOCUMENT_FILE_NAME`         | Formatted filename, not including paths       |
-| `DOCUMENT_CREATED`           | Date & time when document created             |
-| `DOCUMENT_MODIFIED`          | Date & time when document was last modified   |
-| `DOCUMENT_ADDED`             | Date & time when document was added           |
-| `DOCUMENT_SOURCE_PATH`       | Path to the original document file            |
-| `DOCUMENT_ARCHIVE_PATH`      | Path to the generate archive file (if any)    |
-| `DOCUMENT_THUMBNAIL_PATH`    | Path to the generated thumbnail               |
-| `DOCUMENT_DOWNLOAD_URL`      | URL for document download                     |
-| `DOCUMENT_THUMBNAIL_URL`     | URL for the document thumbnail                |
-| `DOCUMENT_CORRESPONDENT`     | Assigned correspondent (if any)               |
-| `DOCUMENT_TAGS`              | Comma separated list of tags applied (if any) |
-| `DOCUMENT_ORIGINAL_FILENAME` | Filename of original document                 |
+| Environment Variable         | Description                                    |
+| ---------------------------- | ---------------------------------------------- |
+| `DOCUMENT_ID`                | Database primary key of the document           |
+| `DOCUMENT_FILE_NAME`         | Formatted filename, not including paths        |
+| `DOCUMENT_CREATED`           | Date & time when document created              |
+| `DOCUMENT_MODIFIED`          | Date & time when document was last modified    |
+| `DOCUMENT_ADDED`             | Date & time when document was added            |
+| `DOCUMENT_SOURCE_PATH`       | Path to the original document file             |
+| `DOCUMENT_ARCHIVE_PATH`      | Path to the generate archive file (if any)     |
+| `DOCUMENT_THUMBNAIL_PATH`    | Path to the generated thumbnail                |
+| `DOCUMENT_DOWNLOAD_URL`      | URL for document download                      |
+| `DOCUMENT_THUMBNAIL_URL`     | URL for the document thumbnail                 |
+| `DOCUMENT_CORRESPONDENT`     | Assigned correspondent (if any)                |
+| `DOCUMENT_TAGS`              | Comma separated list of tags applied (if any)  |
+| `DOCUMENT_ORIGINAL_FILENAME` | Filename of original document                  |
+| `TASK_ID`                    | Task UUID used to import the document (if any) |
 
 The script can be in any language, A simple shell script example:
 
@@ -197,7 +200,7 @@ The script can be in any language, A simple shell script example:
 !!! warning
 
     The post consumption script should not modify the document files
-    directly
+    directly.
 
 The script's stdout and stderr will be logged line by line to the
 webserver log, along with the exit code of the script.
@@ -233,8 +236,8 @@ webserver:
 
 Troubleshooting:
 
-- Monitor the docker-compose log
-  `cd ~/paperless-ngx; docker-compose logs -f`
+- Monitor the Docker Compose log
+  `cd ~/paperless-ngx; docker compose logs -f`
 - Check your script's permission e.g. in case of permission error
   `sudo chmod 755 post-consumption-example.sh`
 - Pipe your scripts's output to a log file e.g.
@@ -248,7 +251,7 @@ document. You will end up getting files like `0000123.pdf` in your media
 directory. This isn't necessarily a bad thing, because you normally
 don't have to access these files manually. However, if you wish to name
 your files differently, you can do that by adjusting the
-`PAPERLESS_FILENAME_FORMAT` configuration option. Paperless adds the
+[`PAPERLESS_FILENAME_FORMAT`](configuration.md#PAPERLESS_FILENAME_FORMAT) configuration option. Paperless adds the
 correct file extension e.g. `.pdf`, `.jpg` automatically.
 
 This variable allows you to configure the filename (folders are allowed)
@@ -311,6 +314,7 @@ Paperless provides the following placeholders within filenames:
 - `{added_day}`: Day added only (number 01-31).
 - `{owner_username}`: Username of document owner, if any, or "none"
 - `{original_name}`: Document original filename, minus the extension, if any, or "none"
+- `{doc_pk}`: The paperless identifier (primary key) for the document.
 
 Paperless will try to conserve the information from your database as
 much as possible. However, some characters that you can use in document
@@ -340,7 +344,7 @@ value.
     Paperless checks the filename of a document whenever it is saved.
     Therefore, you need to update the filenames of your documents and move
     them after altering this setting by invoking the
-    [`document renamer`](/administration#renamer).
+    [`document renamer`](administration.md#renamer).
 
 !!! warning
 
@@ -375,7 +379,7 @@ When a single storage layout is not sufficient for your use case,
 storage paths come to the rescue. Storage paths allow you to configure
 more precisely where each document is stored in the file system.
 
-- Each storage path is a `PAPERLESS_FILENAME_FORMAT` and
+- Each storage path is a [`PAPERLESS_FILENAME_FORMAT`](configuration.md#PAPERLESS_FILENAME_FORMAT) and
   follows the rules described above
 - Each document is assigned a storage path using the matching
   algorithms described above, but can be overwritten at any time
@@ -415,7 +419,7 @@ Insurances/                             # Insurances
 !!! tip
 
     Defining a storage path is optional. If no storage path is defined for a
-    document, the global `PAPERLESS_FILENAME_FORMAT` is applied.
+    document, the global [`PAPERLESS_FILENAME_FORMAT`](configuration.md#PAPERLESS_FILENAME_FORMAT) is applied.
 
 ## Celery Monitoring {#celery-monitoring}
 
@@ -488,7 +492,7 @@ database to be case sensitive. This would prevent a user from creating a
 tag `Name` and `NAME` as they are considered the same.
 
 Per Django documentation, to enable this requires manual intervention.
-To enable case sensetive tables, you can execute the following command
+To enable case sensitive tables, you can execute the following command
 against each table:
 
 `ALTER TABLE <table_name> CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;`
@@ -506,9 +510,9 @@ existing tables) with:
 
 ## Barcodes {#barcodes}
 
-Paperless is able to utilize barcodes for automatically preforming some tasks.
+Paperless is able to utilize barcodes for automatically performing some tasks.
 
-At this time, the library utilized for detection of bacodes supports the following types:
+At this time, the library utilized for detection of barcodes supports the following types:
 
 - AN-13/UPC-A
 - UPC-E
@@ -524,11 +528,11 @@ At this time, the library utilized for detection of bacodes supports the followi
 You may check for updates on the [zbar library homepage](https://github.com/mchehab/zbar).
 For usage in Paperless, the type of barcode does not matter, only the contents of it.
 
-For how to enable barcode usage, see [the configuration](/configuration#barcodes).
+For how to enable barcode usage, see [the configuration](configuration.md#barcodes).
 The two settings may be enabled independently, but do have interactions as explained
 below.
 
-### Document Splitting
+### Document Splitting {#document-splitting}
 
 When enabled, Paperless will look for a barcode with the configured value and create a new document
 starting from the next page. The page with the barcode on it will _not_ be retained. It
@@ -543,3 +547,75 @@ If document splitting via barcode is also enabled, documents will be split when
 barcode is located. However, differing from the splitting, the page with the
 barcode _will_ be retained. This allows application of a barcode to any page, including
 one which holds data to keep in the document.
+
+## Automatic collation of double-sided documents {#collate}
+
+!!! note
+
+    If your scanner supports double-sided scanning natively, you do not need this feature.
+
+This feature is turned off by default, see [configuration](configuration.md#collate) on how to turn it on.
+
+### Summary
+
+If you have a scanner with an automatic document feeder (ADF) that only scans a single side,
+this feature makes scanning double-sided documents much more convenient by automatically
+collating two separate scans into one document, reordering the pages as necessary.
+
+### Usage example
+
+Suppose you have a double-sided document with 6 pages (3 sheets of paper). First,
+put the stack into your ADF as normal, ensuring that page 1 is scanned first. Your ADF
+will now scan pages 1, 3, and 5. Then you (or your scanner, if it supports it) upload
+the scan into the correct sub-directory of the consume folder (`double-sided` by default;
+keep in mind that Paperless will _not_ automatically create the directory for you.)
+Paperless will then process the scan and move it into an internal staging area.
+
+The next step is to turn your stack upside down (without reordering the sheets of paper),
+and scan it once again, your ADF will now scan pages 6, 4, and 2, in that order. Once this
+scan is copied into the sub-directory, Paperless will collate the previous scan with the
+new one, reversing the order of the pages on the second, "even numbered" scan. The
+resulting document will have the pages 1-6 in the correct order, and this new file will
+then be processed as normal.
+
+!!! tip
+
+    When scanning the even numbered pages, you can omit the last empty pages, if there are
+    any. For example, if page 6 is empty, you only need to scan pages 2 and 4. _Do not_ omit
+    empty pages in the middle of the document.
+
+### Things that could go wrong
+
+Paperless will notice when the first, "odd numbered" scan has less pages than the second
+scan (this can happen when e.g. the ADF skipped a few pages in the first pass). In that
+case, Paperless will remove the staging copy as well as the scan, and give you an error
+message asking you to restart the process from scratch, by scanning the odd pages again,
+followed by the even pages.
+
+It's important that the scan files get consumed in the correct order, and one at a time.
+You therefore need to make sure that Paperless is running while you upload the files into
+the directory; and if you're using [polling](configuration.md#polling), make sure that
+`CONSUMER_POLLING` is set to a value lower than it takes for the second scan to appear,
+like 5-10 or even lower.
+
+Another thing that might happen is that you start a double sided scan, but then forget
+to upload the second file. To avoid collating the wrong documents if you then come back
+a day later to scan a new double-sided document, Paperless will only keep an "odd numbered
+pages" file for up to 30 minutes. If more time passes, it will consider the next incoming
+scan a completely new "odd numbered pages" one. The old staging file will get discarded.
+
+### Interaction with "subdirs as tags"
+
+The collation feature can be used together with the [subdirs as tags](configuration.md#consume_config)
+feature (but this is not a requirement). Just create a correctly named double-sided subdir
+in the hierachy and upload your scans there. For example, both `double-sided/foo/bar` as
+well as `foo/bar/double-sided` will cause the collated document to be treated as if it
+were uploaded into `foo/bar` and receive both `foo` and `bar` tags, but not `double-sided`.
+
+### Interaction with document splitting
+
+You can use the [document splitting](#document-splitting) feature, but if you use a normal
+single-sided split marker page, the split document(s) will have an empty page at the front (or
+whatever else was on the backside of the split marker page.) You can work around that by having
+a split marker page that has the split barcode on _both_ sides. This way, the extra page will
+get automatically removed.

docs/api.md

@@ -6,7 +6,7 @@ provides a browsable API for most of its endpoints, which you can
 inspect at `http://<paperless-host>:<port>/api/`. This also documents
 most of the available filters and ordering fields.
 
-The API provides 7 main endpoints:
+The API provides the following main endpoints:
 
 - `/api/documents/`: Full CRUD support, except POSTing new documents.
   See below.
@@ -19,6 +19,9 @@ The API provides 7 main endpoints:
 - `/api/mail_rules/`: Full CRUD support.
 - `/api/users/`: Full CRUD support.
 - `/api/groups/`: Full CRUD support.
+- `/api/share_links/`: Full CRUD support.
+- `/api/custom_fields/`: Full CRUD support.
+- `/api/profile/`: GET, PATCH
 
 All of these endpoints except for the logging endpoint allow you to
 fetch (and edit and delete where appropriate) individual objects by
@@ -47,6 +50,11 @@ fields:
   Read-only.
 - `archived_file_name`: Verbose filename of the archived document.
   Read-only. Null if no archived document is available.
+- `notes`: Array of notes associated with the document.
+- `set_permissions`: Allows setting document permissions. Optional,
+  write-only. See [below](#permissions).
+- `custom_fields`: Array of custom fields & values, specified as
+  `{ field: CUSTOM_FIELD_ID, value: VALUE }`
 
 ## Downloading documents
 
@@ -122,6 +130,11 @@ File metadata is reported as a list of objects in the following form:
 depends on the file type and the metadata available in that specific
 document. Paperless only reports PDF metadata at this point.
 
+## Documents additional endpoints
+
+- `/api/documents/<id>/notes/`: Retrieve notes for a document.
+- `/api/documents/<id>/share_links/`: Retrieve share links for a document.
+
 ## Authorization
 
 The REST api provides three different forms of authentication.
@@ -145,6 +158,10 @@ The REST api provides three different forms of authentication.
 
 3.  Token authentication
 
+    You can create (or re-create) an API token by opening the "My Profile"
+    link in the user dropdown found in the web UI and clicking the circular
+    arrow button.
+
     Paperless also offers an endpoint to acquire authentication tokens.
 
     POST a username and password as a form or json string to
@@ -156,7 +173,7 @@ The REST api provides three different forms of authentication.
     Authorization: Token <token>
     ```
 
-    Tokens can be managed and revoked in the paperless admin.
+    Tokens can also be managed in the Django admin.
 
 ## Searching for documents
 
@@ -165,7 +182,7 @@ specific query parameters cause the API to return full text search
 results:
 
 - `/api/documents/?query=your%20search%20query`: Search for a document
-  using a full text query. For details on the syntax, see [Basic Usage - Searching](/usage#basic-usage_searching).
+  using a full text query. For details on the syntax, see [Basic Usage - Searching](usage.md#basic-usage_searching).
 - `/api/documents/?more_like=1234`: Search for documents similar to
   the document with id 1234.
 
@@ -257,7 +274,6 @@ The endpoint supports the following optional form fields:
 - `document_type`: Similar to correspondent.
 - `tags`: Similar to correspondent. Specify this multiple times to
   have multiple tags added to the document.
-- `owner`: An optional user ID to set as the owner.
 - `archive_serial_number`: An optional archive serial number to set.
 
 The endpoint will immediately return HTTP 200 if the document consumption
@@ -268,6 +284,43 @@ However, querying the tasks endpoint with the returned UUID e.g.
 `/api/tasks/?task_id={uuid}` will provide information on the state of the
 consumption including the ID of a created document if consumption succeeded.
 
+## Permissions
+
+All objects (documents, tags, etc.) allow setting object-level permissions
+with optional `owner` and / or a `set_permissions` parameters which are of
+the form:
+
+```
+"owner": ...,
+"set_permissions": {
+    "view": {
+        "users": [...],
+        "groups": [...],
+    },
+    "change": {
+        "users": [...],
+        "groups": [...],
+    },
+}
+```
+
+!!! note
+
+    Arrays should contain user or group ID numbers.
+
+If these parameters are supplied the object's permissions will be overwritten,
+assuming the authenticated user has permission to do so (the user must be
+the object owner or a superuser).
+
+### Retrieving full permissions
+
+By default, the API will return a truncated version of object-level
+permissions, returning `user_can_change` indicating whether the current user
+can edit the object (either because they are the object owner or have permissions
+granted). You can pass the parameter `full_perms=true` to API calls to view the
+full permissions of objects in a format that mirrors the `set_permissions`
+parameter above.
+
 ## API Versioning
 
 The REST API is versioned since Paperless-ngx 1.3.0.

docs/assets/extra.css

@@ -20,6 +20,28 @@
         margin-left: 4%;
         float: left;
     }
+
+    .grid-flipped-left {
+        width: 66%;
+        float: left;
+    }
+
+    .grid-flipped-right {
+        width: 29%;
+        margin-left: 4%;
+        float: left;
+    }
+
+    .grid-half-left {
+        width: 48%;
+        float: left;
+    }
+
+    .grid-half-right {
+        width: 48%;
+        margin-left: 4%;
+        float: left;
+    }
 }
 
 .grid-left > p {
@@ -31,6 +53,48 @@
     margin: 0;
 }
 
+.clear {
+    clear: both;
+    margin-bottom: 20px;
+    display: block;
+}
+
 .index-callout {
     margin-right: .5rem;
 }
+
+/* make code in headers not bold */
+h4 code {
+    font-weight: normal;
+}
+
+/* Hide config vars from sidebar, toc and move the border on mobile case their hidden */
+.md-nav.md-nav--secondary .md-nav__item .md-nav__link[href*="PAPERLESS_"],
+.md-nav.md-nav--secondary .md-nav__item .md-nav__link[href*="USERMAP_"] {
+    display: none;
+}
+
+@media screen and (max-width: 76.1875em) {
+    .md-nav--primary .md-nav__item {
+        border-top: none;
+    }
+
+    .md-nav--primary .md-nav__link {
+        border-top: .05rem solid var(--md-default-fg-color--lightest);
+    }
+}
+
+/* Show search shortcut key */
+[data-md-toggle="search"]:not(:checked) ~ .md-header .md-search__form::after {
+    position: absolute;
+    top: .3rem;
+    right: .3rem;
+    display: block;
+    padding: .1rem .4rem;
+    color: var(--md-default-fg-color--lighter);
+    font-weight: bold;
+    font-size: .8rem;
+    border: .05rem solid var(--md-default-fg-color--lighter);
+    border-radius: .1rem;
+    content: "/";
+  }

docs/development.md

@@ -9,7 +9,7 @@ following way:
 - `main` always represents the latest release and will only see
   changes when a new release is made.
 - `dev` contains the code that will be in the next release.
-- `feature-X` contain bigger changes that will be in some release, but
+- `feature-X` contains bigger changes that will be in some release, but
   not necessarily the next one.
 
 When making functional changes to Paperless-ngx, _always_ make your changes
@@ -58,10 +58,10 @@ first-time setup.
 
 !!! note
 
-    Every command is executed directly from the root folder of the project unless specified otherwise.
+      Every command is executed directly from the root folder of the project unless specified otherwise.
 
 1.  Install prerequisites + pipenv as mentioned in
-    [Bare metal route](/setup#bare_metal).
+    [Bare metal route](setup.md#bare_metal).
 
 2.  Copy `paperless.conf.example` to `paperless.conf` and enable debug
     mode within the file via `PAPERLESS_DEBUG=true`.
@@ -177,69 +177,69 @@ The front end is built using AngularJS. In order to get started, you need Node.j
 
     The following commands are all performed in the `src-ui`-directory. You will need a running back end (including an active session) to connect to the back end API. To spin it up refer to the commands under the section [above](#back-end-development).
 
-1. Install the Angular CLI. You might need sudo privileges
-   to perform this command:
+1.  Install the Angular CLI. You might need sudo privileges to perform this command:
 
-   ```bash
-   $ npm install -g @angular/cli
-   ```
+    ```bash
+    $ npm install -g @angular/cli
+    ```
 
-2. Make sure that it's on your path.
+2.  Make sure that it's on your path.
 
-3. Install all neccessary modules:
+3.  Install all necessary modules:
 
-   ```bash
-   $ npm install
-   ```
+    ```bash
+    $ npm install
+    ```
 
-4. You can launch a development server by running:
+4.  You can launch a development server by running:
 
-   ```bash
-   $ ng serve
-   ```
+    ```bash
+    $ ng serve
+    ```
 
-   This will automatically update whenever you save. However, in-place
-   compilation might fail on syntax errors, in which case you need to
-   restart it.
+    This will automatically update whenever you save. However, in-place
+    compilation might fail on syntax errors, in which case you need to
+    restart it.
 
-   By default, the development server is available on `http://localhost:4200/` and is configured to access the API at
-   `http://localhost:8000/api/`, which is the default of the backend. If you enabled `DEBUG` on the back end, several security overrides for allowed hosts, CORS and X-Frame-Options are in place so that the front end behaves exactly as in production.
+    By default, the development server is available on `http://localhost:4200/` and is configured to access the API at
+    `http://localhost:8000/api/`, which is the default of the backend. If you enabled `DEBUG` on the back end, several security overrides for allowed hosts, CORS and X-Frame-Options are in place so that the front end behaves exactly as in production.
 
 ### Testing and code style
 
-- The front end code (.ts, .html, .scss) use `prettier` for code
-  formatting via the Git `pre-commit` hooks which run automatically on
-  commit. See [above](#code-formatting-with-pre-commit-hooks) for installation instructions. You can also run this via the CLI with a
-  command such as
+The front end code (.ts, .html, .scss) use `prettier` for code
+formatting via the Git `pre-commit` hooks which run automatically on
+commit. See [above](#code-formatting-with-pre-commit-hooks) for installation instructions. You can also run this via the CLI with a
+command such as
 
-  ```bash
-  $ git ls-files -- '*.ts' | xargs pre-commit run prettier --files
-  ```
+```bash
+$ git ls-files -- '*.ts' | xargs pre-commit run prettier --files
+```
 
-- Front end testing uses jest and cypress. There is currently a need
-  for significantly more front end tests. Unit tests and e2e tests,
-  respectively, can be run non-interactively with:
+Front end testing uses Jest and Playwright. Unit tests and e2e tests,
+respectively, can be run non-interactively with:
 
-  ```bash
-  $ ng test
-  $ npm run e2e:ci
-  ```
+```bash
+$ ng test
+$ npx playwright test
+```
 
-  - Cypress also includes a UI which can be run with:
+Playwright also includes a UI which can be run with:
 
-    ```bash
-    $ ./node_modules/.bin/cypress open
-    ```
+```bash
+$ npx playwright test --ui
+```
 
-- In order to build the front end and serve it as part of Django, execute:
+### Building the frontend
 
-  ```bash
-  $ ng build --configuration production
-  ```
+In order to build the front end and serve it as part of Django, execute:
 
-  This will build the front end and put it in a location from which the
-  Django server will serve it as static content. This way, you can verify
-  that authentication is working.
+```bash
+$ ng build --configuration production
+```
+
+This will build the front end and put it in a location from which the
+Django server will serve it as static content. This way, you can verify
+that authentication is working.
 
 ## Localization
 
@@ -362,7 +362,7 @@ If you want to build the documentation locally, this is how you do it:
 
 3.  Serve the documentation. This will spin up a
     copy of the documentation at http://127.0.0.1:8000
-    that will automatically refresh everytime you change
+    that will automatically refresh every time you change
     something.
 
     ```bash
@@ -374,13 +374,10 @@ If you want to build the documentation locally, this is how you do it:
 The docker image is primarily built by the GitHub actions workflow, but
 it can be faster when developing to build and tag an image locally.
 
-To provide the build arguments automatically, build the image using the
-helper script `build-docker-image.sh`.
+Building the image works as with any image:
 
-Building the docker image from source:
-
-```bash
-./build-docker-image.sh Dockerfile -t <your-tag>
+```
+docker build --file Dockerfile --tag paperless:local --progress simple .
 ```
 
 ## Extending Paperless-ngx
@@ -398,7 +395,7 @@ responsible for:
 - Retrieving the content from the original
 - Creating a thumbnail
 - _optional:_ Retrieving a created date from the original
-- _optional:_ Creainge an archived document from the original
+- _optional:_ Creating an archived document from the original
 
 Custom parsers can be added to Paperless-ngx to support more file types. In
 order to do that, you need to write the parser itself and announce its

docs/faq.md

@@ -3,15 +3,16 @@
 ## _What's the general plan for Paperless-ngx?_
 
 **A:** While Paperless-ngx is already considered largely
-"feature-complete" it is a community-driven project and development
-will be guided in this way. New features can be submitted via GitHub
-discussions and "up-voted" by the community but this is not a
-guarantee the feature will be implemented. This project will always be
+"feature-complete", it is a community-driven project and development
+will be guided in this way. New features can be submitted via
+[GitHub discussions](https://github.com/paperless-ngx/paperless-ngx/discussions)
+and "up-voted" by the community, but this is not a
+guarantee that the feature will be implemented. This project will always be
 open to collaboration in the form of PRs, ideas etc.
 
 ## _I'm using docker. Where are my documents?_
 
-**A:** Your documents are stored inside the docker volume
+**A:** By default, your documents are stored inside the docker volume
 `paperless_media`. Docker manages this volume automatically for you. It
 is a persistent storage and will persist as long as you don't
 explicitly delete it. The actual location depends on your host operating
@@ -27,6 +28,12 @@ system. On Linux, chances are high that this location is
     files around manually. This folder is meant to be entirely managed by
     docker and paperless.
 
+!!! note
+
+    Files consumed from the consumption directory are re-created inside
+    this media directory and are removed from the consumption directory
+    itself.
+
 ## Let's say I want to switch tools in a year. Can I easily move to other systems?
 
 **A:** Your documents are stored as plain files inside the media folder.
@@ -39,8 +46,8 @@ elsewhere. Here are a couple notes about that.
 - By default, paperless uses the internal ID of each document as its
   filename. This might not be very convenient for export. However, you
   can adjust the way files are stored in paperless by
-  [configuring the filename format](/advanced_usage#file-name-handling).
-- [The exporter](/administration#exporter) is
+  [configuring the filename format](advanced_usage.md#file-name-handling).
+- [The exporter](administration.md#exporter) is
   another easy way to get your files out of paperless with reasonable
   file names.
 
@@ -52,7 +59,7 @@ elsewhere. Here are a couple notes about that.
   WebP images are processed with OCR and converted into PDF documents.
 - Plain text documents are supported as well and are added verbatim to
   paperless.
-- With the optional Tika integration enabled (see [Tika configuration](/configuration#tika),
+- With the optional Tika integration enabled (see [Tika configuration](https://docs.paperless-ngx.com/configuration#tika)),
   Paperless also supports various Office documents (.docx, .doc, odt,
   .ppt, .pptx, .odp, .xls, .xlsx, .ods).
 
@@ -71,20 +78,27 @@ has to do much less work to serve the data.
 !!! note
 
     You can adjust some of the settings so that paperless uses less
-    processing power. See [setup](/setup#less-powerful-devices) for details.
+    processing power. See [setup](setup.md#less-powerful-devices) for details.
 
 ## _How do I install paperless-ngx on Raspberry Pi?_
 
-**A:** Docker images are available for armv7 and arm64 hardware, so just
-follow the docker-compose instructions. Apart from more required disk
+**A:** Docker images are available for arm64 hardware, so just
+follow the [Docker Compose instructions](https://docs.paperless-ngx.com/setup/#installation). Apart from more required disk
 space compared to a bare metal installation, docker comes with close to
 zero overhead, even on Raspberry Pi.
 
-If you decide to got with the bare metal route, be aware that some of
+If you decide to go with the bare metal route, be aware that some of
 the python requirements do not have precompiled packages for ARM /
 ARM64. Installation of these will require additional development
 libraries and compilation will take a long time.
 
+!!! note
+
+    For ARMv7 (32-bit) systems, paperless may still function, but it could require
+    modifications to the Dockerfile (if using Docker) or additional
+    tools for installing bare metal.  It is suggested to upgrade to arm64
+    instead.
+
 ## _How do I run this on Unraid?_
 
 **A:** Paperless-ngx is available as [community
@@ -96,14 +110,11 @@ Fahrer](https://github.com/Tooa) created a container template for that.
 **A:** I honestly don't know! As for all other devices that might be
 able to run paperless, you're a bit on your own. If you can't run the
 docker image, the documentation has instructions for bare metal
-installs. I'm running paperless on an i3 processor from 2015 or so.
-This is also what I use to test new releases with. Apart from that, I
-also have a Raspberry Pi, which I occasionally build the image on and
-see if it works.
+installs.
 
 ## _How do I proxy this with NGINX?_
 
-**A:** See [here](/setup#nginx).
+**A:** See [the wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Using-a-Reverse-Proxy-with-Paperless-ngx#nginx).
 
 ## _How do I get WebSocket support with Apache mod_wsgi_?
 

docs/index.md

@@ -5,7 +5,7 @@
 **Paperless-ngx** is a _community-supported_ open-source document management system that transforms your
 physical documents into a searchable online archive so you can keep, well, _less paper_.
 
-[Get started](/setup){ .md-button .md-button--primary .index-callout }
+[Get started](setup.md){ .md-button .md-button--primary .index-callout }
 [Demo](https://demo.paperless-ngx.com){ .md-button .md-button--secondary target=\_blank }
 
 </div>
@@ -15,103 +15,161 @@ physical documents into a searchable online archive so you can keep, well, _less
 </div>
 <div class="clear"></div>
 
-## Why This Exists
+## Features
 
-Paper is a nightmare. Environmental issues aside, there's no excuse for
-it in the 21st century. It takes up space, collects dust, doesn't
-support any form of a search feature, indexing is tedious, it's heavy
-and prone to damage & loss.
+-   **Organize and index** your scanned documents with tags, correspondents, types, and more.
+-   Performs **OCR** on your documents, adding searchable and selectable text, even to documents scanned with only images.
+-   Utilizes the open-source Tesseract engine to recognize more than 100 languages.
+-   Documents are saved as PDF/A format which is designed for long term storage, alongside the unaltered originals.
+-   Uses machine-learning to automatically add tags, correspondents and document types to your documents.
+-   Supports PDF documents, images, plain text files, Office documents (Word, Excel, Powerpoint, and LibreOffice equivalents)[^1] and more.
+-   Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and their format can be configured freely with different configurations assigned to different documents.
+-   **Beautiful, modern web application** that features:
+    -   Customizable dashboard with statistics.
+    -   Filtering by tags, correspondents, types, and more.
+    -   Bulk editing of tags, correspondents, types and more.
+    -   Drag-and-drop uploading of documents throughout the app.
+    -   Customizable views can be saved and displayed on the dashboard and / or sidebar.
+    -   Support for custom fields of various data types.
+    -   Shareable public links with optional expiration.
+-   **Full text search** helps you find what you need:
+    -   Auto completion suggests relevant words from your documents.
+    -   Results are sorted by relevance to your search query.
+    -   Highlighting shows you which parts of the document matched the query.
+    -   Searching for similar documents ("More like this")
+-   **Email processing**[^1]: import documents from your email accounts:
+    -   Configure multiple accounts and rules for each account.
+    -   After processing, paperless can perform actions on the messages such as marking as read, deleting and more.
+-   A built-in robust **multi-user permissions** system that supports 'global' permissions as well as per document or object.
+-   A powerful templating system that gives you more control over the consumption pipeline.
+-   **Optimized** for multi core systems: Paperless-ngx consumes multiple documents in parallel.
+-   The integrated sanity checker makes sure that your document archive is in good health.
 
-This software is designed to make "going paperless" easier. No more worrying
-about finding stuff again, feed documents right from the post box into
-the scanner and then shred them. Perhaps you might find it useful too.
+[^1]: Office document and email consumption support is optional and provided by Apache Tika (see [configuration](https://docs.paperless-ngx.com/configuration/#tika))
 
 ## Paperless, a history
 
-Paperless is a simple Django application running in two parts: a
-_Consumer_ (the thing that does the indexing) and the _Web server_ (the
-part that lets you search & download already-indexed documents). If you
-want to learn more about its functions keep on reading after the
-installation section.
+Paperless-ngx is the official successor to the original [Paperless](https://github.com/the-paperless-project/paperless) & [Paperless-ng](https://github.com/jonaswinkler/paperless-ng) projects and is designed to distribute the responsibility of advancing and supporting the project among a team of people. [Consider joining us!](https://github.com/paperless-ngx/paperless-ngx#community-support)
 
-Paperless-ngx is a document management system that transforms your
-physical documents into a searchable online archive so you can keep,
-well, _less paper_.
-
-Paperless-ngx forked from paperless-ng to continue the great work and
-distribute responsibility of supporting and advancing the project among
-a team of people.
-
-NG stands for both Angular (the framework used for the Frontend) and
-next-gen. Publishing this project under a different name also avoids
-confusion between paperless and paperless-ngx.
-
-If you want to learn about what's different in paperless-ngx from
-Paperless, check out these resources in the documentation:
-
-- [Some screenshots](#screenshots) of the new UI are available.
-- Read [this section](/advanced_usage#automatic-matching) if you want to learn about how paperless automates all
-  tagging using machine learning.
-- Paperless now comes with a [proper email consumer](/usage#usage-email) that's fully tested and production ready.
-- Paperless creates searchable PDF/A documents from whatever you put into the consumption directory. This means
-  that you can select text in image-only documents coming from your scanner.
-- See [this note](/administration#encryption) about GnuPG encryption in paperless-ngx.
-- Paperless is now integrated with a
-  [task processing queue](/setup#task_processor) that tells you at a glance when and why something is not working.
-- The [changelog](/changelog) contains a detailed list of all changes in paperless-ngx.
+Further discussion of the transition between these projects can be found at
+[ng#1599](https://github.com/jonaswinkler/paperless-ng/issues/1599) and [ng#1632](https://github.com/jonaswinkler/paperless-ng/issues/1632).
 
 ## Screenshots
 
-This is what Paperless-ngx looks like.
+Paperless-ngx aims to be as nice to use as it is useful. Check out some screenshots below.
 
-The dashboard shows customizable views on your document and allows
-document uploads:
+<div class="grid-flipped-left" markdown>
+  ![image](assets/screenshots/dashboard.png)
+</div>
+<div class="grid-flipped-right" markdown>
+  The dashboard shows saved views which can be sorted. Documents can be uploaded with the button or dropped anywhere in the application.
+</div>
+<div class="clear"></div>
 
-[![image](assets/screenshots/dashboard.png)](assets/screenshots/dashboard.png)
+The document list provides three different styles to browse your documents.
 
-The document list provides three different styles to scroll through your
-documents:
+![image](assets/screenshots/documents-table.png){: style="width:32%"}
+![image](assets/screenshots/documents-smallcards.png){: style="width:32%"}
+![image](assets/screenshots/documents-largecards.png){: style="width:32%"}
 
-[![image](assets/screenshots/documents-table.png)](assets/screenshots/documents-table.png)
+<div class="clear"></div>
 
-[![image](assets/screenshots/documents-smallcards.png)](assets/screenshots/documents-smallcards.png)
+<div class="grid-left" markdown>
+  Use the 'slim' sidebar to focus on your docs and minimize the UI.
+</div>
+<div class="grid-right" markdown>
+  ![image](assets/screenshots/documents-smallcards-slimsidebar.png)
+</div>
+<div class="clear"></div>
 
-[![image](assets/screenshots/documents-largecards.png)](assets/screenshots/documents-largecards.png)
+Of course, Paperless-ngx also supports dark mode:
 
-Paperless-ngx also supports dark mode:
+![image](assets/screenshots/documents-smallcards-dark.png)
 
-[![image](assets/screenshots/documents-smallcards-dark.png)](assets/screenshots/documents-smallcards-dark.png)
+<div class="clear"></div>
 
-Extensive filtering mechanisms:
+<div class="grid-left" markdown>
+  Quickly find documents with extensive filtering mechanisms.
+</div>
+<div class="grid-right" markdown>
+  ![image](assets/screenshots/documents-filter.png)
+</div>
+<div class="clear"></div>
+<div class="grid-left" markdown>
+  And perform bulk edit operations to set tags, correspondents, etc. as well as permissions.
+</div>
+<div class="grid-right" markdown>
+  ![image](assets/screenshots/bulk-edit.png)
+</div>
+<div class="clear"></div>
 
-[![image](assets/screenshots/documents-filter.png)](assets/screenshots/documents-filter.png)
+Side-by-side editing of documents.
 
-Bulk editing of document tags, correspondents, etc.:
+![image](assets/screenshots/editing.png)
 
-[![image](assets/screenshots/bulk-edit.png)](assets/screenshots/bulk-edit.png)
+<div class="grid-left" markdown>
+  Support for custom fields.
 
-Side-by-side editing of documents:
+![image](assets/screenshots/custom_field1.png)
 
-[![image](assets/screenshots/editing.png)](assets/screenshots/editing.png)
+</div>
+<div class="grid-right" markdown>
+  ![image](assets/screenshots/custom_field2.png)
+</div>
+<div class="clear"></div>
 
-Tag editing. This looks about the same for correspondents and document
-types.
+<div class="grid-left" markdown>
+  A robust permissions system with support for 'global' and document / object permissions.
 
-[![image](assets/screenshots/new-tag.png)](assets/screenshots/new-tag.png)
+![image](assets/screenshots/permissions_global.png)
 
-Searching provides auto complete and highlights the results.
+</div>
+<div class="grid-right" markdown>
+  ![image](assets/screenshots/permissions_document.png)
+</div>
+<div class="clear"></div>
 
-[![image](assets/screenshots/search-preview.png)](assets/screenshots/search-preview.png)
+<div class="grid-left" markdown>
+  Searching provides auto complete and highlights the results.
 
-[![image](assets/screenshots/search-results.png)](assets/screenshots/search-results.png)
+![image](assets/screenshots/search-preview.png)
 
-Fancy mail filters!
+</div>
+<div class="grid-right" markdown>
+  ![image](assets/screenshots/search-results.png)
+</div>
+<div class="clear"></div>
 
-[![image](assets/screenshots/mail-rules-edited.png)](assets/screenshots/mail-rules-edited.png)
+Tag, correspondent, document type and storage path editing.
+
+![image](assets/screenshots/new-tag.png){: style="width:21%; float: left"}
+![image](assets/screenshots/new-correspondent.png){: style="width:21%; margin-left: 4%; float: left"}
+![image](assets/screenshots/new-document_type.png){: style="width:21%; margin-left: 4%; float: left"}
+![image](assets/screenshots/new-storage_path.png){: style="width:21%; margin-left: 4%; float: left"}
+
+<div class="clear"></div>
+
+<div class="grid-half-left" markdown>
+  Mail rules support various filters and actions for incoming e-mails.
+
+![image](assets/screenshots/mail-rules-edited.png)
+
+</div>
+<div class="grid-half-right" markdown>
+  Consumption templates provide finer control over the document pipeline.
+
+![image](assets/screenshots/consumption_template.png)
+
+</div>
+<div class="clear"></div>
+
+<div class="clear"></div>
 
 Mobile devices are supported.
 
-[![image](assets/screenshots/mobile.png)](assets/screenshots/mobile.png)
+![image](assets/screenshots/mobile1.png){: style="width:32%"}
+![image](assets/screenshots/mobile2.png){: style="width:32%"}
+![image](assets/screenshots/mobile3.png){: style="width:32%"}
 
 ## Support
 
@@ -131,7 +189,7 @@ People interested in continuing the work on paperless-ngx are encouraged to reac
 
 ### Translation
 
-Paperless-ngx is available in many languages that are coordinated on [Crowdin](https://crwd.in/paperless-ngx). If you want to help out by translating paperless-ngx into your language, please head over to https://crwd.in/paperless-ngx, and thank you!
+Paperless-ngx is available in many languages that are coordinated on [Crowdin](https://crwd.in/paperless-ngx). If you want to help out by translating paperless-ngx into your language, please head over to the [Paperless-ngx project at Crowdin](https://crwd.in/paperless-ngx), and thank you!
 
 ## Scanners & Software
 

docs/setup.md

@@ -25,14 +25,22 @@ necessary configuration files, pull the docker image, start paperless
 and create your user account. This script essentially performs all the
 steps described in [Docker setup](#docker_hub) automatically.
 
-1.  Make sure that docker and docker-compose are installed.
+1.  Make sure that Docker and Docker Compose are installed.
+
+    !!! tip
+    See the Docker installation instructions at https://docs.docker.com/engine/install/
 
 2.  Download and run the installation script:
 
     ```shell-session
-    $ bash -c "$(curl -L https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/install-paperless-ngx.sh)"
+    $ bash -c "$(curl --location --silent --show-error https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/install-paperless-ngx.sh)"
     ```
 
+    !!! note
+
+        macOS users will need to install e.g. [gnu-sed](https://formulae.brew.sh/formula/gnu-sed) with support
+        for running as `sed`.
+
 ### From GHCR / Docker Hub {#docker_hub}
 
 1.  Login with your user and create a folder in your home-directory to have a place for your
@@ -57,19 +65,19 @@ steps described in [Docker setup](#docker_hub) automatically.
         For new installations, it is recommended to use PostgreSQL as the
         database backend.
 
-3.  Install [Docker](https://www.docker.com/) and
-    [docker-compose](https://docs.docker.com/compose/install/).
+3.  Install [Docker](https://docs.docker.com/engine/install/) and
+    [Docker Compose](https://docs.docker.com/compose/install/).
 
     !!! warning
 
         If you want to use the included `docker-compose.*.yml` file, you
-        need to have at least Docker version **17.09.0** and docker-compose
-        version **1.17.0**. To check do: `docker-compose -v` or `docker -v`
+        need to have at least Docker version **17.09.0** and Docker Compose
+        version **v2**. To check do: `docker compose -v` or `docker -v`
 
         See the [Docker installation guide](https://docs.docker.com/engine/install/) on how to install the current
         version of Docker for your operating system or Linux distribution of
-        choice. To get the latest version of docker-compose, follow the
-        [docker-compose installation guide](https://docs.docker.com/compose/install/linux/) if your package repository
+        choice. To get the latest version of Docker Compose, follow the
+        [Docker Compose installation guide](https://docs.docker.com/compose/install/linux/) if your package repository
         doesn't include it.
 
 4.  Modify `docker-compose.yml` to your preferences. You may want to
@@ -119,7 +127,7 @@ steps described in [Docker setup](#docker_hub) automatically.
       user in the container. This value (`user_id` below), should be
       the same id that `USERMAP_UID` and `USERMAP_GID` are set to in
       the next step. See `USERMAP_UID` and `USERMAP_GID`
-      [here](/configuration#docker).
+      [here](configuration.md#docker).
 
     Your entry for Paperless should contain something like:
 
@@ -143,12 +151,12 @@ steps described in [Docker setup](#docker_hub) automatically.
     !!! note
 
         You can copy any setting from the file `paperless.conf.example` and
-        paste it here. Have a look at [configuration](/configuration) to see what's available.
+        paste it here. Have a look at [configuration](configuration.md) to see what's available.
 
     !!! note
 
         You can utilize Docker secrets for configuration settings by
-        appending `_FILE` to configuration values. For example `PAPERLESS_DBUSER`
+        appending `_FILE` to configuration values. For example [`PAPERLESS_DBUSER`](configuration.md#PAPERLESS_DBUSER)
         can be set using `PAPERLESS_DBUSER_FILE=/var/run/secrets/password.txt`.
 
     !!! warning
@@ -157,16 +165,16 @@ steps described in [Docker setup](#docker_hub) automatically.
         system notifications with `inotify`. When storing the consumption
         directory on such a file system, paperless will not pick up new
         files with the default configuration. You will need to use
-        `PAPERLESS_CONSUMER_POLLING`, which will disable inotify. See
-        [here](/configuration#polling).
+        [`PAPERLESS_CONSUMER_POLLING`](configuration.md#PAPERLESS_CONSUMER_POLLING), which will disable inotify. See
+        [here](configuration.md#polling).
 
-6.  Run `docker-compose pull`. This will pull the image.
+6.  Run `docker compose pull`. This will pull the image.
 
 7.  To be able to login, you will need a super user. To create it,
     execute the following command:
 
     ```shell-session
-    $ docker-compose run --rm webserver createsuperuser
+    $ docker compose run --rm webserver createsuperuser
     ```
 
     or using docker exec from within the container:
@@ -178,7 +186,7 @@ steps described in [Docker setup](#docker_hub) automatically.
     This will prompt you to set a username, an optional e-mail address
     and finally a password (at least 8 characters).
 
-8.  Run `docker-compose up -d`. This will create and start the necessary containers.
+8.  Run `docker compose up -d`. This will create and start the necessary containers.
 
 9.  The default `docker-compose.yml` exports the webserver on your local
     port
@@ -204,39 +212,27 @@ steps described in [Docker setup](#docker_hub) automatically.
     root as well.
 
 3.  In the `docker-compose.yml` file, find the line that instructs
-    docker-compose to pull the paperless image from Docker Hub:
+    Docker Compose to pull the paperless image from Docker Hub:
 
     ```yaml
     webserver:
       image: ghcr.io/paperless-ngx/paperless-ngx:latest
     ```
 
-    and replace it with a line that instructs docker-compose to build
+    and replace it with a line that instructs Docker Compose to build
     the image from the current working directory instead:
 
     ```yaml
     webserver:
       build:
         context: .
-        args:
-          QPDF_VERSION: x.y.x
-          PIKEPDF_VERSION: x.y.z
-          PSYCOPG2_VERSION: x.y.z
-          JBIG2ENC_VERSION: 0.29
     ```
 
-    !!! note
-
-        You should match the build argument versions to the version for the
-        release you have checked out. These are pre-built images with
-        certain, more updated software. If you want to build these images
-        your self, that is possible, but beyond the scope of these steps.
-
 4.  Follow steps 3 to 8 of [Docker Setup](#docker_hub). When asked to run
-    `docker-compose pull` to pull the image, do
+    `docker compose pull` to pull the image, do
 
     ```shell-session
-    $ docker-compose build
+    $ docker compose build
     ```
 
     instead to build the image.
@@ -250,10 +246,11 @@ supported.
 
 1.  Install dependencies. Paperless requires the following packages.
 
-    - `python3` 3.8, 3.9
+    - `python3` - 3.9 - 3.11 are supported
     - `python3-pip`
     - `python3-dev`
     - `default-libmysqlclient-dev` for MariaDB
+    - `pkg-config` for mysqlclient (python dependency)
     - `fonts-liberation` for generating thumbnails for plain text
       files
     - `imagemagick` >= 6 for PDF conversion
@@ -268,7 +265,7 @@ supported.
     Use this list for your preferred package management:
 
     ```
-    python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev libmagic-dev mime-support libzbar0 poppler-utils
+    python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev mime-support libzbar0 poppler-utils
     ```
 
     These dependencies are required for OCRmyPDF, which is used for text
@@ -336,41 +333,41 @@ supported.
     home folder of the user you created before (`/opt/paperless`).
 
     Optional: If you cloned the git repo, you will have to
-    compile the frontend yourself, see [here](/development#front-end-development)
+    compile the frontend yourself, see [here](development.md#front-end-development)
     and use the `build` step, not `serve`.
 
-6.  Configure paperless. See [configuration](/configuration) for details.
+6.  Configure paperless. See [configuration](configuration.md) for details.
     Edit the included `paperless.conf` and adjust the settings to your
     needs. Required settings for getting
     paperless running are:
 
-    - `PAPERLESS_REDIS` should point to your redis server, such as
+    - [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) should point to your redis server, such as
       <redis://localhost:6379>.
-    - `PAPERLESS_DBENGINE` optional, and should be one of `postgres`,
+    - [`PAPERLESS_DBENGINE`](configuration.md#PAPERLESS_DBENGINE) optional, and should be one of `postgres`,
       `mariadb`, or `sqlite`
-    - `PAPERLESS_DBHOST` should be the hostname on which your
+    - [`PAPERLESS_DBHOST`](configuration.md#PAPERLESS_DBHOST) should be the hostname on which your
       PostgreSQL server is running. Do not configure this to use
       SQLite instead. Also configure port, database name, user and
       password as necessary.
-    - `PAPERLESS_CONSUMPTION_DIR` should point to a folder which
+    - [`PAPERLESS_CONSUMPTION_DIR`](configuration.md#PAPERLESS_CONSUMPTION_DIR) should point to a folder which
       paperless should watch for documents. You might want to have
-      this somewhere else. Likewise, `PAPERLESS_DATA_DIR` and
-      `PAPERLESS_MEDIA_ROOT` define where paperless stores its data.
+      this somewhere else. Likewise, [`PAPERLESS_DATA_DIR`](configuration.md#PAPERLESS_DATA_DIR) and
+      [`PAPERLESS_MEDIA_ROOT`](configuration.md#PAPERLESS_MEDIA_ROOT) define where paperless stores its data.
       If you like, you can point both to the same directory.
-    - `PAPERLESS_SECRET_KEY` should be a random sequence of
+    - [`PAPERLESS_SECRET_KEY`](configuration.md#PAPERLESS_SECRET_KEY) should be a random sequence of
       characters. It's used for authentication. Failure to do so
       allows third parties to forge authentication credentials.
-    - `PAPERLESS_URL` if you are behind a reverse proxy. This should
+    - [`PAPERLESS_URL`](configuration.md#PAPERLESS_URL) if you are behind a reverse proxy. This should
       point to your domain. Please see
-      [configuration](/configuration) for more
+      [configuration](configuration.md) for more
       information.
 
     Many more adjustments can be made to paperless, especially the OCR
     part. The following options are recommended for everyone:
 
-    - Set `PAPERLESS_OCR_LANGUAGE` to the language most of your
+    - Set [`PAPERLESS_OCR_LANGUAGE`](configuration.md#PAPERLESS_OCR_LANGUAGE) to the language most of your
       documents are written in.
-    - Set `PAPERLESS_TIME_ZONE` to your local time zone.
+    - Set [`PAPERLESS_TIME_ZONE`](configuration.md#PAPERLESS_TIME_ZONE) to your local time zone.
 
     !!! warning
 
@@ -478,7 +475,7 @@ supported.
         in front of gunicorn instead.
 
         For instructions on how to use nginx for that,
-        [see the instructions below](/setup#nginx).
+        [see the wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Using-a-Reverse-Proxy-with-Paperless-ngx#nginx).
 
     !!! warning
 
@@ -516,7 +513,7 @@ supported.
     not available for most distributions.
 
 15. Optional: If using the NLTK machine learning processing (see
-    `PAPERLESS_ENABLE_NLTK` in [configuration](/configuration#software_tweaks) for details),
+    [`PAPERLESS_ENABLE_NLTK`](configuration.md#PAPERLESS_ENABLE_NLTK) for details),
     download the NLTK data for the Snowball
     Stemmer, Stopwords and Punkt tokenizer to your
     `PAPERLESS_DATA_DIR/nltk`. Refer to the [NLTK
@@ -547,14 +544,14 @@ to
 image: ghcr.io/paperless-ngx/paperless-ngx:latest
 ```
 
-and then run `docker-compose up -d` which will pull the new image
+and then run `docker compose up -d` which will pull the new image
 recreate the container. That's it!
 
 Users who installed with the bare-metal route should also update their
 Git clone to point to `https://github.com/paperless-ngx/paperless-ngx`,
 e.g. using the command
 `git remote set-url origin https://github.com/paperless-ngx/paperless-ngx`
-and then pull the lastest version.
+and then pull the latest version.
 
 ## Migrating from Paperless
 
@@ -565,7 +562,7 @@ your setup depending on how you installed paperless.
 This setup describes how to update an existing paperless Docker
 installation. The important things to keep in mind are as follows:
 
-- Read the [changelog](/changelog) and
+- Read the [changelog](changelog.md) and
   take note of breaking changes.
 - You should decide if you want to stick with SQLite or want to
   migrate your database to PostgreSQL. See [documentation](#sqlite_to_psql)
@@ -576,7 +573,7 @@ installation. The important things to keep in mind are as follows:
 - The task scheduler of paperless, which is used to execute periodic
   tasks such as email checking and maintenance, requires a
   [redis](https://redis.io/) message broker instance. The
-  docker-compose route takes care of that.
+  Docker Compose route takes care of that.
 - The layout of the folder structure for your documents and data
   remains the same, so you can just plug your old docker volumes into
   paperless-ngx and expect it to find everything where it should be.
@@ -587,7 +584,7 @@ Migration to paperless-ngx is then performed in a few simple steps:
 
     ```bash
     $ cd /path/to/current/paperless
-    $ docker-compose down
+    $ docker compose down
     ```
 
 2.  Do a backup for two purposes: If something goes wrong, you still
@@ -595,7 +592,7 @@ Migration to paperless-ngx is then performed in a few simple steps:
     switch back to paperless.
 
 3.  Download the latest release of paperless-ngx. You can either go with
-    the docker-compose files from
+    the Docker Compose files from
     [here](https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose)
     or clone the repository to build the image yourself (see
     [above](#docker_build)). You can
@@ -625,14 +622,14 @@ Migration to paperless-ngx is then performed in a few simple steps:
     See [Docker setup](#docker_hub) details on
     which edits are advised.
 
-6.  [Update paperless.](/administration#updating)
+6.  [Update paperless.](administration.md#updating)
 
 7.  In order to find your existing documents with the new search
     feature, you need to invoke a one-time operation that will create
     the search index:
 
     ```shell-session
-    $ docker-compose run --rm webserver document_index reindex
+    $ docker compose run --rm webserver document_index reindex
     ```
 
     This will migrate your database and create the search index. After
@@ -641,7 +638,7 @@ Migration to paperless-ngx is then performed in a few simple steps:
 8.  Start paperless-ngx.
 
     ```bash
-    $ docker-compose up -d
+    $ docker compose up -d
     ```
 
     This will run paperless in the background and automatically start it
@@ -664,28 +661,28 @@ commands as well.
 1.  Stop and remove the paperless container
 2.  If using an external database, stop the container
 3.  Update Redis configuration
-    a) If `REDIS_URL` is already set, change it to `PAPERLESS_REDIS`
+    a) If `REDIS_URL` is already set, change it to [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS)
     and continue to step 4.
     b) Otherwise, in the `docker-compose.yml` add a new service for
     Redis, following [the example compose
     files](https://github.com/paperless-ngx/paperless-ngx/tree/main/docker/compose)
-    c) Set the environment variable `PAPERLESS_REDIS` so it points to
+    c) Set the environment variable [`PAPERLESS_REDIS`](configuration.md#PAPERLESS_REDIS) so it points to
     the new Redis container
 4.  Update user mapping
     a) If set, change the environment variable `PUID` to `USERMAP_UID`
     b) If set, change the environment variable `PGID` to `USERMAP_GID`
 5.  Update configuration paths
-    a) Set the environment variable `PAPERLESS_DATA_DIR` to `/config`
+    a) Set the environment variable [`PAPERLESS_DATA_DIR`](configuration.md#PAPERLESS_DATA_DIR) to `/config`
 6.  Update media paths
-    a) Set the environment variable `PAPERLESS_MEDIA_ROOT` to
+    a) Set the environment variable [`PAPERLESS_MEDIA_ROOT`](configuration.md#PAPERLESS_MEDIA_ROOT) to
     `/data/media`
 7.  Update timezone
-    a) Set the environment variable `PAPERLESS_TIME_ZONE` to the same
+    a) Set the environment variable [`PAPERLESS_TIME_ZONE`](configuration.md#PAPERLESS_TIME_ZONE) to the same
     value as `TZ`
 8.  Modify the `image:` to point to
     `ghcr.io/paperless-ngx/paperless-ngx:latest` or a specific version
     if preferred.
-9.  Start the containers as before, using `docker-compose`.
+9.  Start the containers as before, using `docker compose`.
 
 ## Moving data from SQLite to PostgreSQL or MySQL/MariaDB {#sqlite_to_psql}
 
@@ -712,7 +709,7 @@ below use PostgreSQL, but are applicable to MySQL/MariaDB with the
 !!! warning
 
     MySQL is case insensitive by default, treating values like "Name" and
-    "NAME" as identical. See [MySQL caveats](/advanced_usage#mysql-caveats) for details.
+    "NAME" as identical. See [MySQL caveats](advanced_usage.md#mysql-caveats) for details.
 
 !!! warning
 
@@ -733,7 +730,7 @@ below use PostgreSQL, but are applicable to MySQL/MariaDB with the
     file to `docker-compose.yml`. Remember to adjust the consumption
     directory, if necessary.
     b) Without docker, configure the database in your `paperless.conf`
-    file. See [configuration](/configuration) for
+    file. See [configuration](configuration.md) for
     details.
 
 3.  Open a shell and initialize the database:
@@ -743,7 +740,7 @@ below use PostgreSQL, but are applicable to MySQL/MariaDB with the
 
         ``` shell-session
         $ cd /path/to/paperless
-        $ docker-compose run --rm webserver /bin/bash
+        $ docker compose run --rm webserver /bin/bash
         ```
 
         This will launch the container and initialize the PostgreSQL
@@ -796,7 +793,7 @@ Execute this:
 
 ```shell-session
 $ cd /path/to/paperless
-$ docker-compose run --rm webserver migrate documents 0023
+$ docker compose run --rm webserver migrate documents 0023
 ```
 
 Or without docker:
@@ -817,36 +814,36 @@ the Pi and configuring some options in paperless can help improve
 performance immensely:
 
 - Stick with SQLite to save some resources.
-- Consider setting `PAPERLESS_OCR_PAGES` to 1, so that paperless will
+- Consider setting [`PAPERLESS_OCR_PAGES`](configuration.md#PAPERLESS_OCR_PAGES) to 1, so that paperless will
   only OCR the first page of your documents. In most cases, this page
   contains enough information to be able to find it.
-- `PAPERLESS_TASK_WORKERS` and `PAPERLESS_THREADS_PER_WORKER` are
+- [`PAPERLESS_TASK_WORKERS`](configuration.md#PAPERLESS_TASK_WORKERS) and [`PAPERLESS_THREADS_PER_WORKER`](configuration.md#PAPERLESS_THREADS_PER_WORKER) are
   configured to use all cores. The Raspberry Pi models 3 and up have 4
   cores, meaning that paperless will use 2 workers and 2 threads per
   worker. This may result in sluggish response times during
   consumption, so you might want to lower these settings (example: 2
   workers and 1 thread to always have some computing power left for
   other tasks).
-- Keep `PAPERLESS_OCR_MODE` at its default value `skip` and consider
+- Keep [`PAPERLESS_OCR_MODE`](configuration.md#PAPERLESS_OCR_MODE) at its default value `skip` and consider
   OCR'ing your documents before feeding them into paperless. Some
   scanners are able to do this!
-- Set `PAPERLESS_OCR_SKIP_ARCHIVE_FILE` to `with_text` to skip archive
+- Set [`PAPERLESS_OCR_SKIP_ARCHIVE_FILE`](configuration.md#PAPERLESS_OCR_SKIP_ARCHIVE_FILE) to `with_text` to skip archive
   file generation for already ocr'ed documents, or `always` to skip it
   for all documents.
 - If you want to perform OCR on the device, consider using
   `PAPERLESS_OCR_CLEAN=none`. This will speed up OCR times and use
   less memory at the expense of slightly worse OCR results.
-- If using docker, consider setting `PAPERLESS_WEBSERVER_WORKERS` to 1. This will save some memory.
-- Consider setting `PAPERLESS_ENABLE_NLTK` to false, to disable the
+- If using docker, consider setting [`PAPERLESS_WEBSERVER_WORKERS`](configuration.md#PAPERLESS_WEBSERVER_WORKERS) to 1. This will save some memory.
+- Consider setting [`PAPERLESS_ENABLE_NLTK`](configuration.md#PAPERLESS_ENABLE_NLTK) to false, to disable the
   more advanced language processing, which can take more memory and
   processing time.
 
-For details, refer to [configuration](/configuration).
+For details, refer to [configuration](configuration.md).
 
 !!! note
 
     Updating the
-    [automatic matching algorithm](/advanced_usage#automatic-matching) takes quite a bit of time. However, the update mechanism
+    [automatic matching algorithm](advanced_usage.md#automatic-matching) takes quite a bit of time. However, the update mechanism
     checks if your data has changed before doing the heavy lifting. If you
     experience the algorithm taking too much cpu time, consider changing the
     schedule in the admin interface to daily. You can also manually invoke
@@ -857,45 +854,8 @@ For details, refer to [configuration](/configuration).
 
 # Using nginx as a reverse proxy {#nginx}
 
-If you want to expose paperless to the internet, you should hide it
-behind a reverse proxy with SSL enabled.
+Please see [the wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Using-a-Reverse-Proxy-with-Paperless-ngx#nginx) for user-maintained documentation of using nginx with Paperless-ngx.
 
-In addition to the usual configuration for SSL, the following
-configuration is required for paperless to operate:
+# Enhancing security {#security}
 
-```nginx
-http {
-
-    # Adjust as required. This is the maximum size for file uploads.
-    # The default value 1M might be a little too small.
-    client_max_body_size 10M;
-
-    server {
-
-        location / {
-
-            # Adjust host and port as required.
-            proxy_pass http://localhost:8000/;
-
-            # These configuration options are required for WebSockets to work.
-            proxy_http_version 1.1;
-            proxy_set_header Upgrade $http_upgrade;
-            proxy_set_header Connection "upgrade";
-
-            proxy_redirect off;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Host $server_name;
-            add_header P3P 'CP=""'; # may not be required in all setups
-        }
-    }
-}
-```
-
-The `PAPERLESS_URL` configuration variable is also required when using a
-reverse proxy. Please refer to the [hosting and security](/configuration#hosting-and-security) docs.
-
-Also read
-[this](https://channels.readthedocs.io/en/stable/deploying.html#nginx-supervisor-ubuntu),
-towards the end of the section.
+Please see [the wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Using-Security-Tools-with-Paperless-ngx) for user-maintained documentation of how to configure security tools like Fail2ban with Paperless-ngx.

docs/troubleshooting.md

@@ -6,7 +6,7 @@ Check for the following issues:
 
 - Ensure that the directory you're putting your documents in is the
   folder paperless is watching. With docker, this setting is performed
-  in the `docker-compose.yml` file. Without docker, look at the
+  in the `docker-compose.yml` file. Without Docker, look at the
   `CONSUMPTION_DIR` setting. Don't adjust this setting if you're
   using docker.
 
@@ -46,8 +46,7 @@ run:
 If you notice that the consumer will only pickup files in the
 consumption directory at startup, but won't find any other files added
 later, you will need to enable filesystem polling with the configuration
-option `PAPERLESS_CONSUMER_POLLING`, see
-`[here](/configuration#polling).
+option [`PAPERLESS_CONSUMER_POLLING`](configuration.md#PAPERLESS_CONSUMER_POLLING).
 
 This will disable listening to filesystem changes with inotify and
 paperless will manually check the consumption directory for changes
@@ -121,7 +120,7 @@ Gotenberg raises this error.
 
 You can increase the timeout by configuring a command flag for Gotenberg
 (see also [here](https://gotenberg.dev/docs/modules/api#properties)). If
-using docker-compose, this is achieved by the following configuration
+using Docker Compose, this is achieved by the following configuration
 change in the `docker-compose.yml` file:
 
 ```yaml
@@ -145,7 +144,7 @@ The following error occured while consuming document.pdf: [Errno 13] Permission
 This happens when paperless does not have permission to delete files
 inside the consumption directory. Ensure that `USERMAP_UID` and
 `USERMAP_GID` are set to the user id and group id you use on the host
-operating system, if these are different from `1000`. See [Docker setup](/setup#docker_hub).
+operating system, if these are different from `1000`. See [Docker setup](setup.md#docker_hub).
 
 Also ensure that you are able to read and write to the consumption
 directory on the host.
@@ -265,8 +264,8 @@ This probably indicates paperless tried to consume the same file twice.
 This can happen for a number of reasons, depending on how documents are
 placed into the consume folder. If paperless is using inotify (the
 default) to check for documents, try adjusting the
-[inotify configuration](/configuration#inotify). If polling is enabled, try adjusting the
-[polling configuration](/configuration#polling).
+[inotify configuration](configuration.md#inotify). If polling is enabled, try adjusting the
+[polling configuration](configuration.md#polling).
 
 ## Consumer fails waiting for file to remain unmodified.
 
@@ -278,7 +277,7 @@ You might find messages like these in your log files:
 
 This indicates paperless timed out while waiting for the file to be
 completely written to the consume folder. Adjusting
-[polling configuration](/configuration#polling) values should resolve the issue.
+[polling configuration](configuration.md#polling) values should resolve the issue.
 
 !!! note
 
@@ -297,8 +296,8 @@ This indicates paperless was unable to open the file, as the OS reported
 the file as still being in use. To prevent a crash, paperless did not
 try to consume the file. If paperless is using inotify (the default) to
 check for documents, try adjusting the
-[inotify configuration](/configuration#inotify). If polling is enabled, try adjusting the
-[polling configuration](/configuration#polling).
+[inotify configuration](configuration.md#inotify). If polling is enabled, try adjusting the
+[polling configuration](configuration.md#polling).
 
 !!! note
 
@@ -320,7 +319,7 @@ many workers attempting to access the database simultaneously.
 
 Consider changing to the PostgreSQL database if you will be processing
 many documents at once often. Otherwise, try tweaking the
-`PAPERLESS_DB_TIMEOUT` setting to allow more time for the database to
+[`PAPERLESS_DB_TIMEOUT`](configuration.md#PAPERLESS_DB_TIMEOUT) setting to allow more time for the database to
 unlock. This may have minor performance implications.
 
 ## gunicorn fails to start with "is not a valid port number"
@@ -330,7 +329,7 @@ environment variable named `${serviceName}_PORT`. This is
 the same environment variable which is used by Paperless to optionally
 change the port gunicorn listens on.
 
-To fix this, set `PAPERLESS_PORT` again to your desired port, or the
+To fix this, set [`PAPERLESS_PORT`](configuration.md#PAPERLESS_PORT) again to your desired port, or the
 default of 8000.
 
 ## Database Warns about unique constraint "documents_tag_name_uniq
@@ -345,3 +344,15 @@ STATEMENT:  INSERT INTO "documents_tag" ("owner_id", "name", "match", "matching_
 
 This can happen during heavy consumption when using polling. Paperless will handle it correctly and the file
 will still be consumed
+
+## Consumption fails with "Ghostscript PDF/A rendering failed"
+
+Newer versions of OCRmyPDF will fail if it encounters errors during processing.
+This is intentional as the output archive file may differ in unexpected or undesired
+ways from the original. As the logs indicate, if you encounter this error you can set
+`PAPERLESS_OCR_USER_ARGS: '{"continue_on_soft_render_error": true}'` to try to 'force'
+processing documents with this issue.
+
+## Platform-Specific Deployment Troubleshooting
+
+A user-maintained wiki page is available to help troubleshoot issues that may arise when trying to deploy Paperless-ngx on specific platforms, for example SELinux. Please see [the wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Platform%E2%80%90Specific-Troubleshooting).

docs/usage.md

@@ -62,14 +62,16 @@ following operations on your documents:
     paperless to create archived versions for digital documents, you can
     configure that by configuring
     `PAPERLESS_OCR_SKIP_ARCHIVE_FILE=with_text`. Please read the
-    [relevant section in the documentation](/configuration#ocr).
+    [relevant section in the documentation](configuration.md#ocr).
 
 !!! note
 
     No matter which options you choose, Paperless will always store the
     original document that it found in the consumption directory or in the
     mail and will never overwrite that document. Archived versions are
-    stored alongside the original versions.
+    stored alongside the original versions. Any files found in the
+    consumption directory will stored inside the Paperless-ngx file
+    structure and will not be retained in the consumption directory.
 
 ### The consumption directory
 
@@ -77,7 +79,9 @@ The primary method of getting documents into your database is by putting
 them in the consumption directory. The consumer waits patiently, looking
 for new additions to this directory. When it finds them,
 the consumer goes about the process of parsing them with the OCR,
-indexing what it finds, and storing it in the media directory.
+indexing what it finds, and storing it in the media directory. You should
+think of this folder as a temporary location, as files will be re-created
+inside Paperless-ngx and removed from the consumption folder.
 
 Getting stuff into this directory is up to you. If you're running
 Paperless on your local computer, you might just want to drag and drop
@@ -88,27 +92,25 @@ Typically, you're looking at an FTP server like
 [Proftpd](http://www.proftpd.org/) or a Windows folder share with
 [Samba](https://www.samba.org/).
 
+!!! warning
+
+    Files found in the consumption directory that are consumed will be
+    removed from the consumption directory and stored inside the
+    Paperless-ngx file structure using any settings / storage paths
+    you have specified. This action is performed as safely as possible
+    but this means it is expected that files in the consumption
+    directory will no longer exist (there) after being consumed.
+
 ### Web UI Upload
 
-The dashboard has a file drop field to upload documents to paperless.
-Simply drag a file onto this field or select a file with the file
-dialog. Multiple files are supported.
-
-You can also upload documents on any other page of the web UI by
-dragging-and-dropping files into your browser window.
+The dashboard has a button to upload documents to paperless or you
+can simply drag a file anywhere into the app to initiate the consumption
+process.
 
 ### Mobile upload {#usage-mobile_upload}
 
-The mobile app over at [https://github.com/qcasey/paperless_share](https://github.com/qcasey/paperless_share)
-allows Android users to share any documents with paperless. This can be
-combined with any of the mobile scanning apps out there, such as Office
-Lens.
-
-Furthermore, there is the [Paperless
-App](https://github.com/bauerj/paperless_app) as well, which not only
-has document upload, but also document browsing and download features.
-
-Another option is [Paperless Mobile](https://github.com/astubenbord/paperless-mobile), an Android app that supports document upload, scanning, management of labels and more.
+Please see [the wiki](https://github.com/paperless-ngx/paperless-ngx/wiki/Affiliated-Projects) for a user-maintained list of affiliated projects and
+software (e.g. for mobile devices) that is compatible with Paperless-ngx.
 
 ### IMAP (Email) {#usage-email}
 
@@ -132,9 +134,9 @@ These rules perform the following:
 5.  If documents were consumed from a mail, the rule action is performed
     on that mail.
 
-Paperless will completely ignore mails that do not match your filters.
-It will also only perform the action on mails that it has consumed
-documents from.
+Paperless will check all emails only once and completely ignore messages
+that do not match your filters. It will also only perform the rule action
+on e-mails that it has consumed documents from.
 
 The actions all ensure that the same mail is not consumed twice by
 different means. These are as follows:
@@ -195,46 +197,167 @@ different means. These are as follows:
     them further.
 
 Paperless is set up to check your mails every 10 minutes. This can be
-configured via `PAPERLESS_EMAIL_TASK_CRON` (see [software tweaks](/configuration#software_tweaks))
+configured via [`PAPERLESS_EMAIL_TASK_CRON`](configuration.md#PAPERLESS_EMAIL_TASK_CRON)
 
 ### REST API
 
-You can also submit a document using the REST API, see [POSTing documents](/api#file-uploads)
+You can also submit a document using the REST API, see [POSTing documents](api.md#file-uploads)
 for details.
 
 ## Permissions
 
-As of version 1.13.0 Paperless-ngx added core support for user / group permissions. Permissions is
-based around an object 'owner' and 'view' and 'edit' permissions can be granted to other users
-or groups.
+As of version 1.14.0 Paperless-ngx added core support for user / group permissions. Permissions is
+based around 'global' permissions as well as 'object-level' permissions. Global permissions designate
+which parts of the application a user can access (e.g. Documents, Tags, Settings) and object-level
+determine which objects are visible or editable. All objects have an 'owner' and 'view' and 'edit'
+permissions which can be granted to other users or groups. The paperless-ngx permissions system uses
+the built-in user model of the backend framework, Django.
 
-Permissions uses the built-in user model of the backend framework, Django.
+!!! tip
 
-!!! note
-
-    After migration to version 1.13.0 all existing documents, tags etc. will have no explicit owner
-    set which means they will be visible / editable by all users. Once an object has an owner set,
-    only the owner can explicitly grant / revoke permissions.
-
-!!! note
-
-    When first migrating to permissions it is recommended to user a 'superuser' account (which
-    would usually have been setup during installation) to ensure you have full permissions.
-
-    Note that superusers have access to all objects.
+    Object-level permissions only apply to the object itself. In other words, setting permissions
+    for a Tag will _not_ affect the permissions of documents that have the Tag.
 
 Permissions can be set using the new "Permissions" tab when editing documents, or bulk-applied
 in the UI by selecting documents and choosing the "Permissions" button. Owner can also optionally
 be set for documents uploaded via the API. Documents consumed via the consumption dir currently
 do not have an owner set.
 
+!!! note
+
+    After migration to version 1.14.0 all existing documents, tags etc. will have no explicit owner
+    set which means they will be visible / editable by all users. Once an object has an owner set,
+    only the owner can explicitly grant / revoke permissions.
+
+!!! note
+
+    When first migrating to permissions it is recommended to use a 'superuser' account (which
+    would usually have been setup during installation) to ensure you have full permissions.
+
+    Note that superusers have access to all objects.
+
+### Default permissions
+
+Default permissions for documents can be set using consumption templates.
+
+For objects created via the web UI (tags, doc types, etc.) the default is to set the current user
+as owner and no extra permissions, but you explicitly set these under Settings > Permissions.
+
 ### Users and Groups
 
-Paperless-ngx versions after 1.13.0 allow creating and editing users and groups via the 'frontend' UI.
+Paperless-ngx versions after 1.14.0 allow creating and editing users and groups via the 'frontend' UI.
 These can be found under Settings > Users & Groups, assuming the user has access. If a user is designated
 as a member of a group those permissions will be inherited and this is reflected in the UI. Explicit
 permissions can be granted to limit access to certain parts of the UI (and corresponding API endpoints).
 
+### Password reset
+
+In order to enable the password reset feature you will need to setup an SMTP backend, see
+[`PAPERLESS_EMAIL_HOST`](configuration.md#PAPERLESS_EMAIL_HOST)
+
+## Consumption templates
+
+Consumption templates were introduced in v2.0 and allow for finer control over what metadata (tags, doc
+types) and permissions (owner, privileges) are assigned to documents during consumption. In general,
+templates are applied sequentially (by sort order) but subsequent templates will never override an
+assignment from a preceding template. The same is true for mail rules, e.g. if you set the correspondent
+in a mail rule any subsequent consumption templates that are applied _will not_ overwrite this. The
+exception to this is assignments that can be multiple e.g. tags and permissions, which will be merged.
+
+Consumption templates allow you to filter by:
+
+- Source, e.g. documents uploaded via consume folder, API (& the web UI) and mail fetch
+- File name, including wildcards e.g. \*.pdf will apply to all pdfs
+- File path, including wildcards. Note that enabling `PAPERLESS_CONSUMER_RECURSIVE` would allow, for
+  example, automatically assigning documents to different owners based on the upload directory.
+- Mail rule. Choosing this option will force 'mail fetch' to be the template source.
+
+!!! note
+
+    You must include a file name filter, a path filter or a mail rule filter. Use * for either to apply
+    to all files.
+
+Consumption templates can assign:
+
+- Title, see [title placeholders](usage.md#title-placeholders) below
+- Tags, correspondent, document types
+- Document owner
+- View and / or edit permissions to users or groups
+- Custom fields. Note that no value for the field will be set
+
+### Consumption template permissions
+
+All users who have application permissions for editing consumption templates can see the same set
+of templates. In other words, templates themselves intentionally do not have an owner or permissions.
+
+Given their potentially far-reaching capabilities, you may want to restrict access to templates.
+
+Upon migration, existing installs will grant access to consumption templates to users who can add
+documents (and superusers who can always access all parts of the app).
+
+### Title placeholders
+
+Consumption template titles can include placeholders, _only for items that are assigned within the template_.
+This is because at the time of consumption (when the title is to be set), no automatic tags etc. have been
+applied. You can use the following placeholders:
+
+- `{correspondent}`: assigned correspondent name
+- `{document_type}`: assigned document type name
+- `{owner_username}`: assigned owner username
+- `{added}`: added datetime
+- `{added_year}`: added year
+- `{added_year_short}`: added year
+- `{added_month}`: added month
+- `{added_month_name}`: added month name
+- `{added_month_name_short}`: added month short name
+- `{added_day}`: added day
+
+## Custom Fields {#custom-fields}
+
+Paperless-ngx supports the use of custom fields for documents as of v2.0, allowing a user
+to optionally attach data to documents which does not fit in the existing set of fields
+Paperless-ngx provides.
+
+1. First, create a custom field (under "Manage"), with a given name and data type. This could be something like "Invoice Number" or "Date Paid", with a data type of "Number", "Date", "String", etc.
+2. Once created, a field can be used with documents and data stored. To do so, use the "Custom Fields" menu on the document detail page, choose your existing field and click "Add". Once the field is visible in the form you can enter the appropriate
+   data which will be validated according to the custom field "data type".
+3. Fields can be removed by hovering over the field name revealing a "Remove" button.
+
+!!! important
+
+    Added / removed fields, as well as any data is not saved to the document until you
+    actually hit the "Save" button, similar to other changes on the document details page.
+
+!!! note
+
+    Once the data type for a field is set, it cannot be changed.
+
+Multiple fields may be attached to a document but the same field name cannot be assigned multiple times to the a single document.
+
+The following custom field types are supported:
+
+- `Text`: any text
+- `Boolean`: true / false (check / unchecked) field
+- `Date`: date
+- `URL`: a valid url
+- `Integer`: integer number e.g. 12
+- `Number`: float number e.g. 12.3456
+- `Monetary`: float number with exactly two decimals, e.g. 12.30
+- `Document Link`: reference(s) to other document(s), displayed as links
+
+## Share Links
+
+Paperless-ngx added the abiltiy to create shareable links to files in version 2.0. You can find the button for this on the document detail screen.
+
+- Share links do not require a user to login and thus link directly to a file.
+- Links are unique and are of the form `{paperless-url}/share/{randomly-generated-slug}`.
+- Links can optionally have an expiration time set.
+- After a link expires or is deleted users will be redirected to the regular paperless-ngx login.
+
+!!! tip
+
+    If your paperless-ngx instance is behind a reverse-proxy you may want to create an exception to bypass any authentication layers that are part of your setup in order to make links truly publicly-accessible. Of course, do so with caution.
+
 ## Best practices {#basic-searching}
 
 Paperless offers a couple tools that help you organize your document
@@ -443,7 +566,7 @@ Once you have scanned in a document, proceed in paperless as follows.
     paperless will assign them automatically. After consuming a couple
     documents, you can even ask paperless to *learn* when to assign tags and
     correspondents by itself. For details on this feature, see
-    [advanced matching](/advanced_usage#matching).
+    [advanced matching](advanced_usage.md#matching).
 
 ### Task management
 

install-paperless-ngx.sh

@@ -38,7 +38,6 @@ ask_docker_folder() {
 			echo "Invalid folder: $result"
 		fi
 
-
 	done
 }
 
@@ -57,14 +56,9 @@ if ! command -v docker &> /dev/null ; then
 	exit 1
 fi
 
-DOCKER_COMPOSE_CMD="docker-compose"
-if ! command -v ${DOCKER_COMPOSE_CMD} ; then
-	if docker compose version &> /dev/null ; then
-		DOCKER_COMPOSE_CMD="docker compose"
-	else
-		echo "docker-compose executable not found. Is docker-compose installed?"
-		exit 1
-	fi
+if ! command -v docker compose &> /dev/null ; then
+	echo "docker compose executable not found. Is docker compose installed?"
+	exit 1
 fi
 
 # Check if user has permissions to run Docker by trying to get the status of Docker (docker status).
@@ -72,7 +66,7 @@ fi
 if ! docker stats --no-stream &> /dev/null ; then
 	echo ""
 	echo "WARN: It look like the current user does not have Docker permissions."
-	echo "WARN: Use 'sudo usermod -aG docker $USER' to assign Docker permissions to the user."
+	echo "WARN: Use 'sudo usermod -aG docker $USER' to assign Docker permissions to the user (may require restarting shell)."
 	echo ""
 	sleep 3
 fi
@@ -321,7 +315,8 @@ fi
 wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/docker-compose.$DOCKER_COMPOSE_VERSION.yml" -O docker-compose.yml
 wget "https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/main/docker/compose/.env" -O .env
 
-SECRET_KEY=$(tr --delete --complement 'a-zA-Z0-9' < /dev/urandom 2>/dev/null | head --bytes 64)
+SECRET_KEY=$(LC_ALL=C tr -dc 'a-zA-Z0-9!"#$%&'\''()*+,-./:;<=>?@[\]^_`{|}~' < /dev/urandom | head --bytes 64)
+
 
 DEFAULT_LANGUAGES=("deu eng fra ita spa")
 
@@ -382,8 +377,16 @@ if [ "$l1" -eq "$l2" ] ; then
 fi
 
 
-${DOCKER_COMPOSE_CMD} pull
+docker compose pull
 
-${DOCKER_COMPOSE_CMD} run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
+if [ "$DATABASE_BACKEND" == "postgres" ] || [ "$DATABASE_BACKEND" == "mariadb" ] ; then
+	echo "Starting DB first for initilzation"
+	docker compose up --detach db
+	# hopefully enough time for even the slower systems
+	sleep 15
+	docker compose stop
+fi
 
-${DOCKER_COMPOSE_CMD} up --detach
+docker compose run --rm -e DJANGO_SUPERUSER_PASSWORD="$PASSWORD" webserver createsuperuser --noinput --username "$USERNAME" --email "$EMAIL"
+
+docker compose up --detach

mkdocs.yml

@@ -28,6 +28,7 @@ theme:
     repo: fontawesome/brands/github
   favicon: assets/favicon.png
 repo_url: https://github.com/paperless-ngx/paperless-ngx
+repo_name: paperless-ngx/paperless-ngx
 edit_uri: blob/main/docs/
 extra_css:
   - assets/extra.css
@@ -42,6 +43,7 @@ markdown_extensions:
   - pymdownx.superfences
   - pymdownx.inlinehilite
   - pymdownx.snippets
+  - footnotes
 strict: true
 nav:
     - index.md
@@ -64,3 +66,6 @@ extra:
       link: https://hub.docker.com/r/paperlessngx/paperless-ngx
     - icon: material/chat
       link: https://matrix.to/#/#paperless:matrix.org
+plugins:
+  - search
+  - glightbox

paperless.conf.example

@@ -66,6 +66,11 @@
 #PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false
 #PAPERLESS_CONSUMER_ENABLE_BARCODES=false
 #PAPERLESS_CONSUMER_BARCODE_STRING=PATCHT
+#PAPERLESS_CONSUMER_BARCODE_UPSCALE=0.0
+#PAPERLESS_CONSUMER_BARCODE_DPI=300
+#PAPERLESS_CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED=false
+#PAPERLESS_CONSUMER_COLLATE_DOUBLE_SIDED_SUBDIR_NAME=double-sided
+#PAPERLESS_CONSUMER_COLLATE_DOUBLE_SIDED_TIFF_SUPPORT=false
 #PAPERLESS_PRE_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh
 #PAPERLESS_FILENAME_DATE_ORDER=YMD

scripts/start_services.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:13
+docker run -p 5432:5432 -e POSTGRES_PASSWORD=password -v paperless_pgdata:/var/lib/postgresql/data -d postgres:15
 docker run -d -p 6379:6379 redis:latest
 docker run -p 3000:3000 -d gotenberg/gotenberg:7.8 gotenberg --chromium-disable-javascript=true --chromium-allow-list="file:///tmp/.*"
 docker run -p 9998:9998 -d ghcr.io/paperless-ngx/tika:latest

src-ui/.eslintrc.json

@@ -1,7 +1,8 @@
 {
   "root": true,
   "ignorePatterns": [
-    "projects/**/*"
+    "projects/**/*",
+    "/src/app/components/common/pdf-viewer/**"
   ],
   "overrides": [
     {
@@ -24,7 +25,7 @@
           "error",
           {
             "type": "attribute",
-            "prefix": "app",
+            "prefix": "pngx",
             "style": "camelCase"
           }
         ],
@@ -32,7 +33,7 @@
           "error",
           {
             "type": "element",
-            "prefix": "app",
+            "prefix": "pngx",
             "style": "kebab-case"
           }
         ]

src-ui/.gitignore

@@ -49,3 +49,6 @@ Thumbs.db
 # Cypress
 cypress/videos/**/*
 cypress/screenshots/**/*
+/test-results/
+/playwright-report/
+/playwright/.cache/

src-ui/angular.json

@@ -12,31 +12,39 @@
       },
       "root": "",
       "sourceRoot": "src",
-      "prefix": "app",
+      "prefix": "pngx",
       "i18n": {
         "sourceLocale": "en-US",
         "locales": {
           "ar-AR": "src/locale/messages.ar_AR.xlf",
+          "af-ZA": "src/locale/messages.af_ZA.xlf",
+          "bg-BG": "src/locale/messages.bg_BG.xlf",
           "be-BY": "src/locale/messages.be_BY.xlf",
           "ca-ES": "src/locale/messages.ca_ES.xlf",
           "cs-CZ": "src/locale/messages.cs_CZ.xlf",
           "da-DK": "src/locale/messages.da_DK.xlf",
           "de-DE": "src/locale/messages.de_DE.xlf",
+          "el-GR": "src/locale/messages.el_GR.xlf",
           "en-GB": "src/locale/messages.en_GB.xlf",
           "es-ES": "src/locale/messages.es_ES.xlf",
+          "fi-FI": "src/locale/messages.fi_FI.xlf",
           "fr-FR": "src/locale/messages.fr_FR.xlf",
+          "hu-HU": "src/locale/messages.hu_HU.xlf",
           "it-IT": "src/locale/messages.it_IT.xlf",
           "lb-LU": "src/locale/messages.lb_LU.xlf",
           "nl-NL": "src/locale/messages.nl_NL.xlf",
+          "no-NO": "src/locale/messages.no_NO.xlf",
           "pl-PL": "src/locale/messages.pl_PL.xlf",
           "pt-BR": "src/locale/messages.pt_BR.xlf",
           "pt-PT": "src/locale/messages.pt_PT.xlf",
           "ro-RO": "src/locale/messages.ro_RO.xlf",
           "ru-RU": "src/locale/messages.ru_RU.xlf",
+          "sk-SK": "src/locale/messages.sk_SK.xlf",
           "sl-SI": "src/locale/messages.sl_SI.xlf",
           "sr-CS": "src/locale/messages.sr_CS.xlf",
           "sv-SE": "src/locale/messages.sv_SE.xlf",
           "tr-TR": "src/locale/messages.tr_TR.xlf",
+          "uk-UA": "src/locale/messages.uk_UA.xlf",
           "zh-CN": "src/locale/messages.zh_CN.xlf"
         }
       },
@@ -57,7 +65,7 @@
               "src/assets",
               "src/manifest.webmanifest",
               {
-                "glob": "pdf.worker.min.js",
+                "glob": "{pdf.worker.min.js,pdf.min.js}",
                 "input": "node_modules/pdfjs-dist/build/",
                 "output": "/assets/js/"
               }
@@ -67,7 +75,8 @@
             ],
             "scripts": [],
             "allowedCommonJsDependencies": [
-              "ng2-pdf-viewer"
+              "pdfjs-dist",
+              "pdfjs-dist/web/pdf_viewer"
             ],
             "vendorChunk": true,
             "extractLicenses": false,
@@ -101,7 +110,7 @@
                 {
                   "type": "anyComponentStyle",
                   "maximumWarning": "6kb",
-                  "maximumError": "10kb"
+                  "maximumError": "30kb"
                 }
               ]
             },
@@ -146,37 +155,6 @@
             "scripts": []
           }
         },
-        "e2e": {
-          "builder": "@cypress/schematic:cypress",
-          "options": {
-            "devServerTarget": "paperless-ui:serve",
-            "watch": true,
-            "headless": false
-          },
-          "configurations": {
-            "production": {
-              "devServerTarget": "paperless-ui:serve:production"
-            }
-          }
-        },
-        "cypress-run": {
-          "builder": "@cypress/schematic:cypress",
-          "options": {
-            "devServerTarget": "paperless-ui:serve"
-          },
-          "configurations": {
-            "production": {
-              "devServerTarget": "paperless-ui:serve:production"
-            }
-          }
-        },
-        "cypress-open": {
-          "builder": "@cypress/schematic:cypress",
-          "options": {
-            "watch": true,
-            "headless": false
-          }
-        },
         "lint": {
           "builder": "@angular-eslint/builder:lint",
           "options": {
@@ -189,7 +167,6 @@
       }
     }
   },
-  "defaultProject": "paperless-ui",
   "cli": {
     "schematicCollections": [
       "@angular-eslint/schematics"

src-ui/cypress.config.ts

@@ -1,14 +0,0 @@
-import { defineConfig } from 'cypress'
-
-export default defineConfig({
-  videosFolder: 'cypress/videos',
-  video: false,
-  screenshotsFolder: 'cypress/screenshots',
-  fixturesFolder: 'cypress/fixtures',
-  e2e: {
-    setupNodeEvents(on, config) {
-      return require('./cypress/plugins/index.ts')(on, config)
-    },
-    baseUrl: 'http://localhost:4200',
-  },
-})

src-ui/cypress/e2e/auth/auth.cy.ts

@@ -1,68 +0,0 @@
-describe('settings', () => {
-  beforeEach(() => {
-    // also uses global fixtures from cypress/support/e2e.ts
-
-    // mock restricted permissions
-    cy.intercept('http://localhost:8000/api/ui_settings/', {
-      fixture: 'ui_settings/settings_restricted.json',
-    })
-  })
-
-  it('should not allow user to edit settings', () => {
-    cy.visit('/dashboard')
-    cy.contains('Settings').should('not.exist')
-    cy.visit('/settings').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-
-  it('should not allow user to view documents', () => {
-    cy.visit('/dashboard')
-    cy.contains('Documents').should('not.exist')
-    cy.visit('/documents').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-    cy.visit('/documents/1').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-
-  it('should not allow user to view correspondents', () => {
-    cy.visit('/dashboard')
-    cy.contains('Correspondents').should('not.exist')
-    cy.visit('/correspondents').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-
-  it('should not allow user to view tags', () => {
-    cy.visit('/dashboard')
-    cy.contains('Tags').should('not.exist')
-    cy.visit('/tags').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-
-  it('should not allow user to view document types', () => {
-    cy.visit('/dashboard')
-    cy.contains('Document Types').should('not.exist')
-    cy.visit('/documenttypes').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-
-  it('should not allow user to view storage paths', () => {
-    cy.visit('/dashboard')
-    cy.contains('Storage Paths').should('not.exist')
-    cy.visit('/storagepaths').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-
-  it('should not allow user to view logs', () => {
-    cy.visit('/dashboard')
-    cy.contains('Logs').should('not.exist')
-    cy.visit('/logs').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-
-  it('should not allow user to view tasks', () => {
-    cy.visit('/dashboard')
-    cy.contains('Tasks').should('not.exist')
-    cy.visit('/tasks').wait(2000)
-    cy.contains("You don't have permissions to do that").should('exist')
-  })
-})

src-ui/cypress/e2e/documents/document-detail.cy.ts

@@ -1,114 +0,0 @@
-describe('document-detail', () => {
-  beforeEach(() => {
-    // also uses global fixtures from cypress/support/e2e.ts
-
-    this.modifiedDocuments = []
-
-    cy.fixture('documents/documents.json').then((documentsJson) => {
-      cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => {
-        let response = { ...documentsJson }
-        response = response.results.find((d) => d.id == 1)
-        req.reply(response)
-      })
-    })
-
-    cy.intercept('PUT', 'http://localhost:8000/api/documents/1/', (req) => {
-      this.modifiedDocuments.push(req.body) // store this for later
-      req.reply({ result: 'OK' })
-    }).as('saveDoc')
-
-    cy.fixture('documents/1/notes.json').then((notesJson) => {
-      cy.intercept(
-        'GET',
-        'http://localhost:8000/api/documents/1/notes/',
-        (req) => {
-          req.reply(notesJson.filter((c) => c.id != 10)) // 3
-        }
-      )
-
-      cy.intercept(
-        'DELETE',
-        'http://localhost:8000/api/documents/1/notes/?id=9',
-        (req) => {
-          req.reply(notesJson.filter((c) => c.id != 9 && c.id != 10)) // 2
-        }
-      )
-
-      cy.intercept(
-        'POST',
-        'http://localhost:8000/api/documents/1/notes/',
-        (req) => {
-          req.reply(notesJson) // 4
-        }
-      )
-    })
-
-    cy.viewport(1024, 1024)
-    cy.visit('/documents/1/').wait('@ui-settings')
-  })
-
-  it('should activate / deactivate save button when changes are saved', () => {
-    cy.contains('button', 'Save').should('be.disabled')
-    cy.get('app-input-text[formcontrolname="title"]')
-      .type(' additional')
-      .wait(1500) // this delay is for frontend debounce
-    cy.contains('button', 'Save').should('not.be.disabled')
-  })
-
-  it('should warn on unsaved changes', () => {
-    cy.get('app-input-text[formcontrolname="title"]')
-      .type(' additional')
-      .wait(1500) // this delay is for frontend debounce
-    cy.get('button[title="Close"]').click()
-    cy.contains('You have unsaved changes')
-    cy.contains('button', 'Cancel').click().wait(150)
-    cy.contains('button', 'Save').click().wait('@saveDoc').wait(2000) // navigates away after saving
-    cy.contains('You have unsaved changes').should('not.exist')
-  })
-
-  it('should show a mobile preview', () => {
-    cy.viewport(440, 1000)
-    cy.get('a')
-      .contains('Preview')
-      .scrollIntoView({ offset: { top: 150, left: 0 } })
-      .click()
-    cy.get('pdf-viewer').should('be.visible')
-  })
-
-  it('should show a list of notes', () => {
-    cy.wait(1000).get('a').contains('Notes').click({ force: true }).wait(1000)
-    cy.get('app-document-notes').find('.card').its('length').should('eq', 3)
-  })
-
-  it('should support note deletion', () => {
-    cy.wait(1000).get('a').contains('Notes').click().wait(1000)
-    cy.get('app-document-notes')
-      .find('.card')
-      .first()
-      .find('button')
-      .click({ force: true })
-      .wait(500)
-    cy.get('app-document-notes').find('.card').its('length').should('eq', 2)
-  })
-
-  it('should support note insertion', () => {
-    cy.wait(1000).get('a').contains('Notes').click().wait(1000)
-    cy.get('app-document-notes')
-      .find('form textarea')
-      .type('Testing new note')
-      .wait(500)
-    cy.get('app-document-notes').find('form button').click().wait(1500)
-    cy.get('app-document-notes').find('.card').its('length').should('eq', 4)
-  })
-
-  it('should support navigation to notes tab by url', () => {
-    cy.visit('/documents/1/notes')
-    cy.get('app-document-notes').should('exist')
-  })
-
-  it('should dynamically update note counts', () => {
-    cy.visit('/documents/1/notes')
-    cy.get('app-document-notes').within(() => cy.contains('Delete').click())
-    cy.get('ul.nav').find('li').contains('Notes').find('.badge').contains('2')
-  })
-})