Merge branch 'dev' into feature-permissions

2025-04-02 13:45:10 -05:00 · 2023-01-27 12:35:48 -08:00 · 2023-01-27 12:35:48 -08:00 · d15773f282
commit d15773f282
parent 118e3703dc d2b290f789
21 changed files with 854 additions and 389 deletions
--- a/.github/scripts/cleanup-tags.py
+++ b/.github/scripts/cleanup-tags.py
@ -15,6 +15,8 @@ from github import ContainerPackage
 from github import GithubBranchApi
 from github import GithubContainerRegistryApi

+import docker
+
 logger = logging.getLogger("cleanup-tags")


@ -151,12 +153,16 @@ class RegistryTagsCleaner:
            for tag in sorted(self.tags_to_keep):
                full_name = f"ghcr.io/{self.repo_owner}/{self.package_name}:{tag}"
                logger.info(f"Checking manifest for {full_name}")
+                # TODO: It would be nice to use RegistryData from docker
+                # except the ID doesn't map to anything in the manifest
                try:
                    proc = subprocess.run(
                        [
                            shutil.which("docker"),
-                            "manifest",
+                            "buildx",
+                            "imagetools",
                            "inspect",
+                            "--raw",
                            full_name,
                        ],
                        capture_output=True,
@ -241,6 +247,65 @@ class RegistryTagsCleaner:
        # By default, keep anything which is tagged
        self.tags_to_keep = list(set(self.all_pkgs_tags_to_version.keys()))

+    def check_tags_pull(self):
+        """
+        This method uses the Docker Python SDK to confirm all tags which were
+        kept still pull, for all platforms.
+
+        TODO: This is much slower (although more comprehensive).  Maybe a Pool?
+        """
+        logger.info("Beginning confirmation step")
+        client = docker.from_env()
+        imgs = []
+        for tag in sorted(self.tags_to_keep):
+            repository = f"ghcr.io/{self.repo_owner}/{self.package_name}"
+            for arch, variant in [("amd64", None), ("arm64", None), ("arm", "v7")]:
+                # From 11.2.0 onwards, qpdf is cross compiled, so there is a single arch, amd64
+                # skip others in this case
+                if "qpdf" in self.package_name and arch != "amd64" and tag == "11.2.0":
+                    continue
+                # Skip beta and release candidate tags
+                elif "beta" in tag:
+                    continue
+
+                # Build the platform name
+                if variant is not None:
+                    platform = f"linux/{arch}/{variant}"
+                else:
+                    platform = f"linux/{arch}"
+
+                try:
+                    logger.info(f"Pulling {repository}:{tag} for {platform}")
+                    image = client.images.pull(
+                        repository=repository,
+                        tag=tag,
+                        platform=platform,
+                    )
+                    imgs.append(image)
+                except docker.errors.APIError as e:
+                    logger.error(
+                        f"Failed to pull {repository}:{tag}: {e}",
+                    )
+
+            # Prevent out of space errors by removing after a few
+            # pulls
+            if len(imgs) > 50:
+                for image in imgs:
+                    try:
+                        client.images.remove(image.id)
+                    except docker.errors.APIError as e:
+                        err_str = str(e)
+                        # Ignore attempts to remove images that are partly shared
+                        # Ignore images which are somehow gone already
+                        if (
+                            "must be forced" not in err_str
+                            and "No such image" not in err_str
+                        ):
+                            logger.error(
+                                f"Remove image ghcr.io/{self.repo_owner}/{self.package_name}:{tag} failed: {e}",
+                            )
+                imgs = []
+

 class MainImageTagsCleaner(RegistryTagsCleaner):
    def decide_what_tags_to_keep(self):
@ -397,6 +462,10 @@ def _main():
            # Clean images which are untagged
            cleaner.clean_untagged(args.is_manifest)

+            # Verify remaining tags still pull
+            if args.is_manifest:
+                cleaner.check_tags_pull()
+

 if __name__ == "__main__":
    _main()
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -212,12 +212,6 @@ jobs:
    name: Prepare Docker Pipeline Data
    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
    runs-on: ubuntu-22.04
-    # If the push triggered the installer library workflow, wait for it to
-    # complete here.  This ensures the required versions for the final
-    # image have been built, while not waiting at all if the versions haven't changed
-    concurrency:
-      group: build-installer-library
-      cancel-in-progress: false
    needs:
      - documentation
      - tests-backend
@ -500,7 +494,7 @@ jobs:
      -
        name: Create Release and Changelog
        id: create-release
-        uses: paperless-ngx/release-drafter@master
+        uses: release-drafter/release-drafter@v5
        with:
          name: Paperless-ngx ${{ steps.get_version.outputs.version }}
          tag: ${{ steps.get_version.outputs.version }}
--- a/.github/workflows/cleanup-tags.yml
+++ b/.github/workflows/cleanup-tags.yml
@ -62,9 +62,9 @@ jobs:
        with:
          python-version: "3.10"
      -
-        name: Install httpx
+        name: Install Python libraries
        run: |
-          python -m pip install httpx
+          python -m pip install httpx docker
      #
      # Clean up primary package
      #
@ -81,13 +81,3 @@ jobs:
        if: "${{ env.TOKEN != '' }}"
        run: |
          python ${GITHUB_WORKSPACE}/.github/scripts/cleanup-tags.py --untagged --delete "${{ matrix.cache-name }}"
-      #
-      # Verify tags which are left still pull
-      #
-      -
-        name: Check all tags still pull
-        run: |
-          ghcr_name=$(echo "ghcr.io/${GITHUB_REPOSITORY_OWNER}/${{ matrix.primary-name }}" | awk '{ print tolower($0) }')
-          echo "Pulling all tags of ${ghcr_name}"
-          docker pull --quiet --all-tags ${ghcr_name}
-          docker image list
--- a/.github/workflows/installer-library.yml
+++ b/.github/workflows/installer-library.yml
@ -169,3 +169,142 @@ jobs:
        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
        PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
        LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}
+
+  commit-binary-files:
+    name: Store installers
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+      - build-jbig2enc
+      - build-psycopg2-wheel
+      - build-pikepdf-wheel
+    runs-on: ubuntu-22.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: binary-library
+      -
+        name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends tree
+      -
+        name: Extract qpdf files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+
+          docker pull --quiet ${tag}
+          docker create --name qpdf-extract ${tag}
+
+          mkdir --parents qpdf/${version}/amd64
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/amd64 qpdf/${version}
+
+          mkdir --parents qpdf/${version}/arm64
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/arm64 qpdf/${version}
+
+          mkdir --parents qpdf/${version}/armv7
+          docker cp qpdf-extract:/usr/src/qpdf/${version}/armv7 qpdf/${version}
+      -
+        name: Extract psycopg2 files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/amd64
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/amd64
+          mv psycopg2/${version}/amd64/wheels/* psycopg2/${version}/amd64
+          rm -r psycopg2/${version}/amd64/wheels/
+          docker rm psycopg2-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/arm64
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/arm64
+          mv psycopg2/${version}/arm64/wheels/* psycopg2/${version}/arm64
+          rm -r psycopg2/${version}/arm64/wheels/
+          docker rm psycopg2-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name psycopg2-extract ${tag}
+          mkdir --parents psycopg2/${version}/armv7
+          docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/armv7
+          mv psycopg2/${version}/armv7/wheels/* psycopg2/${version}/armv7
+          rm -r psycopg2/${version}/armv7/wheels/
+          docker rm psycopg2-extract
+      -
+        name: Extract pikepdf files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/amd64
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/amd64
+          mv pikepdf/${version}/amd64/wheels/* pikepdf/${version}/amd64
+          rm -r pikepdf/${version}/amd64/wheels/
+          docker rm pikepdf-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/arm64
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/arm64
+          mv pikepdf/${version}/arm64/wheels/* pikepdf/${version}/arm64
+          rm -r pikepdf/${version}/arm64/wheels/
+          docker rm pikepdf-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name pikepdf-extract ${tag}
+          mkdir --parents pikepdf/${version}/armv7
+          docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/armv7
+          mv pikepdf/${version}/armv7/wheels/* pikepdf/${version}/armv7
+          rm -r pikepdf/${version}/armv7/wheels/
+          docker rm pikepdf-extract
+      -
+        name: Extract jbig2enc files
+        run: |
+          version=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+          tag=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
+
+          docker pull --quiet --platform linux/amd64 ${tag}
+          docker create --platform linux/amd64 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/amd64
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/amd64/
+          mv jbig2enc/${version}/amd64/build/* jbig2enc/${version}/amd64/
+          docker rm jbig2enc-extract
+
+          docker pull --quiet --platform linux/arm64 ${tag}
+          docker create --platform linux/arm64 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/arm64
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/arm64
+          mv jbig2enc/${version}/arm64/build/* jbig2enc/${version}/arm64/
+          docker rm jbig2enc-extract
+
+          docker pull --quiet --platform linux/arm/v7 ${tag}
+          docker create --platform linux/arm/v7 --name jbig2enc-extract ${tag}
+          mkdir --parents jbig2enc/${version}/armv7
+          docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/armv7
+          mv jbig2enc/${version}/armv7/build/* jbig2enc/${version}/armv7/
+          docker rm jbig2enc-extract
+      -
+        name: Show file structure
+        run: |
+          tree .
+      -
+        name: Commit files
+        run: |
+          git config --global user.name "github-actions"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add pikepdf/ qpdf/ psycopg2/ jbig2enc/
+          git commit -m "Updating installer packages" || true
+          git push origin || true
--- a/53
+++ b/53
@ -1,19 +1,5 @@
 # syntax=docker/dockerfile:1.4

-# Pull the installer images from the library
-# These are all built previously
-# They provide either a .deb or .whl
-
-ARG JBIG2ENC_VERSION
-ARG QPDF_VERSION
-ARG PIKEPDF_VERSION
-ARG PSYCOPG2_VERSION
-
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
-FROM --platform=$BUILDPLATFORM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
-FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
-
 FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend

 # This stage compiles the frontend
@ -58,24 +44,21 @@ LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-n
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"

 ARG DEBIAN_FRONTEND=noninteractive
-# Buildx provided
+# Buildx provided, must be defined to use though
 ARG TARGETARCH
 ARG TARGETVARIANT

 # Workflow provided
+ARG JBIG2ENC_VERSION
 ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION

 #
 # Begin installation and configuration
 # Order the steps below from least often changed to most
 #

-# copy jbig2enc
-# Basically will never change again
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
-COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
-
 # Packages need for running
 ARG RUNTIME_PACKAGES="\
  # Python
@ -198,19 +181,29 @@ RUN set -eux \
 # Install the built packages from the installer library images
 # Use mounts to avoid copying installer files into the image
 # These change sometimes
-RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
-    --mount=type=bind,from=psycopg2-builder,target=/psycopg2 \
-    --mount=type=bind,from=pikepdf-builder,target=/pikepdf \
-  set -eux \
+RUN set -eux \
+  && echo "Getting binaries" \
+    && mkdir paperless-ngx \
+    && curl --fail --silent --show-error --output paperless-ngx.tar.gz --location https://github.com/paperless-ngx/paperless-ngx/archive/41d6e7e407af09a0882736d50c89b6e015997bff.tar.gz \
+    && tar -xf paperless-ngx.tar.gz --directory paperless-ngx --strip-components=1 \
+    && cd paperless-ngx \
+    # Setting a specific revision ensures we know what this installed
+    # and ensures cache breaking on changes
+  && echo "Installing jbig2enc" \
+    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/jbig2 /usr/local/bin/ \
+    && cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/libjbig2enc* /usr/local/lib/ \
  && echo "Installing qpdf" \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
-    && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
+    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
+    && apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
  && echo "Installing pikepdf and dependencies" \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/*.whl \
+    && python3 -m pip install --no-cache-dir ./pikepdf/${PIKEPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.whl \
    && python3 -m pip list \
  && echo "Installing psycopg2" \
-    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
-    && python3 -m pip list
+    && python3 -m pip install --no-cache-dir ./psycopg2/${PSYCOPG2_VERSION}/${TARGETARCH}${TARGETVARIANT}/psycopg2*.whl \
+    && python3 -m pip list \
+  && echo "Cleaning up image layer" \
+    && cd ../ \
+    && rm -rf paperless-ngx

 WORKDIR /usr/src/paperless/src/

--- a/docker-builders/Dockerfile.jbig2enc
+++ b/docker-builders/Dockerfile.jbig2enc
@ -29,7 +29,20 @@ RUN set -eux \
    && ./autogen.sh \
    && ./configure \
    && make \
+  && echo "Gathering package data" \
+    && dpkg-query -f '${Package;-40}${Version}\n' -W > ./pkg-list.txt \
  && echo "Cleaning up image" \
    && apt-get -y purge ${BUILD_PACKAGES} \
    && apt-get -y autoremove --purge \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+  && echo "Moving files around" \
+    && mkdir build \
+    # Unlink a symlink that causes problems
+    && unlink ./src/.libs/libjbig2enc.la \
+    # Move what the link pointed to
+    && mv ./src/libjbig2enc.la ./build/ \
+    # Move the shared library .so files
+    && mv ./src/.libs/libjbig2enc* ./build/ \
+    # And move the cli binary
+    && mv ./src/jbig2 ./build/ \
+    && mv ./pkg-list.txt ./build/
--- a/docker-builders/Dockerfile.pikepdf
+++ b/docker-builders/Dockerfile.pikepdf
@ -7,12 +7,17 @@
 # Default to pulling from the main repo registry when manually building
 ARG REPO="paperless-ngx/paperless-ngx"

+# This does nothing, except provide a name for a copy below
 ARG QPDF_VERSION
 FROM --platform=$BUILDPLATFORM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder

-# This does nothing, except provide a name for a copy below
-
-FROM python:3.9-slim-bullseye as main
+#
+# Stage: builder
+# Purpose:
+#  - Build the pikepdf wheel
+#  - Build any dependent wheels which can't be found
+#
+FROM python:3.9-slim-bullseye as builder

 LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"

@ -100,3 +105,14 @@ RUN set -eux \
    && apt-get -y purge ${BUILD_PACKAGES} \
    && apt-get -y autoremove --purge \
    && rm -rf /var/lib/apt/lists/*
+
+#
+# Stage: package
+# Purpose: Holds the compiled .whl files in a tiny image to pull
+#
+FROM alpine:3.17 as package
+
+WORKDIR /usr/src/wheels/
+
+COPY --from=builder /usr/src/wheels/*.whl ./
+COPY --from=builder /usr/src/wheels/pkg-list.txt ./
--- a/docker-builders/Dockerfile.psycopg2
+++ b/docker-builders/Dockerfile.psycopg2
@ -2,7 +2,12 @@
 # Inputs:
 #    - PSYCOPG2_VERSION - Version to build

-FROM python:3.9-slim-bullseye as main
+#
+# Stage: builder
+# Purpose:
+#  - Build the psycopg2 wheel
+#
+FROM python:3.9-slim-bullseye as builder

 LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"

@ -48,3 +53,14 @@ RUN set -eux \
    && apt-get -y purge ${BUILD_PACKAGES} \
    && apt-get -y autoremove --purge \
    && rm -rf /var/lib/apt/lists/*
+
+#
+# Stage: package
+# Purpose: Holds the compiled .whl files in a tiny image to pull
+#
+FROM alpine:3.17 as package
+
+WORKDIR /usr/src/wheels/
+
+COPY --from=builder /usr/src/wheels/*.whl ./
+COPY --from=builder /usr/src/wheels/pkg-list.txt ./
--- a/docker-builders/README.md
+++ b/docker-builders/README.md
@ -0,0 +1,57 @@
+# Installer Library
+
+This folder contains the Dockerfiles for building certain installers or libraries, which are then pulled into the main image.
+
+## [jbig2enc](https://github.com/agl/jbig2enc)
+
+### Why
+
+JBIG is an image coding which can achieve better compression of images for PDFs.
+
+### What
+
+The Docker image builds a shared library file and utility, which is copied into the correct location in the final image.
+
+### Updating
+
+1. Ensure the given qpdf version is present in [Debian bookworm](https://packages.debian.org/bookworm/qpdf)
+2. Update `.build-config.json` to the given version
+3. If the Debian specific version has incremented, update `Dockerfile.qpdf`
+
+See Also:
+
+- [OCRMyPDF Documentation](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html)
+
+## [psycopg2](https://www.psycopg.org/)
+
+### Why
+
+The pre-built wheels of psycopg2 are built on Debian 9, which provides a quite old version of libpq-dev. This causes issue with authentication methods.
+
+### What
+
+The image builds psycopg2 wheels on Debian 10 and places the produced wheels into `/usr/src/wheels/`.
+
+See Also:
+
+- [Issue 266](https://github.com/paperless-ngx/paperless-ngx/issues/266)
+
+## [qpdf](https://qpdf.readthedocs.io/en/stable/index.html)
+
+### Why
+
+qpdf and it's library provide tools to read, manipulate and fix up PDFs. Version 11 is also required by `pikepdf` 6+ and Debian 9 does not provide above version 10.
+
+### What
+
+The Docker image cross compiles .deb installers for each supported architecture of the main image. The installers are placed in `/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/`
+
+## [pikepdf](https://pikepdf.readthedocs.io/en/latest/)
+
+### Why
+
+Required by OCRMyPdf, this is a general purpose library for PDF manipulation in Python via the qpdf libraries.
+
+### What
+
+The built wheels are placed into `/usr/src/wheels/`
--- a/docker/docker-prepare.sh
+++ b/docker/docker-prepare.sh
@ -80,7 +80,7 @@ django_checks() {

 search_index() {

-	local -r index_version=1
+	local -r index_version=2
 	local -r index_version_file=${DATA_DIR}/.index_version

 	if [[ (! -f "${index_version_file}") || $(<"${index_version_file}") != "$index_version" ]]; then
--- a/docs/changelog.md
+++ b/docs/changelog.md
@ -1,5 +1,123 @@
 # Changelog

+## paperless-ngx 1.12.1
+
+_Note: Version 1.12.x introduced searching of comments which will work for comments added after the upgrade but a reindex of the search index is required in order to be able to search
+older comments. The Docker image will automatically perform this reindex, bare metal installations will have to perform this manually, see [the docs](https://docs.paperless-ngx.com/administration/#index)._
+
+### Bug Fixes
+
+- Fix: comments not showing in search until after manual reindex in v1.12 [@shamoon](https://github.com/shamoon) ([#2513](https://github.com/paperless-ngx/paperless-ngx/pull/2513))
+- Fix: date range search broken in 1.12 [@shamoon](https://github.com/shamoon) ([#2509](https://github.com/paperless-ngx/paperless-ngx/pull/2509))
+
+### Documentation
+
+- [Documentation] Add v1.12.0 changelog [@github-actions](https://github.com/github-actions) ([#2507](https://github.com/paperless-ngx/paperless-ngx/pull/2507))
+
+### Maintenance
+
+- Moves back to the main release-drafter now that it does what we wanted [@stumpylog](https://github.com/stumpylog) ([#2503](https://github.com/paperless-ngx/paperless-ngx/pull/2503))
+
+### All App Changes
+
+- Fix: comments not showing in search until after manual reindex in v1.12 [@shamoon](https://github.com/shamoon) ([#2513](https://github.com/paperless-ngx/paperless-ngx/pull/2513))
+- Fix: date range search broken in 1.12 [@shamoon](https://github.com/shamoon) ([#2509](https://github.com/paperless-ngx/paperless-ngx/pull/2509))
+
+## paperless-ngx 1.12.0
+
+### Features
+
+- New document_exporter options [@mhelleboid](https://github.com/mhelleboid) ([#2448](https://github.com/paperless-ngx/paperless-ngx/pull/2448))
+- Read ASN from barcode on page [@peterkappelt](https://github.com/peterkappelt) ([#2437](https://github.com/paperless-ngx/paperless-ngx/pull/2437))
+- Add AppleMail color tag support [@clemensrieder](https://github.com/clemensrieder) ([#2407](https://github.com/paperless-ngx/paperless-ngx/pull/2407))
+- Feature: Retain original filename on upload [@stumpylog](https://github.com/stumpylog) ([#2404](https://github.com/paperless-ngx/paperless-ngx/pull/2404))
+- Feature: Control scheduled tasks via cron expressions [@stumpylog](https://github.com/stumpylog) ([#2403](https://github.com/paperless-ngx/paperless-ngx/pull/2403))
+- Simplify json parsing in build scripts [@tribut](https://github.com/tribut) ([#2370](https://github.com/paperless-ngx/paperless-ngx/pull/2370))
+- Feature: include comments in advanced search [@shamoon](https://github.com/shamoon) ([#2351](https://github.com/paperless-ngx/paperless-ngx/pull/2351))
+
+### Bug Fixes
+
+- Fix: limit asn integer size [@shamoon](https://github.com/shamoon) ([#2498](https://github.com/paperless-ngx/paperless-ngx/pull/2498))
+- Bugfix: Rescales images for better barcode locating [@stumpylog](https://github.com/stumpylog) ([#2468](https://github.com/paperless-ngx/paperless-ngx/pull/2468))
+- Fix: fix downgrade migration [@shamoon](https://github.com/shamoon) ([#2494](https://github.com/paperless-ngx/paperless-ngx/pull/2494))
+- Fix: Allow setting mailrule order from frontend [@shamoon](https://github.com/shamoon) ([#2459](https://github.com/paperless-ngx/paperless-ngx/pull/2459))
+- Fix: tag color ordering [@shamoon](https://github.com/shamoon) ([#2456](https://github.com/paperless-ngx/paperless-ngx/pull/2456))
+- Fix: Better Handle arbitrary ISO 8601 strings after celery serializing [@shamoon](https://github.com/shamoon) ([#2441](https://github.com/paperless-ngx/paperless-ngx/pull/2441))
+- Use correct canonical path for nltk_data [@amo13](https://github.com/amo13) ([#2429](https://github.com/paperless-ngx/paperless-ngx/pull/2429))
+- Fix: Include optional socket file in release [@stumpylog](https://github.com/stumpylog) ([#2409](https://github.com/paperless-ngx/paperless-ngx/pull/2409))
+- Fix: display rtl content in correct direction [@shamoon](https://github.com/shamoon) ([#2302](https://github.com/paperless-ngx/paperless-ngx/pull/2302))
+- Fixed endpoint count in Docs The REST API [@PascalSenn](https://github.com/PascalSenn) ([#2386](https://github.com/paperless-ngx/paperless-ngx/pull/2386))
+- Fix subpath for websockets [@tribut](https://github.com/tribut) ([#2371](https://github.com/paperless-ngx/paperless-ngx/pull/2371))
+- Fix: Make missing environment from file files informational only [@stumpylog](https://github.com/stumpylog) ([#2368](https://github.com/paperless-ngx/paperless-ngx/pull/2368))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Fix: preview content remains hidden on mobile [@shamoon](https://github.com/shamoon) ([#2346](https://github.com/paperless-ngx/paperless-ngx/pull/2346))
+- Bugfix: Removal of alpha channel truncates multipage TIFFs [@stumpylog](https://github.com/stumpylog) ([#2335](https://github.com/paperless-ngx/paperless-ngx/pull/2335))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+
+### Documentation
+
+- Docs: Fix typo - docker-compose.yml file name in setup doc [@muli](https://github.com/muli) ([#2477](https://github.com/paperless-ngx/paperless-ngx/pull/2477))
+- document existence of document_thumbnails [@frrad](https://github.com/frrad) ([#2470](https://github.com/paperless-ngx/paperless-ngx/pull/2470))
+- Add optional sudo command to bare metal docs [@shamoon](https://github.com/shamoon) ([#2464](https://github.com/paperless-ngx/paperless-ngx/pull/2464))
+- Fix link [@edenhaus](https://github.com/edenhaus) ([#2458](https://github.com/paperless-ngx/paperless-ngx/pull/2458))
+- Documentation: Fix comment re bare metal runserver command [@shamoon](https://github.com/shamoon) ([#2420](https://github.com/paperless-ngx/paperless-ngx/pull/2420))
+- Fix formatting of config variable in docs [@peterkappelt](https://github.com/peterkappelt) ([#2445](https://github.com/paperless-ngx/paperless-ngx/pull/2445))
+- Update docs nginx reverse proxy example [@Sprinterfreak](https://github.com/Sprinterfreak) ([#2443](https://github.com/paperless-ngx/paperless-ngx/pull/2443))
+- [Documentation] Add note re for dev server [@shamoon](https://github.com/shamoon) ([#2387](https://github.com/paperless-ngx/paperless-ngx/pull/2387))
+- Fixed endpoint count in Docs The REST API [@PascalSenn](https://github.com/PascalSenn) ([#2386](https://github.com/paperless-ngx/paperless-ngx/pull/2386))
+- [ Docs] Update bare metal setup instructions [@natrius](https://github.com/natrius) ([#2281](https://github.com/paperless-ngx/paperless-ngx/pull/2281))
+- [Docs] Add Paperless Mobile app to docs [@astubenbord](https://github.com/astubenbord) ([#2378](https://github.com/paperless-ngx/paperless-ngx/pull/2378))
+- Tiny spelling change [@veverkap](https://github.com/veverkap) ([#2369](https://github.com/paperless-ngx/paperless-ngx/pull/2369))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+- [Documentation] Add note that PAPERLESS_URL cant contain a path [@shamoon](https://github.com/shamoon) ([#2319](https://github.com/paperless-ngx/paperless-ngx/pull/2319))
+- [Documentation] Add v1.11.3 changelog [@github-actions](https://github.com/github-actions) ([#2311](https://github.com/paperless-ngx/paperless-ngx/pull/2311))
+
+### Maintenance
+
+- Fix: Include optional socket file in release [@stumpylog](https://github.com/stumpylog) ([#2409](https://github.com/paperless-ngx/paperless-ngx/pull/2409))
+- Chore: remove helm chart code [@shamoon](https://github.com/shamoon) ([#2388](https://github.com/paperless-ngx/paperless-ngx/pull/2388))
+- Simplify json parsing in build scripts [@tribut](https://github.com/tribut) ([#2370](https://github.com/paperless-ngx/paperless-ngx/pull/2370))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Bump tj-actions/changed-files from 34 to 35 [@dependabot](https://github.com/dependabot) ([#2303](https://github.com/paperless-ngx/paperless-ngx/pull/2303))
+
+### Dependencies
+
+<details>
+<summary>4 changes</summary>
+
+- Chore: Backend library updates [@stumpylog](https://github.com/stumpylog) ([#2401](https://github.com/paperless-ngx/paperless-ngx/pull/2401))
+- Bump tj-actions/changed-files from 34 to 35 [@dependabot](https://github.com/dependabot) ([#2303](https://github.com/paperless-ngx/paperless-ngx/pull/2303))
+- Bump [@<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2306](https://github.com/paperless-ngx/paperless-ngx/pull/2306))
+- Bump [@<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2308](https://github.com/paperless-ngx/paperless-ngx/pull/2308))
+</details>
+
+### All App Changes
+
+- New document_exporter options [@mhelleboid](https://github.com/mhelleboid) ([#2448](https://github.com/paperless-ngx/paperless-ngx/pull/2448))
+- Fix: limit asn integer size [@shamoon](https://github.com/shamoon) ([#2498](https://github.com/paperless-ngx/paperless-ngx/pull/2498))
+- Fix: fix downgrade migration [@shamoon](https://github.com/shamoon) ([#2494](https://github.com/paperless-ngx/paperless-ngx/pull/2494))
+- Read ASN from barcode on page [@peterkappelt](https://github.com/peterkappelt) ([#2437](https://github.com/paperless-ngx/paperless-ngx/pull/2437))
+- Fix: Allow setting mailrule order from frontend [@shamoon](https://github.com/shamoon) ([#2459](https://github.com/paperless-ngx/paperless-ngx/pull/2459))
+- Chore: Update to Angular 15 \& associated frontend deps [@shamoon](https://github.com/shamoon) ([#2411](https://github.com/paperless-ngx/paperless-ngx/pull/2411))
+- Fix: tag color ordering [@shamoon](https://github.com/shamoon) ([#2456](https://github.com/paperless-ngx/paperless-ngx/pull/2456))
+- Fix: Better Handle arbitrary ISO 8601 strings after celery serializing [@shamoon](https://github.com/shamoon) ([#2441](https://github.com/paperless-ngx/paperless-ngx/pull/2441))
+- Use correct canonical path for nltk_data [@amo13](https://github.com/amo13) ([#2429](https://github.com/paperless-ngx/paperless-ngx/pull/2429))
+- Add AppleMail color tag support [@clemensrieder](https://github.com/clemensrieder) ([#2407](https://github.com/paperless-ngx/paperless-ngx/pull/2407))
+- Chore: Convert document exporter to use pathlib [@stumpylog](https://github.com/stumpylog) ([#2416](https://github.com/paperless-ngx/paperless-ngx/pull/2416))
+- Feature: Retain original filename on upload [@stumpylog](https://github.com/stumpylog) ([#2404](https://github.com/paperless-ngx/paperless-ngx/pull/2404))
+- Feature: Control scheduled tasks via cron expressions [@stumpylog](https://github.com/stumpylog) ([#2403](https://github.com/paperless-ngx/paperless-ngx/pull/2403))
+- Fix: display rtl content in correct direction [@shamoon](https://github.com/shamoon) ([#2302](https://github.com/paperless-ngx/paperless-ngx/pull/2302))
+- Fix subpath for websockets [@tribut](https://github.com/tribut) ([#2371](https://github.com/paperless-ngx/paperless-ngx/pull/2371))
+- Bugfix: Backend tests weren't using correct Python version [@stumpylog](https://github.com/stumpylog) ([#2363](https://github.com/paperless-ngx/paperless-ngx/pull/2363))
+- Feature: include comments in advanced search [@shamoon](https://github.com/shamoon) ([#2351](https://github.com/paperless-ngx/paperless-ngx/pull/2351))
+- Chore: More frontend tests [@shamoon](https://github.com/shamoon) ([#2352](https://github.com/paperless-ngx/paperless-ngx/pull/2352))
+- Chore: Fixing up some minor annoyances [@stumpylog](https://github.com/stumpylog) ([#2348](https://github.com/paperless-ngx/paperless-ngx/pull/2348))
+- Bugfix: Removal of alpha channel truncates multipage TIFFs [@stumpylog](https://github.com/stumpylog) ([#2335](https://github.com/paperless-ngx/paperless-ngx/pull/2335))
+- Documentation: update build instructions to remove deprecated [@shamoon](https://github.com/shamoon) ([#2334](https://github.com/paperless-ngx/paperless-ngx/pull/2334))
+- Add Arabic language to frontend [@KhaledEmad7](https://github.com/KhaledEmad7) ([#2313](https://github.com/paperless-ngx/paperless-ngx/pull/2313))
+- Bump [@<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/parser from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2306](https://github.com/paperless-ngx/paperless-ngx/pull/2306))
+- Bump [@<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot](https://github.com/<!---->typescript-eslint/eslint-plugin from 5.43.0 to 5.47.1 in /src-ui @dependabot) ([#2308](https://github.com/paperless-ngx/paperless-ngx/pull/2308))
+
 ## paperless-ngx 1.11.3

 ### Breaking Changes
--- a/src-ui/messages.xlf
+++ b/src-ui/messages.xlf
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
@ -26,11 +26,11 @@
        </div>
        <p class="card-text">
          <span *ngIf="document.__search_hit__ && document.__search_hit__.highlights" [innerHtml]="document.__search_hit__.highlights"></span>
-          <span *ngIf="document.__search_hit__ && document.__search_hit__.comment_highlights">
+          <span *ngFor="let highlight of searchCommentHighlights" class="d-block">
            <svg width="1em" height="1em" fill="currentColor" class="me-2">
              <use xlink:href="assets/bootstrap-icons.svg#chat-left-text"/>
            </svg>
-            <span [innerHtml]="document.__search_hit__.comment_highlights"></span>
+            <span [innerHtml]="highlight"></span>
          </span>
          <span *ngIf="!document.__search_hit__" class="result-content">{{contentTrimmed}}</span>
        </p>
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
@ -73,6 +73,22 @@ export class DocumentCardLargeComponent extends ComponentWithPermissions {
    }
  }

+  get searchCommentHighlights() {
+    let highlights = []
+    if (
+      this.document['__search_hit__'] &&
+      this.document['__search_hit__'].comment_highlights
+    ) {
+      // only show comments with a match
+      highlights = (
+        this.document['__search_hit__'].comment_highlights as string
+      )
+        .split(',')
+        .filter((higlight) => higlight.includes('<span'))
+    }
+    return highlights
+  }
+
  getIsThumbInverted() {
    return this.settingsService.get(SETTINGS_KEYS.DARK_MODE_THUMB_INVERTED)
  }
--- a/src-ui/src/app/components/manage/settings/settings.component.html
+++ b/src-ui/src/app/components/manage/settings/settings.component.html
@ -143,7 +143,7 @@
            <p i18n>
              <em>No tracking data is collected by the app in any way.</em>
            </p>
-            <app-input-check i18n-title title="Enable update checking" formControlName="updateCheckingEnabled" i18n-hint hint="Note that for users of thirdy-party containers e.g. linuxserver.io this notification may be 'ahead' of the current third-party release."></app-input-check>
+            <app-input-check i18n-title title="Enable update checking" formControlName="updateCheckingEnabled" i18n-hint hint="Note that for users of third-party containers e.g. linuxserver.io this notification may be 'ahead' of the current third-party release."></app-input-check>
          </div>
        </div>

--- a/src-ui/src/environments/environment.prod.ts
+++ b/src-ui/src/environments/environment.prod.ts
@ -5,7 +5,7 @@ export const environment = {
  apiBaseUrl: document.baseURI + 'api/',
  apiVersion: '2',
  appTitle: 'Paperless-ngx',
-  version: '1.12.0-dev',
+  version: '1.12.1-dev',
  webSocketHost: window.location.host,
  webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
  webSocketBaseUrl: base_url.pathname + 'ws/',
--- a/src/documents/views.py
+++ b/src/documents/views.py
@ -481,6 +481,10 @@ class DocumentViewSet(
                )
                c.save()

+                from documents import index
+
+                index.add_or_update_document(self.get_object())
+
                return Response(self.getComments(doc))
            except Exception as e:
                logger.warning(f"An error occurred saving comment: {str(e)}")
@ -492,6 +496,11 @@ class DocumentViewSet(
        elif request.method == "DELETE":
            comment = Comment.objects.get(id=int(request.GET.get("id")))
            comment.delete()
+
+            from documents import index
+
+            index.add_or_update_document(self.get_object())
+
            return Response(self.getComments(doc))

        return Response(
@ -504,21 +513,14 @@ class DocumentViewSet(
 class SearchResultSerializer(DocumentSerializer, PassUserMixin):
    def to_representation(self, instance):
        doc = Document.objects.get(id=instance["id"])
-        comments = ""
-        if hasattr(instance.results.q, "subqueries"):
-            commentTerm = instance.results.q.subqueries[0]
-            comments = ",".join(
-                [
-                    str(c.comment)
-                    for c in Comment.objects.filter(document=instance["id"])
-                    if commentTerm.text in c.comment
-                ],
-            )
+        comments = ",".join(
+            [str(c.comment) for c in Comment.objects.filter(document=instance["id"])],
+        )
        r = super().to_representation(doc)
        r["__search_hit__"] = {
            "score": instance.score,
            "highlights": instance.highlights("content", text=doc.content),
-            "comment_highlights": instance.highlights("content", text=comments)
+            "comment_highlights": instance.highlights("comments", text=comments)
            if doc
            else None,
            "rank": instance.rank,
--- a/src/paperless/version.py
+++ b/src/paperless/version.py
@ -1,7 +1,7 @@
 from typing import Final
 from typing import Tuple

-__version__: Final[Tuple[int, int, int]] = (1, 12, 0)
+__version__: Final[Tuple[int, int, int]] = (1, 12, 1)
 # Version string like X.Y.Z
 __full_version_str__: Final[str] = ".".join(map(str, __version__))
 # Version string like X.Y
--- a/src/paperless_mail/parsers.py
+++ b/src/paperless_mail/parsers.py
@ -271,6 +271,16 @@ class MailDocumentParser(DocumentParser):
                "paperHeight": "11.7",
                "scale": "1.0",
            }
+
+            # Set the output format of the resulting PDF
+            # Valid inputs: https://gotenberg.dev/docs/modules/pdf-engines#uno
+            if settings.OCR_OUTPUT_TYPE in {"pdfa", "pdfa-2"}:
+                data["pdfFormat"] = "PDF/A-2b"
+            elif settings.OCR_OUTPUT_TYPE == "pdfa-1":
+                data["pdfFormat"] = "PDF/A-1a"
+            elif settings.OCR_OUTPUT_TYPE == "pdfa-3":
+                data["pdfFormat"] = "PDF/A-3b"
+
            try:
                response = requests.post(
                    url,
--- a/src/paperless_mail/tests/test_parsers.py
+++ b/src/paperless_mail/tests/test_parsers.py
@ -573,8 +573,8 @@ class TestParser(TestCase):
            self.parser.gotenberg_server + "/forms/chromium/convert/html",
            mock_post.call_args.args[0],
        )
-        self.assertEqual({}, mock_post.call_args.kwargs["headers"])
-        self.assertEqual(
+        self.assertDictEqual({}, mock_post.call_args.kwargs["headers"])
+        self.assertDictEqual(
            {
                "marginTop": "0.1",
                "marginBottom": "0.1",
@ -583,6 +583,7 @@ class TestParser(TestCase):
                "paperWidth": "8.27",
                "paperHeight": "11.7",
                "scale": "1.0",
+                "pdfFormat": "PDF/A-2b",
            },
            mock_post.call_args.kwargs["data"],
        )
@ -663,8 +664,8 @@ class TestParser(TestCase):
            self.parser.gotenberg_server + "/forms/chromium/convert/html",
            mock_post.call_args.args[0],
        )
-        self.assertEqual({}, mock_post.call_args.kwargs["headers"])
-        self.assertEqual(
+        self.assertDictEqual({}, mock_post.call_args.kwargs["headers"])
+        self.assertDictEqual(
            {
                "marginTop": "0.1",
                "marginBottom": "0.1",
--- a/src/paperless_tika/parsers.py
+++ b/src/paperless_tika/parsers.py
@ -95,9 +95,19 @@ class TikaDocumentParser(DocumentParser):
                ),
            }
            headers = {}
+            data = {}
+
+            # Set the output format of the resulting PDF
+            # Valid inputs: https://gotenberg.dev/docs/modules/pdf-engines#uno
+            if settings.OCR_OUTPUT_TYPE in {"pdfa", "pdfa-2"}:
+                data["pdfFormat"] = "PDF/A-2b"
+            elif settings.OCR_OUTPUT_TYPE == "pdfa-1":
+                data["pdfFormat"] = "PDF/A-1a"
+            elif settings.OCR_OUTPUT_TYPE == "pdfa-3":
+                data["pdfFormat"] = "PDF/A-3b"

            try:
-                response = requests.post(url, files=files, headers=headers)
+                response = requests.post(url, files=files, headers=headers, data=data)
                response.raise_for_status()  # ensure we notice bad responses
            except Exception as err:
                raise ParseError(