Updates the wheel builders to let pip handle finding the sdist of the required package and version

.github/scripts/get-build-json.py

@@ -50,7 +50,6 @@ def _main():
 
     # Default output values
     version = None
-    git_tag = None
     extra_config = {}
 
     if args.package in pipfile_data["default"]:
@@ -59,12 +58,6 @@ def _main():
         pkg_version = pkg_data["version"].split("==")[-1]
         version = pkg_version
 
-        # Based on the package, generate the expected Git tag name
-        if args.package == "pikepdf":
-            git_tag = f"v{pkg_version}"
-        elif args.package == "psycopg2":
-            git_tag = pkg_version.replace(".", "_")
-
         # Any extra/special values needed
         if args.package == "pikepdf":
             extra_config["qpdf_version"] = build_json["qpdf"]["version"]
@@ -72,8 +65,6 @@ def _main():
     elif args.package in build_json:
         version = build_json[args.package]["version"]
 
-        if "git_tag" in build_json[args.package]:
-            git_tag = build_json[args.package]["git_tag"]
     else:
         raise NotImplementedError(args.package)
 
@@ -81,7 +72,6 @@ def _main():
     output = {
         "name": args.package,
         "version": version,
-        "git_tag": git_tag,
         "image_tag": get_image_tag(repo_name, args.package, version),
         "cache_tag": get_cache_image_tag(
             repo_name,

.github/workflows/installer-library.yml

@@ -122,7 +122,6 @@ jobs:
       dockerfile: ./docker-builders/Dockerfile.psycopg2
       build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
       build-args: |
-        PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
         PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
 
   build-pikepdf-wheel:
@@ -137,5 +136,4 @@ jobs:
       build-args: |
         REPO=${{ github.repository }}
         QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-        PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
         PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}

Dockerfile

@@ -151,14 +151,14 @@ RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
     && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \
     && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \
   && echo "Installing pikepdf and dependencies" \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/packaging*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/lxml*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/Pillow*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pyparsing*.whl \
-    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pikepdf*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \
     && python -m pip list \
   && echo "Installing psycopg2" \
-    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/psycopg2/wheels/psycopg2*.whl \
+    && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
     && python -m pip list
 
 # Python dependencies

build-docker-image.sh

@@ -34,6 +34,7 @@ branch_name=$(git rev-parse --abbrev-ref HEAD)
 export DOCKER_BUILDKIT=1
 
 docker build --file "$1" \
+	--progress=plain \
 	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
 	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
 	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \

docker-builders/Dockerfile.pikepdf

@@ -23,7 +23,6 @@ ARG BUILD_PACKAGES="\
   build-essential \
   python3-dev \
   python3-pip \
-  git \
   # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers
   libgnutls28-dev \
   # lxml requrements - https://lxml.de/installation.html
@@ -72,21 +71,19 @@ RUN set -eux \
 # For better caching, seperate the basic installs from
 # the building
 
-ARG PIKEPDF_GIT_TAG
 ARG PIKEPDF_VERSION
 
 RUN set -eux \
-  && echo "building pikepdf wheel" \
-  # Note the v in the tag name here
-  && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
-  && cd pikepdf \
-  # pikepdf seems to specifciy either a next version when built OR
-  # a post release tag.
-  # In either case, this won't match what we want from requirements.txt
-  # Directly modify the setup.py to set the version we just checked out of Git
-  && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \
-  # https://github.com/pikepdf/pikepdf/issues/323
-  && rm pyproject.toml \
+  && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \
   && mkdir wheels \
-  && python3 -m pip wheel . --wheel-dir wheels \
+  && python3 -m pip wheel \
+    # Build the package at the requried version
+    pikepdf==${PIKEPDF_VERSION} \
+    # Output the *.whl into this directory
+    --wheel-dir wheels \
+    # Do not use a binary packge for the package being built
+    --no-binary=pikepdf \
+    # Do use binary packages for dependencies
+    --prefer-binary \
+    --no-cache-dir \
   && ls -ahl wheels

docker-builders/Dockerfile.psycopg2

@@ -11,7 +11,6 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 ARG BUILD_PACKAGES="\
   build-essential \
-  git \
   python3-dev \
   python3-pip \
   # https://www.psycopg.org/docs/install.html#prerequisites
@@ -32,14 +31,20 @@ RUN set -eux \
 # For better caching, seperate the basic installs from
 # the building
 
-ARG PSYCOPG2_GIT_TAG
 ARG PSYCOPG2_VERSION
 
 RUN set -eux \
-  && echo "Building psycopg2 wheel" \
+  && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \
   && cd /usr/src \
-  && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \
-  && cd psycopg2 \
   && mkdir wheels \
-  && python3 -m pip wheel . --wheel-dir wheels \
+  && python3 -m pip wheel \
+    # Build the package at the requried version
+    psycopg2==${PSYCOPG2_VERSION} \
+    # Output the *.whl into this directory
+    --wheel-dir wheels \
+    # Do not use a binary packge for the package being built
+    --no-binary=psycopg2 \
+    # Do use binary packages for dependencies
+    --prefer-binary \
+    --no-cache-dir \
   && ls -ahl wheels/