From 100f5422f655c9ecda8735b0ea32eac5ed223d2d Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Wed, 8 Jun 2022 10:33:19 -0700 Subject: [PATCH 1/2] Updates the wheel builders to let pip handle finding the sdist of the required package and version --- .github/scripts/get-build-json.py | 10 ---------- .github/workflows/installer-library.yml | 2 -- Dockerfile | 12 ++++++------ build-docker-image.sh | 1 + docker-builders/Dockerfile.pikepdf | 25 +++++++++++-------------- docker-builders/Dockerfile.psycopg2 | 17 +++++++++++------ 6 files changed, 29 insertions(+), 38 deletions(-) diff --git a/.github/scripts/get-build-json.py b/.github/scripts/get-build-json.py index 42299abae..0bb140b03 100755 --- a/.github/scripts/get-build-json.py +++ b/.github/scripts/get-build-json.py @@ -50,7 +50,6 @@ def _main(): # Default output values version = None - git_tag = None extra_config = {} if args.package in pipfile_data["default"]: @@ -59,12 +58,6 @@ def _main(): pkg_version = pkg_data["version"].split("==")[-1] version = pkg_version - # Based on the package, generate the expected Git tag name - if args.package == "pikepdf": - git_tag = f"v{pkg_version}" - elif args.package == "psycopg2": - git_tag = pkg_version.replace(".", "_") - # Any extra/special values needed if args.package == "pikepdf": extra_config["qpdf_version"] = build_json["qpdf"]["version"] @@ -72,8 +65,6 @@ def _main(): elif args.package in build_json: version = build_json[args.package]["version"] - if "git_tag" in build_json[args.package]: - git_tag = build_json[args.package]["git_tag"] else: raise NotImplementedError(args.package) @@ -81,7 +72,6 @@ def _main(): output = { "name": args.package, "version": version, - "git_tag": git_tag, "image_tag": get_image_tag(repo_name, args.package, version), "cache_tag": get_cache_image_tag( repo_name, diff --git a/.github/workflows/installer-library.yml b/.github/workflows/installer-library.yml index 77a66e5d2..effa8278f 100644 --- a/.github/workflows/installer-library.yml +++ b/.github/workflows/installer-library.yml @@ -122,7 +122,6 @@ jobs: dockerfile: ./docker-builders/Dockerfile.psycopg2 build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }} build-args: | - PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }} PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} build-pikepdf-wheel: @@ -137,5 +136,4 @@ jobs: build-args: | REPO=${{ github.repository }} QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }} PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} diff --git a/Dockerfile b/Dockerfile index b1ff73181..c75a3e3aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -151,14 +151,14 @@ RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \ && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \ && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \ && echo "Installing pikepdf and dependencies" \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/packaging*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/lxml*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/Pillow*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pyparsing*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pikepdf*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \ && python -m pip list \ && echo "Installing psycopg2" \ - && python3 -m pip install --no-cache-dir /psycopg2/usr/src/psycopg2/wheels/psycopg2*.whl \ + && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \ && python -m pip list # Python dependencies diff --git a/build-docker-image.sh b/build-docker-image.sh index ed20589f4..3c868344e 100755 --- a/build-docker-image.sh +++ b/build-docker-image.sh @@ -34,6 +34,7 @@ branch_name=$(git rev-parse --abbrev-ref HEAD) export DOCKER_BUILDKIT=1 docker build --file "$1" \ + --progress=plain \ --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \ --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \ --build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \ diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf index d37014126..3e16d8067 100644 --- a/docker-builders/Dockerfile.pikepdf +++ b/docker-builders/Dockerfile.pikepdf @@ -23,7 +23,6 @@ ARG BUILD_PACKAGES="\ build-essential \ python3-dev \ python3-pip \ - git \ # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers libgnutls28-dev \ # lxml requrements - https://lxml.de/installation.html @@ -72,21 +71,19 @@ RUN set -eux \ # For better caching, seperate the basic installs from # the building -ARG PIKEPDF_GIT_TAG ARG PIKEPDF_VERSION RUN set -eux \ - && echo "building pikepdf wheel" \ - # Note the v in the tag name here - && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \ - && cd pikepdf \ - # pikepdf seems to specifciy either a next version when built OR - # a post release tag. - # In either case, this won't match what we want from requirements.txt - # Directly modify the setup.py to set the version we just checked out of Git - && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \ - # https://github.com/pikepdf/pikepdf/issues/323 - && rm pyproject.toml \ + && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \ && mkdir wheels \ - && python3 -m pip wheel . --wheel-dir wheels \ + && python3 -m pip wheel \ + # Build the package at the requried version + pikepdf==${PIKEPDF_VERSION} \ + # Output the *.whl into this directory + --wheel-dir wheels \ + # Do not use a binary packge for the package being built + --no-binary=pikepdf \ + # Do use binary packages for dependencies + --prefer-binary \ + --no-cache-dir \ && ls -ahl wheels diff --git a/docker-builders/Dockerfile.psycopg2 b/docker-builders/Dockerfile.psycopg2 index 78bdf2438..7fc59fc22 100644 --- a/docker-builders/Dockerfile.psycopg2 +++ b/docker-builders/Dockerfile.psycopg2 @@ -11,7 +11,6 @@ ARG DEBIAN_FRONTEND=noninteractive ARG BUILD_PACKAGES="\ build-essential \ - git \ python3-dev \ python3-pip \ # https://www.psycopg.org/docs/install.html#prerequisites @@ -32,14 +31,20 @@ RUN set -eux \ # For better caching, seperate the basic installs from # the building -ARG PSYCOPG2_GIT_TAG ARG PSYCOPG2_VERSION RUN set -eux \ - && echo "Building psycopg2 wheel" \ + && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \ && cd /usr/src \ - && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \ - && cd psycopg2 \ && mkdir wheels \ - && python3 -m pip wheel . --wheel-dir wheels \ + && python3 -m pip wheel \ + # Build the package at the requried version + psycopg2==${PSYCOPG2_VERSION} \ + # Output the *.whl into this directory + --wheel-dir wheels \ + # Do not use a binary packge for the package being built + --no-binary=psycopg2 \ + # Do use binary packages for dependencies + --prefer-binary \ + --no-cache-dir \ && ls -ahl wheels/ From ea8e108cdfb703ad0a2424c95e6cdefcf11188d0 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Wed, 8 Jun 2022 12:30:18 -0700 Subject: [PATCH 2/2] Removes references to now not set or used build args --- docker-builders/Dockerfile.pikepdf | 5 ++--- docker-builders/Dockerfile.psycopg2 | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf index 3e16d8067..ff7086b7c 100644 --- a/docker-builders/Dockerfile.pikepdf +++ b/docker-builders/Dockerfile.pikepdf @@ -2,8 +2,7 @@ # Inputs: # - REPO - Docker repository to pull qpdf from # - QPDF_VERSION - The image qpdf version to copy .deb files from -# - PIKEPDF_GIT_TAG - The Git tag to clone and build from -# - PIKEPDF_VERSION - Used to force the built pikepdf version to match +# - PIKEPDF_VERSION - Version of pikepdf to build wheel for # Default to pulling from the main repo registry when manually building ARG REPO="paperless-ngx/paperless-ngx" @@ -77,7 +76,7 @@ RUN set -eux \ && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \ && mkdir wheels \ && python3 -m pip wheel \ - # Build the package at the requried version + # Build the package at the required version pikepdf==${PIKEPDF_VERSION} \ # Output the *.whl into this directory --wheel-dir wheels \ diff --git a/docker-builders/Dockerfile.psycopg2 b/docker-builders/Dockerfile.psycopg2 index 7fc59fc22..5e6157a02 100644 --- a/docker-builders/Dockerfile.psycopg2 +++ b/docker-builders/Dockerfile.psycopg2 @@ -1,7 +1,6 @@ # This Dockerfile builds the psycopg2 wheel # Inputs: -# - PSYCOPG2_GIT_TAG - The Git tag to clone and build from -# - PSYCOPG2_VERSION - Unused, kept for future possible usage +# - PSYCOPG2_VERSION - Version to build FROM python:3.9-slim-bullseye as main @@ -38,7 +37,7 @@ RUN set -eux \ && cd /usr/src \ && mkdir wheels \ && python3 -m pip wheel \ - # Build the package at the requried version + # Build the package at the required version psycopg2==${PSYCOPG2_VERSION} \ # Output the *.whl into this directory --wheel-dir wheels \