From 100f5422f655c9ecda8735b0ea32eac5ed223d2d Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Wed, 8 Jun 2022 10:33:19 -0700 Subject: [PATCH] Updates the wheel builders to let pip handle finding the sdist of the required package and version --- .github/scripts/get-build-json.py | 10 ---------- .github/workflows/installer-library.yml | 2 -- Dockerfile | 12 ++++++------ build-docker-image.sh | 1 + docker-builders/Dockerfile.pikepdf | 25 +++++++++++-------------- docker-builders/Dockerfile.psycopg2 | 17 +++++++++++------ 6 files changed, 29 insertions(+), 38 deletions(-) diff --git a/.github/scripts/get-build-json.py b/.github/scripts/get-build-json.py index 42299abae..0bb140b03 100755 --- a/.github/scripts/get-build-json.py +++ b/.github/scripts/get-build-json.py @@ -50,7 +50,6 @@ def _main(): # Default output values version = None - git_tag = None extra_config = {} if args.package in pipfile_data["default"]: @@ -59,12 +58,6 @@ def _main(): pkg_version = pkg_data["version"].split("==")[-1] version = pkg_version - # Based on the package, generate the expected Git tag name - if args.package == "pikepdf": - git_tag = f"v{pkg_version}" - elif args.package == "psycopg2": - git_tag = pkg_version.replace(".", "_") - # Any extra/special values needed if args.package == "pikepdf": extra_config["qpdf_version"] = build_json["qpdf"]["version"] @@ -72,8 +65,6 @@ def _main(): elif args.package in build_json: version = build_json[args.package]["version"] - if "git_tag" in build_json[args.package]: - git_tag = build_json[args.package]["git_tag"] else: raise NotImplementedError(args.package) @@ -81,7 +72,6 @@ def _main(): output = { "name": args.package, "version": version, - "git_tag": git_tag, "image_tag": get_image_tag(repo_name, args.package, version), "cache_tag": get_cache_image_tag( repo_name, diff --git a/.github/workflows/installer-library.yml b/.github/workflows/installer-library.yml index 77a66e5d2..effa8278f 100644 --- a/.github/workflows/installer-library.yml +++ b/.github/workflows/installer-library.yml @@ -122,7 +122,6 @@ jobs: dockerfile: ./docker-builders/Dockerfile.psycopg2 build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }} build-args: | - PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }} PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} build-pikepdf-wheel: @@ -137,5 +136,4 @@ jobs: build-args: | REPO=${{ github.repository }} QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }} PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} diff --git a/Dockerfile b/Dockerfile index b1ff73181..c75a3e3aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -151,14 +151,14 @@ RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \ && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/libqpdf28_*.deb \ && apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/qpdf_*.deb \ && echo "Installing pikepdf and dependencies" \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/packaging*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/lxml*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/Pillow*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pyparsing*.whl \ - && python3 -m pip install --no-cache-dir /pikepdf/usr/src/pikepdf/wheels/pikepdf*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/packaging*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/lxml*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/Pillow*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pyparsing*.whl \ + && python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/pikepdf*.whl \ && python -m pip list \ && echo "Installing psycopg2" \ - && python3 -m pip install --no-cache-dir /psycopg2/usr/src/psycopg2/wheels/psycopg2*.whl \ + && python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \ && python -m pip list # Python dependencies diff --git a/build-docker-image.sh b/build-docker-image.sh index ed20589f4..3c868344e 100755 --- a/build-docker-image.sh +++ b/build-docker-image.sh @@ -34,6 +34,7 @@ branch_name=$(git rev-parse --abbrev-ref HEAD) export DOCKER_BUILDKIT=1 docker build --file "$1" \ + --progress=plain \ --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \ --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \ --build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \ diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf index d37014126..3e16d8067 100644 --- a/docker-builders/Dockerfile.pikepdf +++ b/docker-builders/Dockerfile.pikepdf @@ -23,7 +23,6 @@ ARG BUILD_PACKAGES="\ build-essential \ python3-dev \ python3-pip \ - git \ # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers libgnutls28-dev \ # lxml requrements - https://lxml.de/installation.html @@ -72,21 +71,19 @@ RUN set -eux \ # For better caching, seperate the basic installs from # the building -ARG PIKEPDF_GIT_TAG ARG PIKEPDF_VERSION RUN set -eux \ - && echo "building pikepdf wheel" \ - # Note the v in the tag name here - && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \ - && cd pikepdf \ - # pikepdf seems to specifciy either a next version when built OR - # a post release tag. - # In either case, this won't match what we want from requirements.txt - # Directly modify the setup.py to set the version we just checked out of Git - && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \ - # https://github.com/pikepdf/pikepdf/issues/323 - && rm pyproject.toml \ + && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \ && mkdir wheels \ - && python3 -m pip wheel . --wheel-dir wheels \ + && python3 -m pip wheel \ + # Build the package at the requried version + pikepdf==${PIKEPDF_VERSION} \ + # Output the *.whl into this directory + --wheel-dir wheels \ + # Do not use a binary packge for the package being built + --no-binary=pikepdf \ + # Do use binary packages for dependencies + --prefer-binary \ + --no-cache-dir \ && ls -ahl wheels diff --git a/docker-builders/Dockerfile.psycopg2 b/docker-builders/Dockerfile.psycopg2 index 78bdf2438..7fc59fc22 100644 --- a/docker-builders/Dockerfile.psycopg2 +++ b/docker-builders/Dockerfile.psycopg2 @@ -11,7 +11,6 @@ ARG DEBIAN_FRONTEND=noninteractive ARG BUILD_PACKAGES="\ build-essential \ - git \ python3-dev \ python3-pip \ # https://www.psycopg.org/docs/install.html#prerequisites @@ -32,14 +31,20 @@ RUN set -eux \ # For better caching, seperate the basic installs from # the building -ARG PSYCOPG2_GIT_TAG ARG PSYCOPG2_VERSION RUN set -eux \ - && echo "Building psycopg2 wheel" \ + && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \ && cd /usr/src \ - && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \ - && cd psycopg2 \ && mkdir wheels \ - && python3 -m pip wheel . --wheel-dir wheels \ + && python3 -m pip wheel \ + # Build the package at the requried version + psycopg2==${PSYCOPG2_VERSION} \ + # Output the *.whl into this directory + --wheel-dir wheels \ + # Do not use a binary packge for the package being built + --no-binary=psycopg2 \ + # Do use binary packages for dependencies + --prefer-binary \ + --no-cache-dir \ && ls -ahl wheels/