From d19015579c8e74fea118e09ab1d11526683e8873 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Thu, 21 Apr 2022 15:47:50 -0700 Subject: [PATCH] Instead of using a full image name, use the repo and version to build the image to pull from. Removes building of the frontend for multiple platforms --- .github/workflows/ci.yml | 45 +++++++++++++------------- Dockerfile | 23 +++++++------ docker-builders/Dockerfile.pikepdf | 10 ++++-- docker-builders/get-build-json.py | 52 ++++++++++++++---------------- 4 files changed, 67 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9ed4306c..fe9afb1a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,10 +63,6 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Get branch name - id: branch-name - uses: tj-actions/branch-names@v5 - name: Login to Github Container Registry uses: docker/login-action@v1 @@ -123,16 +119,14 @@ jobs: name: Setup frontend image id: frontend-setup run: | - frontend_image=ghcr.io/${{ github.repository }}/ngx-frontend:${{ steps.branch-name.outputs.current_branch }} + build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py frontend) - echo ${frontend_image} + echo ${build_json} - echo ::set-output name=frontend-image-tag::${frontend_image} + echo ::set-output name=frontend-json::${build_json} outputs: - frontend-image-tag: ${{ steps.frontend-setup.outputs.frontend-image-tag }} - qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }} pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }} @@ -141,6 +135,8 @@ jobs: jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}} + frontend-json: ${{ steps.frontend-setup.outputs.frontend-json}} + build-qpdf-debs: name: qpdf needs: @@ -185,14 +181,15 @@ jobs: dockerfile: ./docker-builders/Dockerfile.pikepdf build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }} build-args: | - QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }} + REPO=${{ github.repository }} + QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }} VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} build-frontend: name: Compile frontend concurrency: - group: ${{ github.workflow }}-build-frontend-${{ github.ref }} + group: ${{ github.workflow }}-build-frontend-${{ github.ref_name }} cancel-in-progress: false needs: - prepare-docker-build @@ -222,7 +219,7 @@ jobs: id: build-skip-check # Skip building the frontend if the tag exists and no src-ui files changed run: | - if ! docker manifest inspect ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} &> /dev/null ; then + if ! docker manifest inspect ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} &> /dev/null ; then echo "Build required, no existing image" echo ::set-output name=frontend-build-needed::true elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then @@ -247,15 +244,18 @@ jobs: with: context: . file: ./docker-builders/Dockerfile.frontend - tags: ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} - platforms: linux/amd64,linux/arm64,linux/arm/v7 + tags: ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} + # The compilation is identical between different platforms + # The buildx and QEMU setup is left, just in case that ever changes + # But the platform is set to the runner's native for speedup + platforms: linux/amd64 push: true cache-from: type=gha cache-to: type=gha,mode=max - name: Export frontend artifact from docker run: | - docker create --name frontend-extract ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} + docker create --name frontend-extract ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/ - name: Upload frontend artifact @@ -271,7 +271,7 @@ jobs: cancel-in-progress: true runs-on: ubuntu-20.04 concurrency: - group: ${{ github.workflow }}-build-docker-image-${{ github.ref }} + group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }} cancel-in-progress: true needs: - prepare-docker-build @@ -317,11 +317,12 @@ jobs: tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} build-args: | - JBIG2ENC_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }} - QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }} - PIKEPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }} - PSYCOPG2_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }} - FRONTEND_BASE_IMAGE=${{ needs.prepare-docker-build.outputs.frontend-image-tag }} + REPO=${{ github.repository }} + JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }} + QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} + PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} + PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} + FRONTEND_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).version }} cache-from: type=gha cache-to: type=gha,mode=max - @@ -401,7 +402,7 @@ jobs: runs-on: ubuntu-20.04 needs: - build-release - if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-') + if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'ngx-') || startsWith(github.ref_name, 'beta-')) steps: - name: Download release artifact diff --git a/Dockerfile b/Dockerfile index 34f4b2c27..3b71421d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,19 @@ +# Default to pulling from the main repo registry when manually building +ARG REPO="paperless-ngx/paperless-ngx" + # These are all built previously in the pipeline # They provide either a .deb, .whl or whatever npm outputs -ARG JBIG2ENC_BASE_IMAGE -ARG QPDF_BASE_IMAGE -ARG PIKEPDF_BASE_IMAGE -ARG PSYCOPG2_BASE_IMAGE -ARG FRONTEND_BASE_IMAGE +ARG JBIG2ENC_VERSION +ARG QPDF_VERSION +ARG PIKEPDF_VERSION +ARG PSYCOPG2_VERSION +ARG FRONTEND_VERSION -FROM ${JBIG2ENC_BASE_IMAGE} AS jbig2enc-builder -FROM ${QPDF_BASE_IMAGE} as qpdf-builder -FROM ${PIKEPDF_BASE_IMAGE} as pikepdf-builder -FROM ${PSYCOPG2_BASE_IMAGE} as psycopg2-builder -FROM ${FRONTEND_BASE_IMAGE} as compile-frontend +FROM ghcr.io/${REPO}/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder +FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder +FROM ghcr.io/${REPO}/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder +FROM ghcr.io/${REPO}/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder +FROM ghcr.io/${REPO}/builder/frontend:${FRONTEND_VERSION} as compile-frontend FROM python:3.9-slim-bullseye as main-app diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf index 3769caf5f..be544e282 100644 --- a/docker-builders/Dockerfile.pikepdf +++ b/docker-builders/Dockerfile.pikepdf @@ -1,11 +1,15 @@ # This Dockerfile builds the pikepdf wheel # Inputs: -# - QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from +# - REPO - Docker repository to pull qpdf from +# - QPDF_VERSION - The image qpdf version to copy .deb files from # - GIT_TAG - The Git tag to clone and build from # - VERSION - Used to force the built pikepdf version to match -ARG QPDF_BASE_IMAGE -FROM ${QPDF_BASE_IMAGE} as qpdf-builder +# Default to pulling from the main repo registry when manually building +ARG REPO="paperless-ngx/paperless-ngx" + +ARG QPDF_VERSION +FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder # This does nothing, except provide a name for a copy below diff --git a/docker-builders/get-build-json.py b/docker-builders/get-build-json.py index f8d4f8701..a6d24d3fa 100755 --- a/docker-builders/get-build-json.py +++ b/docker-builders/get-build-json.py @@ -1,8 +1,18 @@ #!/usr/bin/env python3 """ -This is a helper script to either parse the JSON of the Pipfile.lock -or otherwise return a JSON object detailing versioning and image tags -for the packages we build seperately, then copy into the final Docker image +This is a helper script for the mutli-stage Docker image builder. +It provides a single point of configuration for package version control. +The output JSON object is used by the CI workflow to determine what versions +to build and pull into the final Docker image. + +Python package information is obtained from the Pipfile.lock. As this is +kept updated by dependabot, it usually will need no further configuration. +The sole exception currently is pikepdf, which has a dependency on qpdf, +and is configured here to use the latest version of qpdf built by the workflow. + +Other package version information is configured directly below, generally by +setting the version and Git information, if any. + """ import argparse import json @@ -14,11 +24,13 @@ CONFIG: Final = { # All packages need to be in the dict, even if not configured further # as it is used for the possible choices in the argument "psycopg2": {}, + "frontend": {}, # Most information about Python packages comes from the Pipfile.lock + # Excpetion being pikepdf, which needs a specific qpdf "pikepdf": { "qpdf_version": "10.6.3", }, - # For other packages, it is directly configured, for now + # For other packages, version and Git information are directly configured # These require manual updates to this file for version updates "qpdf": { "version": "10.6.3", @@ -59,11 +71,9 @@ def _main(): output = {"name": args.package} # Read Pipfile.lock file - pipfile_data = json.loads(pip_lock.read_text()) # Read the version from Pipfile.lock - if args.package in pipfile_data["default"]: pkg_data = pipfile_data["default"][args.package] @@ -73,7 +83,6 @@ def _main(): output["version"] = pkg_version # Based on the package, generate the expected Git tag name - if args.package == "pikepdf": git_tag_name = f"v{pkg_version}" elif args.package == "psycopg2": @@ -81,31 +90,18 @@ def _main(): output["git_tag"] = git_tag_name - # Based on the package and environment, generate the Docker image tag + # Use the basic ref name, minus refs/heads or refs/tags for frontend builder image + elif args.package == "frontend": + output["version"] = os.environ["GITHUB_REF_NAME"] - image_tag = _get_image_tag(repo_name, args.package, pkg_version) + # Add anything special from the config + output.update(CONFIG[args.package]) - output["image_tag"] = image_tag - - # Check for any special configuration, based on package - - if args.package in CONFIG: - output.update(CONFIG[args.package]) - - elif args.package in CONFIG: - - # This is not a Python package - - output.update(CONFIG[args.package]) - - output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"]) - - else: - raise NotImplementedError(args.package) + # Based on the package and environment, generate the Docker image tag + output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"]) # Output the JSON info to stdout - - print(json.dumps(output, indent=2)) + print(json.dumps(output)) if __name__ == "__main__":