Instead of using a full image name, use the repo and version to build the image to pull from. Removes building of the frontend for multiple platforms

.github/workflows/ci.yml

@@ -63,10 +63,6 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
-      -
-        name: Get branch name
-        id: branch-name
-        uses: tj-actions/branch-names@v5
       -
         name: Login to Github Container Registry
         uses: docker/login-action@v1
@@ -123,16 +119,14 @@ jobs:
         name: Setup frontend image
         id: frontend-setup
         run: |
-          frontend_image=ghcr.io/${{ github.repository }}/ngx-frontend:${{ steps.branch-name.outputs.current_branch }}
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py frontend)
 
-          echo ${frontend_image}
+          echo ${build_json}
 
-          echo ::set-output name=frontend-image-tag::${frontend_image}
+          echo ::set-output name=frontend-json::${build_json}
 
     outputs:
 
-      frontend-image-tag: ${{ steps.frontend-setup.outputs.frontend-image-tag }}
-
       qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
 
       pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
@@ -141,6 +135,8 @@ jobs:
 
       jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
 
+      frontend-json: ${{ steps.frontend-setup.outputs.frontend-json}}
+
   build-qpdf-debs:
     name: qpdf
     needs:
@@ -185,14 +181,15 @@ jobs:
       dockerfile: ./docker-builders/Dockerfile.pikepdf
       build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
       build-args: |
-        QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+        REPO=${{ github.repository }}
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
         GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
         VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
 
   build-frontend:
     name: Compile frontend
     concurrency:
-      group: ${{ github.workflow }}-build-frontend-${{ github.ref }}
+      group: ${{ github.workflow }}-build-frontend-${{ github.ref_name }}
       cancel-in-progress: false
     needs:
       - prepare-docker-build
@@ -222,7 +219,7 @@ jobs:
         id: build-skip-check
         # Skip building the frontend if the tag exists and no src-ui files changed
         run: |
-          if ! docker manifest inspect ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} &> /dev/null ; then
+          if ! docker manifest inspect ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} &> /dev/null ; then
             echo "Build required, no existing image"
             echo ::set-output name=frontend-build-needed::true
           elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then
@@ -247,15 +244,18 @@ jobs:
         with:
           context: .
           file: ./docker-builders/Dockerfile.frontend
-          tags: ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          tags: ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
+          # The compilation is identical between different platforms
+          # The buildx and QEMU setup is left, just in case that ever changes
+          # But the platform is set to the runner's native for speedup
+          platforms: linux/amd64
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
       -
         name: Export frontend artifact from docker
         run: |
-          docker create --name frontend-extract ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+          docker create --name frontend-extract ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
           docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/
       -
         name: Upload frontend artifact
@@ -271,7 +271,7 @@ jobs:
       cancel-in-progress: true
     runs-on: ubuntu-20.04
     concurrency:
-      group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
       cancel-in-progress: true
     needs:
       - prepare-docker-build
@@ -317,11 +317,12 @@ jobs:
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
           build-args: |
-            JBIG2ENC_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
-            QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
-            PIKEPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
-            PSYCOPG2_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
-            FRONTEND_BASE_IMAGE=${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+            REPO=${{ github.repository }}
+            JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+            QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+            PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+            PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+            FRONTEND_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).version }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
       -
@@ -401,7 +402,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs:
       - build-release
-    if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
+    if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'ngx-') || startsWith(github.ref_name, 'beta-'))
     steps:
       -
         name: Download release artifact

Dockerfile

@@ -1,16 +1,19 @@
+# Default to pulling from the main repo registry when manually building
+ARG REPO="paperless-ngx/paperless-ngx"
+
 # These are all built previously in the pipeline
 # They provide either a .deb, .whl or whatever npm outputs
-ARG JBIG2ENC_BASE_IMAGE
-ARG QPDF_BASE_IMAGE
-ARG PIKEPDF_BASE_IMAGE
-ARG PSYCOPG2_BASE_IMAGE
-ARG FRONTEND_BASE_IMAGE
+ARG JBIG2ENC_VERSION
+ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION
+ARG FRONTEND_VERSION
 
-FROM ${JBIG2ENC_BASE_IMAGE} AS jbig2enc-builder
-FROM ${QPDF_BASE_IMAGE} as qpdf-builder
-FROM ${PIKEPDF_BASE_IMAGE} as pikepdf-builder
-FROM ${PSYCOPG2_BASE_IMAGE} as psycopg2-builder
-FROM ${FRONTEND_BASE_IMAGE} as compile-frontend
+FROM ghcr.io/${REPO}/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
+FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+FROM ghcr.io/${REPO}/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
+FROM ghcr.io/${REPO}/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
+FROM ghcr.io/${REPO}/builder/frontend:${FRONTEND_VERSION} as compile-frontend
 
 FROM python:3.9-slim-bullseye as main-app
 

docker-builders/Dockerfile.pikepdf

@@ -1,11 +1,15 @@
 # This Dockerfile builds the pikepdf wheel
 # Inputs:
-#		- QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from
+#		- REPO - Docker repository to pull qpdf from
+#		- QPDF_VERSION - The image qpdf version to copy .deb files from
 #		- GIT_TAG - The Git tag to clone and build from
 #		- VERSION - Used to force the built pikepdf version to match
 
-ARG QPDF_BASE_IMAGE
-FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+# Default to pulling from the main repo registry when manually building
+ARG REPO="paperless-ngx/paperless-ngx"
+
+ARG QPDF_VERSION
+FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
 
 # This does nothing, except provide a name for a copy below
 

docker-builders/get-build-json.py

@@ -1,8 +1,18 @@
 #!/usr/bin/env python3
 """
-This is a helper script to either parse the JSON of the Pipfile.lock
-or otherwise return a JSON object detailing versioning and image tags
-for the packages we build seperately, then copy into the final Docker image
+This is a helper script for the mutli-stage Docker image builder.
+It provides a single point of configuration for package version control.
+The output JSON object is used by the CI workflow to determine what versions
+to build and pull into the final Docker image.
+
+Python package information is obtained from the Pipfile.lock.  As this is
+kept updated by dependabot, it usually will need no further configuration.
+The sole exception currently is pikepdf, which has a dependency on qpdf,
+and is configured here to use the latest version of qpdf built by the workflow.
+
+Other package version information is configured directly below, generally by
+setting the version and Git information, if any.
+
 """
 import argparse
 import json
@@ -14,11 +24,13 @@ CONFIG: Final = {
     # All packages need to be in the dict, even if not configured further
     # as it is used for the possible choices in the argument
     "psycopg2": {},
+    "frontend": {},
     # Most information about Python packages comes from the Pipfile.lock
+    # Excpetion being pikepdf, which needs a specific qpdf
     "pikepdf": {
         "qpdf_version": "10.6.3",
     },
-    # For other packages, it is directly configured, for now
+    # For other packages, version and Git information are directly configured
     # These require manual updates to this file for version updates
     "qpdf": {
         "version": "10.6.3",
@@ -59,11 +71,9 @@ def _main():
     output = {"name": args.package}
 
     # Read Pipfile.lock file
-
     pipfile_data = json.loads(pip_lock.read_text())
 
     # Read the version from Pipfile.lock
-
     if args.package in pipfile_data["default"]:
 
         pkg_data = pipfile_data["default"][args.package]
@@ -73,7 +83,6 @@ def _main():
         output["version"] = pkg_version
 
         # Based on the package, generate the expected Git tag name
-
         if args.package == "pikepdf":
             git_tag_name = f"v{pkg_version}"
         elif args.package == "psycopg2":
@@ -81,31 +90,18 @@ def _main():
 
         output["git_tag"] = git_tag_name
 
-        # Based on the package and environment, generate the Docker image tag
+    # Use the basic ref name, minus refs/heads or refs/tags for frontend builder image
+    elif args.package == "frontend":
+        output["version"] = os.environ["GITHUB_REF_NAME"]
 
-        image_tag = _get_image_tag(repo_name, args.package, pkg_version)
+    # Add anything special from the config
+    output.update(CONFIG[args.package])
 
-        output["image_tag"] = image_tag
-
-        # Check for any special configuration, based on package
-
-        if args.package in CONFIG:
-            output.update(CONFIG[args.package])
-
-    elif args.package in CONFIG:
-
-        # This is not a Python package
-
-        output.update(CONFIG[args.package])
-
-        output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
-
-    else:
-        raise NotImplementedError(args.package)
+    # Based on the package and environment, generate the Docker image tag
+    output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
 
     # Output the JSON info to stdout
-
-    print(json.dumps(output, indent=2))
+    print(json.dumps(output))
 
 
 if __name__ == "__main__":