Changes the intermediate image building steps to use registry caching, allowig us to always rebuild them, but do so very quickly when nothing has changed

.github/scripts/common.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+
+def get_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+) -> str:
+    """
+    Returns a string representing the normal image for a given package
+    """
+    return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+
+
+def get_cache_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+    branch_name: str,
+) -> str:
+    """
+    Returns a string representing the expected image cache tag for a given package
+
+    Registry type caching is utilized for the builder images, to allow fast
+    rebuilds, generally almost instant for the same version
+    """
+    return f"ghcr.io/{repo_name}/builder/cache/{pkg_name}:{pkg_version}"

.github/scripts/get-build-json.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""
+This is a helper script for the mutli-stage Docker image builder.
+It provides a single point of configuration for package version control.
+The output JSON object is used by the CI workflow to determine what versions
+to build and pull into the final Docker image.
+
+Python package information is obtained from the Pipfile.lock.  As this is
+kept updated by dependabot, it usually will need no further configuration.
+The sole exception currently is pikepdf, which has a dependency on qpdf,
+and is configured here to use the latest version of qpdf built by the workflow.
+
+Other package version information is configured directly below, generally by
+setting the version and Git information, if any.
+
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Final
+
+from common import get_cache_image_tag
+from common import get_image_tag
+
+
+def _main():
+    parser = argparse.ArgumentParser(
+        description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
+    )
+    parser.add_argument(
+        "package",
+        help="The name of the package to generate JSON for",
+    )
+
+    PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock")
+    BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json")
+
+    # Read the main config file
+    build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text())
+
+    # Read Pipfile.lock file
+    pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text())
+
+    args: Final = parser.parse_args()
+
+    # Read from environment variables set by GitHub Actions
+    repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"]
+    branch_name: Final[str] = os.environ["GITHUB_REF_NAME"]
+
+    # Default output values
+    version = None
+    git_tag = None
+    extra_config = {}
+
+    if args.package == "frontend":
+        # Version is just the branch or tag name
+        version = branch_name
+    elif args.package in pipfile_data["default"]:
+        # Read the version from Pipfile.lock
+        pkg_data = pipfile_data["default"][args.package]
+        pkg_version = pkg_data["version"].split("==")[-1]
+        version = pkg_version
+
+        # Based on the package, generate the expected Git tag name
+        if args.package == "pikepdf":
+            git_tag = f"v{pkg_version}"
+        elif args.package == "psycopg2":
+            git_tag = pkg_version.replace(".", "_")
+
+        # Any extra/special values needed
+        if args.package == "pikepdf":
+            extra_config["qpdf_version"] = build_json["qpdf"]["version"]
+
+    elif args.package in build_json:
+        version = build_json[args.package]["version"]
+
+        if "git_tag" in build_json[args.package]:
+            git_tag = build_json[args.package]["git_tag"]
+    else:
+        raise NotImplementedError(args.package)
+
+    # The JSON object we'll output
+    output = {
+        "name": args.package,
+        "version": version,
+        "git_tag": git_tag,
+        "image_tag": get_image_tag(repo_name, args.package, version),
+        "cache_tag": get_cache_image_tag(
+            repo_name,
+            args.package,
+            version,
+            branch_name,
+        ),
+    }
+
+    # Add anything special a package may need
+    output.update(extra_config)
+
+    # Output the JSON info to stdout
+    print(json.dumps(output))
+
+
+if __name__ == "__main__":
+    _main()

.github/workflows/ci.yml

@@ -75,15 +75,11 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: "3.9"
-      -
-        name: Make script executable
-        run: |
-          chmod +x ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py
       -
         name: Setup qpdf image
         id: qpdf-setup
         run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py qpdf)
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
 
           echo ${build_json}
 
@@ -92,7 +88,7 @@ jobs:
         name: Setup psycopg2 image
         id: psycopg2-setup
         run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py psycopg2)
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
 
           echo ${build_json}
 
@@ -101,7 +97,7 @@ jobs:
         name: Setup pikepdf image
         id: pikepdf-setup
         run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py pikepdf)
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
 
           echo ${build_json}
 
@@ -110,7 +106,7 @@ jobs:
         name: Setup jbig2enc image
         id: jbig2enc-setup
         run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py jbig2enc)
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
 
           echo ${build_json}
 
@@ -119,7 +115,7 @@ jobs:
         name: Setup frontend image
         id: frontend-setup
         run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py frontend)
+          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py frontend)
 
           echo ${build_json}
 
@@ -198,15 +194,6 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      -
-        name: Get changed frontend files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v18.1
-        with:
-          files: |
-            src-ui/**
       -
         name: Login to Github Container Registry
         uses: docker/login-action@v1
@@ -214,33 +201,15 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Determine if build needed
-        id: build-skip-check
-        # Skip building the frontend if the tag exists and no src-ui files changed
-        run: |
-          if ! docker manifest inspect ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} &> /dev/null ; then
-            echo "Build required, no existing image"
-            echo ::set-output name=frontend-build-needed::true
-          elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then
-            echo "Build required, src-ui changes"
-            echo ::set-output name=frontend-build-needed::true
-          else
-            echo "No build required"
-            echo ::set-output name=frontend-build-needed::false
-          fi
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
       -
         name: Compile frontend
         uses: docker/build-push-action@v2
-        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
         with:
           context: .
           file: ./docker-builders/Dockerfile.frontend
@@ -250,8 +219,8 @@ jobs:
           # But the platform is set to the runner's native for speedup
           platforms: linux/amd64
           push: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=registry,ref=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).cache_tag }}
+          cache-to: type=registry,mode=max,ref=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).cache_tag }}
       -
         name: Export frontend artifact from docker
         run: |

.github/workflows/reusable-ci-backend.yml

@@ -87,7 +87,7 @@ jobs:
       -
         name: Get changed files
         id: changed-files-specific
-        uses: tj-actions/changed-files@v18.1
+        uses: tj-actions/changed-files@v18.7
         with:
           files: |
             src/**

.github/workflows/reusable-workflow-builder.yml

@@ -23,6 +23,9 @@ jobs:
     name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
     runs-on: ubuntu-latest
     steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
       -
         name: Login to Github Container Registry
         uses: docker/login-action@v1
@@ -30,33 +33,15 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Determine if build needed
-        id: build-skip-check
-        run: |
-          if ! docker manifest inspect ${{ fromJSON(inputs.build-json).image_tag }} &> /dev/null ; then
-            echo "Building, no image exists with this version"
-            echo ::set-output name=image-exists::false
-          else
-            echo "Not building, image exists with this version"
-            echo ::set-output name=image-exists::true
-          fi
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v1
-        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
       -
         name: Build ${{ fromJSON(inputs.build-json).name }}
         uses: docker/build-push-action@v2
-        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
         with:
           context: .
           file: ${{ inputs.dockerfile }}
@@ -64,5 +49,5 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/arm/v7
           build-args: ${{ inputs.build-args }}
           push: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}
+          cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }}