From 87fd18bee1b70fa9bdd811cf4584c7cbc7398a75 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Wed, 11 May 2022 10:50:05 -0700 Subject: [PATCH 1/2] Seperates out the library image building into a new, minimal workflow --- .github/workflows/ci.yml | 59 +--------- .github/workflows/installer-library.yml | 139 ++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/installer-library.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c17e2a210..ed2b264ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,12 @@ jobs: name: Prepare Docker Pipeline Data if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v')) runs-on: ubuntu-20.04 + # If the push triggered the installer library workflow, wait for it to + # complete here. This ensures the required versions for the final + # image have been built, while not waiting at all if the versions haven't changed + concurrency: + group: build-installer-library + cancel-in-progress: false needs: - documentation - ci-backend @@ -117,55 +123,6 @@ jobs: jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}} - build-qpdf-debs: - name: qpdf - needs: - - prepare-docker-build - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.qpdf - build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }} - build-args: | - QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - - build-jbig2enc: - name: jbig2enc - needs: - - prepare-docker-build - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.jbig2enc - build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }} - build-args: | - JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }} - - build-psycopg2-wheel: - name: psycopg2 - needs: - - prepare-docker-build - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.psycopg2 - build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }} - build-args: | - PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }} - PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} - - build-pikepdf-wheel: - name: pikepdf - needs: - - prepare-docker-build - - build-qpdf-debs - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.pikepdf - build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }} - build-args: | - REPO=${{ github.repository }} - QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }} - PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} - # build and push image to docker hub. build-docker-image: runs-on: ubuntu-20.04 @@ -174,10 +131,6 @@ jobs: cancel-in-progress: true needs: - prepare-docker-build - - build-psycopg2-wheel - - build-jbig2enc - - build-qpdf-debs - - build-pikepdf-wheel steps: - name: Check pushing to Docker Hub diff --git a/.github/workflows/installer-library.yml b/.github/workflows/installer-library.yml new file mode 100644 index 000000000..31e08638e --- /dev/null +++ b/.github/workflows/installer-library.yml @@ -0,0 +1,139 @@ +# This workflow will run to update the installer library of +# Docker images. These are the images which provide updated wheels +# .deb installation packages or maybe just some compiled library + +name: Build Image Library + +on: + push: + # Must match one of these branches AND one of the paths + # to be triggered + branches: + - "main" + - "dev" + - "library-*" + - "feature-*" + paths: + # Trigger the workflow if a Dockerfile changed + - "docker-builders/**" + # Trigger if a package was updated + - ".build-config.json" + - "Pipfile.lock" + # Also trigger on workflow changes (such as this file) + - ".github/workflows/**" + +# Set a workflow level concurrency group so primary workflow +# can wait for this to complete if needed +# DO NOT CHANGE without updating main workflow group +concurrency: + group: build-installer-library + cancel-in-progress: false + +jobs: + prepare-docker-build: + name: Prepare Docker Image Version Data + runs-on: ubuntu-20.04 + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.9" + - + name: Setup qpdf image + id: qpdf-setup + run: | + build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf) + + echo ${build_json} + + echo ::set-output name=qpdf-json::${build_json} + - + name: Setup psycopg2 image + id: psycopg2-setup + run: | + build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2) + + echo ${build_json} + + echo ::set-output name=psycopg2-json::${build_json} + - + name: Setup pikepdf image + id: pikepdf-setup + run: | + build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf) + + echo ${build_json} + + echo ::set-output name=pikepdf-json::${build_json} + - + name: Setup jbig2enc image + id: jbig2enc-setup + run: | + build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc) + + echo ${build_json} + + echo ::set-output name=jbig2enc-json::${build_json} + + outputs: + + qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }} + + pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }} + + psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }} + + jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}} + + build-qpdf-debs: + name: qpdf + needs: + - prepare-docker-build + uses: ./.github/workflows/reusable-workflow-builder.yml + with: + dockerfile: ./docker-builders/Dockerfile.qpdf + build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }} + build-args: | + QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} + + build-jbig2enc: + name: jbig2enc + needs: + - prepare-docker-build + uses: ./.github/workflows/reusable-workflow-builder.yml + with: + dockerfile: ./docker-builders/Dockerfile.jbig2enc + build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }} + build-args: | + JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }} + + build-psycopg2-wheel: + name: psycopg2 + needs: + - prepare-docker-build + uses: ./.github/workflows/reusable-workflow-builder.yml + with: + dockerfile: ./docker-builders/Dockerfile.psycopg2 + build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }} + build-args: | + PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }} + PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} + + build-pikepdf-wheel: + name: pikepdf + needs: + - prepare-docker-build + - build-qpdf-debs + uses: ./.github/workflows/reusable-workflow-builder.yml + with: + dockerfile: ./docker-builders/Dockerfile.pikepdf + build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }} + build-args: | + REPO=${{ github.repository }} + QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} + PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }} + PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} From 029d81122b474ea2aaf73ab93df356111ffc80f3 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Sun, 15 May 2022 11:25:14 -0700 Subject: [PATCH 2/2] Makes the workflow triggers more specific --- .github/workflows/installer-library.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/installer-library.yml b/.github/workflows/installer-library.yml index 31e08638e..77a66e5d2 100644 --- a/.github/workflows/installer-library.yml +++ b/.github/workflows/installer-library.yml @@ -19,8 +19,10 @@ on: # Trigger if a package was updated - ".build-config.json" - "Pipfile.lock" - # Also trigger on workflow changes (such as this file) - - ".github/workflows/**" + # Also trigger on workflow changes related to the library + - ".github/workflows/installer-library.yml" + - ".github/workflows/reusable-workflow-builder.yml" + - ".github/scripts/**" # Set a workflow level concurrency group so primary workflow # can wait for this to complete if needed