CI cleanup and improvements.

Removes the building of installers from the repo, they can now be built elsewhere,
on demand, as their building is no longer tied to the Dockerfile
This commit is contained in:
Trenton H
2023-04-27 09:05:28 -07:00
parent 01fd400ec7
commit f875ae4abf
17 changed files with 112 additions and 1223 deletions

View File

@@ -16,7 +16,7 @@ on:
env:
# This is the version of pipenv all the steps will use
# If changing this, change Dockerfile
DEFAULT_PIP_ENV_VERSION: "2023.3.20"
DEFAULT_PIP_ENV_VERSION: "2023.4.20"
# This is the default version of Python to use in most steps
# If changing this, change Dockerfile
DEFAULT_PYTHON_VERSION: "3.9"
@@ -197,90 +197,20 @@ jobs:
- run: cd src-ui && npm run test
- run: cd src-ui && npm run e2e:ci
prepare-docker-build:
name: Prepare Docker Pipeline Data
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
runs-on: ubuntu-22.04
needs:
- documentation
- tests-backend
- tests-frontend
steps:
-
name: Set ghcr repository name
id: set-ghcr-repository
run: |
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
-
name: Setup qpdf image
id: qpdf-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
echo ${build_json}
echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup psycopg2 image
id: psycopg2-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
echo ${build_json}
echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup pikepdf image
id: pikepdf-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
echo ${build_json}
echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup jbig2enc image
id: jbig2enc-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
echo ${build_json}
echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
outputs:
ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
# build and push image to docker hub.
build-docker-image:
name: Build Docker image for ${{ github.ref_name }}
runs-on: ubuntu-22.04
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
concurrency:
group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
cancel-in-progress: true
needs:
- prepare-docker-build
- tests-backend
- tests-frontend
steps:
-
name: Check pushing to Docker Hub
id: docker-hub
id: push-other-places
# Only push to Dockerhub from the main repo AND the ref is either:
# main
# dev
@@ -288,22 +218,29 @@ jobs:
# a tag
# Otherwise forks would require a Docker Hub account and secrets setup
run: |
if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then
echo "Enabling DockerHub image push"
echo "enable=true" >> $GITHUB_OUTPUT
else
echo "Not pushing to DockerHub"
echo "enable=false" >> $GITHUB_OUTPUT
fi
-
name: Set ghcr repository name
id: set-ghcr-repository
run: |
ghcr_name=$(echo "${{ github.repository }}" | awk '{ print tolower($0) }')
echo "Name is ${ghcr_name}"
echo "ghcr-repository=${ghcr_name}" >> $GITHUB_OUTPUT
-
name: Gather Docker metadata
id: docker-meta
uses: docker/metadata-action@v4
with:
images: |
ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}
name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }}
ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}
name=paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }}
tags: |
# Tag branches with branch name
type=ref,event=branch
@@ -314,8 +251,11 @@ jobs:
-
name: Checkout
uses: actions/checkout@v3
# If https://github.com/docker/buildx/issues/1044 is resolved,
# the append input with a native arm64 arch could be used to
# significantly speed up building
-
name: Set up Docker Buildx
name: Set up Docker Buildx (fork/failure)
uses: docker/setup-buildx-action@v2
-
name: Set up QEMU
@@ -331,15 +271,15 @@ jobs:
name: Login to Docker Hub
uses: docker/login-action@v2
# Don't attempt to login is not pushing to Docker Hub
if: steps.docker-hub.outputs.enable == 'true'
if: steps.push-other-places.outputs.enable == 'true'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
-
name: Login to Quay.io
uses: docker/login-action@v2
# Don't attempt to login is not pushing to Docker Hub
if: steps.docker-hub.outputs.enable == 'true'
# Don't attempt to login is not pushing to Quay.io
if: steps.push-other-places.outputs.enable == 'true'
with:
registry: quay.io
username: ${{ secrets.QUAY_USERNAME }}
@@ -354,19 +294,13 @@ jobs:
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}
build-args: |
JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
# Get cache layers from this branch, then dev, then main
# This allows new branches to get at least some cache benefits, generally from dev
cache-from: |
type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev
type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:dev
cache-to: |
type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
type=registry,mode=max,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }}
-
name: Inspect image
run: |

View File

@@ -59,6 +59,15 @@ jobs:
- primary-name: "paperless-ngx/builder/cache/jbig2enc"
- primary-name: "paperless-ngx/builder/psycopg2"
- primary-name: "paperless-ngx/builder/cache/psycopg2"
# TODO: Remove the above and replace with the below
# - primary-name: "builder/qpdf"
# - primary-name: "builder/cache/qpdf"
# - primary-name: "builder/pikepdf"
# - primary-name: "builder/cache/pikepdf"
# - primary-name: "builder/jbig2enc"
# - primary-name: "builder/cache/jbig2enc"
# - primary-name: "builder/psycopg2"
# - primary-name: "builder/cache/psycopg2"
env:
# Requires a personal access token with the OAuth scope delete:packages
TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }}

View File

@@ -1,310 +0,0 @@
# This workflow will run to update the installer library of
# Docker images. These are the images which provide updated wheels
# .deb installation packages or maybe just some compiled library
name: Build Image Library
on:
push:
# Must match one of these branches AND one of the paths
# to be triggered
branches:
- "main"
- "dev"
- "library-*"
- "feature-*"
paths:
# Trigger the workflow if a Dockerfile changed
- "docker-builders/**"
# Trigger if a package was updated
- ".build-config.json"
- "Pipfile.lock"
# Also trigger on workflow changes related to the library
- ".github/workflows/installer-library.yml"
- ".github/workflows/reusable-workflow-builder.yml"
- ".github/scripts/**"
# Set a workflow level concurrency group so primary workflow
# can wait for this to complete if needed
# DO NOT CHANGE without updating main workflow group
concurrency:
group: build-installer-library
cancel-in-progress: false
jobs:
prepare-docker-build:
name: Prepare Docker Image Version Data
runs-on: ubuntu-22.04
steps:
-
name: Set ghcr repository name
id: set-ghcr-repository
run: |
ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }')
echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
-
name: Install jq
run: |
sudo apt-get update
sudo apt-get install jq
-
name: Setup qpdf image
id: qpdf-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf)
echo ${build_json}
echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup psycopg2 image
id: psycopg2-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2)
echo ${build_json}
echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup pikepdf image
id: pikepdf-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf)
echo ${build_json}
echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup jbig2enc image
id: jbig2enc-setup
run: |
build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc)
echo ${build_json}
echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT
-
name: Setup other versions
id: cache-bust-setup
run: |
pillow_version=$(jq -r '.default.pillow.version | gsub("=";"")' Pipfile.lock)
lxml_version=$(jq -r '.default.lxml.version | gsub("=";"")' Pipfile.lock)
echo "Pillow is ${pillow_version}"
echo "lxml is ${lxml_version}"
echo "pillow-version=${pillow_version}" >> $GITHUB_OUTPUT
echo "lxml-version=${lxml_version}" >> $GITHUB_OUTPUT
outputs:
ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }}
qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json }}
pillow-version: ${{ steps.cache-bust-setup.outputs.pillow-version }}
lxml-version: ${{ steps.cache-bust-setup.outputs.lxml-version }}
build-qpdf-debs:
name: qpdf
needs:
- prepare-docker-build
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.qpdf
build-platforms: linux/amd64
build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
build-args: |
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
build-jbig2enc:
name: jbig2enc
needs:
- prepare-docker-build
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.jbig2enc
build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
build-args: |
JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
build-psycopg2-wheel:
name: psycopg2
needs:
- prepare-docker-build
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.psycopg2
build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
build-args: |
PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
build-pikepdf-wheel:
name: pikepdf
needs:
- prepare-docker-build
- build-qpdf-debs
uses: ./.github/workflows/reusable-workflow-builder.yml
with:
dockerfile: ./docker-builders/Dockerfile.pikepdf
build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
build-args: |
REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }}
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}
commit-binary-files:
name: Store installers
needs:
- prepare-docker-build
- build-qpdf-debs
- build-jbig2enc
- build-psycopg2-wheel
- build-pikepdf-wheel
runs-on: ubuntu-22.04
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
ref: binary-library
-
name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
-
name: Install system dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -qq --no-install-recommends tree
-
name: Extract qpdf files
run: |
version=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
tag=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
docker pull --quiet ${tag}
docker create --name qpdf-extract ${tag}
mkdir --parents qpdf/${version}/amd64
docker cp qpdf-extract:/usr/src/qpdf/${version}/amd64 qpdf/${version}
mkdir --parents qpdf/${version}/arm64
docker cp qpdf-extract:/usr/src/qpdf/${version}/arm64 qpdf/${version}
mkdir --parents qpdf/${version}/armv7
docker cp qpdf-extract:/usr/src/qpdf/${version}/armv7 qpdf/${version}
-
name: Extract psycopg2 files
run: |
version=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
tag=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
docker pull --quiet --platform linux/amd64 ${tag}
docker create --platform linux/amd64 --name psycopg2-extract ${tag}
mkdir --parents psycopg2/${version}/amd64
docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/amd64
mv psycopg2/${version}/amd64/wheels/* psycopg2/${version}/amd64
rm -r psycopg2/${version}/amd64/wheels/
docker rm psycopg2-extract
docker pull --quiet --platform linux/arm64 ${tag}
docker create --platform linux/arm64 --name psycopg2-extract ${tag}
mkdir --parents psycopg2/${version}/arm64
docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/arm64
mv psycopg2/${version}/arm64/wheels/* psycopg2/${version}/arm64
rm -r psycopg2/${version}/arm64/wheels/
docker rm psycopg2-extract
docker pull --quiet --platform linux/arm/v7 ${tag}
docker create --platform linux/arm/v7 --name psycopg2-extract ${tag}
mkdir --parents psycopg2/${version}/armv7
docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/armv7
mv psycopg2/${version}/armv7/wheels/* psycopg2/${version}/armv7
rm -r psycopg2/${version}/armv7/wheels/
docker rm psycopg2-extract
-
name: Extract pikepdf files
run: |
version=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
tag=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
docker pull --quiet --platform linux/amd64 ${tag}
docker create --platform linux/amd64 --name pikepdf-extract ${tag}
mkdir --parents pikepdf/${version}/amd64
docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/amd64
mv pikepdf/${version}/amd64/wheels/* pikepdf/${version}/amd64
rm -r pikepdf/${version}/amd64/wheels/
docker rm pikepdf-extract
docker pull --quiet --platform linux/arm64 ${tag}
docker create --platform linux/arm64 --name pikepdf-extract ${tag}
mkdir --parents pikepdf/${version}/arm64
docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/arm64
mv pikepdf/${version}/arm64/wheels/* pikepdf/${version}/arm64
rm -r pikepdf/${version}/arm64/wheels/
docker rm pikepdf-extract
docker pull --quiet --platform linux/arm/v7 ${tag}
docker create --platform linux/arm/v7 --name pikepdf-extract ${tag}
mkdir --parents pikepdf/${version}/armv7
docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/armv7
mv pikepdf/${version}/armv7/wheels/* pikepdf/${version}/armv7
rm -r pikepdf/${version}/armv7/wheels/
docker rm pikepdf-extract
-
name: Extract jbig2enc files
run: |
version=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
tag=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
docker pull --quiet --platform linux/amd64 ${tag}
docker create --platform linux/amd64 --name jbig2enc-extract ${tag}
mkdir --parents jbig2enc/${version}/amd64
docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/amd64/
mv jbig2enc/${version}/amd64/build/* jbig2enc/${version}/amd64/
docker rm jbig2enc-extract
docker pull --quiet --platform linux/arm64 ${tag}
docker create --platform linux/arm64 --name jbig2enc-extract ${tag}
mkdir --parents jbig2enc/${version}/arm64
docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/arm64
mv jbig2enc/${version}/arm64/build/* jbig2enc/${version}/arm64/
docker rm jbig2enc-extract
docker pull --quiet --platform linux/arm/v7 ${tag}
docker create --platform linux/arm/v7 --name jbig2enc-extract ${tag}
mkdir --parents jbig2enc/${version}/armv7
docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/armv7
mv jbig2enc/${version}/armv7/build/* jbig2enc/${version}/armv7/
docker rm jbig2enc-extract
-
name: Show file structure
run: |
tree .
-
name: Commit files
run: |
git config --global user.name "github-actions"
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git add pikepdf/ qpdf/ psycopg2/ jbig2enc/
git commit -m "Updating installer packages" || true
git push origin || true

View File

@@ -1,57 +0,0 @@
name: Reusable Image Builder
on:
workflow_call:
inputs:
dockerfile:
required: true
type: string
build-json:
required: true
type: string
build-args:
required: false
default: ""
type: string
build-platforms:
required: false
default: linux/amd64,linux/arm64,linux/arm/v7
type: string
concurrency:
group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
cancel-in-progress: false
jobs:
build-image:
name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
runs-on: ubuntu-22.04
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Login to Github Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Build ${{ fromJSON(inputs.build-json).name }}
uses: docker/build-push-action@v4
with:
context: .
file: ${{ inputs.dockerfile }}
tags: ${{ fromJSON(inputs.build-json).image_tag }}
platforms: ${{ inputs.build-platforms }}
build-args: ${{ inputs.build-args }}
push: true
cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }}
cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }}