paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-10-30 03:56:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Instead of using a full image name, use the repo and version to build the image to pull from. Removes building of the frontend for multiple platforms

Browse Source
This commit is contained in:
Trenton Holmes
2022-04-21 15:47:50 -07:00
committed by Trenton Holmes
parent 6179ca5668
commit d19015579c
4 changed files with 67 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
.github/workflows/ci.yml vendored
View File

@@ -63,10 +63,6 @@ jobs:
-
name: Checkout
uses: actions/checkout@v3
-
name: Get branch name
id: branch-name
uses: tj-actions/branch-names@v5
-
name: Login to Github Container Registry
uses: docker/login-action@v1
@@ -123,16 +119,14 @@ jobs:
name: Setup frontend image
id: frontend-setup
run: |
frontend_image=ghcr.io/${{ github.repository }}/ngx-frontend:${{ steps.branch-name.outputs.current_branch }}
build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py frontend)
echo ${frontend_image}
echo ${build_json}
echo ::set-output name=frontend-image-tag::${frontend_image}
echo ::set-output name=frontend-json::${build_json}
outputs:
frontend-image-tag: ${{ steps.frontend-setup.outputs.frontend-image-tag }}
qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
@@ -141,6 +135,8 @@ jobs:
jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
frontend-json: ${{ steps.frontend-setup.outputs.frontend-json}}
build-qpdf-debs:
name: qpdf
needs:
@@ -185,14 +181,15 @@ jobs:
dockerfile: ./docker-builders/Dockerfile.pikepdf
build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
build-args: |
QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
REPO=${{ github.repository }}
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
build-frontend:
name: Compile frontend
concurrency:
group: ${{ github.workflow }}-build-frontend-${{ github.ref }}
group: ${{ github.workflow }}-build-frontend-${{ github.ref_name }}
cancel-in-progress: false
needs:
- prepare-docker-build
@@ -222,7 +219,7 @@ jobs:
id: build-skip-check
# Skip building the frontend if the tag exists and no src-ui files changed
run: |
if ! docker manifest inspect ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} &> /dev/null ; then
if ! docker manifest inspect ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} &> /dev/null ; then
echo "Build required, no existing image"
echo ::set-output name=frontend-build-needed::true
elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then
@@ -247,15 +244,18 @@ jobs:
with:
context: .
file: ./docker-builders/Dockerfile.frontend
tags: ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
platforms: linux/amd64,linux/arm64,linux/arm/v7
tags: ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
# The compilation is identical between different platforms
# The buildx and QEMU setup is left, just in case that ever changes
# But the platform is set to the runner's native for speedup
platforms: linux/amd64
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
-
name: Export frontend artifact from docker
run: |
docker create --name frontend-extract ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
docker create --name frontend-extract ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/
-
name: Upload frontend artifact
@@ -271,7 +271,7 @@ jobs:
cancel-in-progress: true
runs-on: ubuntu-20.04
concurrency:
group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
cancel-in-progress: true
needs:
- prepare-docker-build
@@ -317,11 +317,12 @@ jobs:
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}
build-args: |
JBIG2ENC_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
PIKEPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
PSYCOPG2_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
FRONTEND_BASE_IMAGE=${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
REPO=${{ github.repository }}
JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
FRONTEND_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).version }}
cache-from: type=gha
cache-to: type=gha,mode=max
-
@@ -401,7 +402,7 @@ jobs:
runs-on: ubuntu-20.04
needs:
- build-release
if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'ngx-') || startsWith(github.ref_name, 'beta-'))
steps:
-
name: Download release artifact

23
Dockerfile
View File

@@ -1,16 +1,19 @@
# Default to pulling from the main repo registry when manually building
ARG REPO="paperless-ngx/paperless-ngx"
# These are all built previously in the pipeline
# They provide either a .deb, .whl or whatever npm outputs
ARG JBIG2ENC_BASE_IMAGE
ARG QPDF_BASE_IMAGE
ARG PIKEPDF_BASE_IMAGE
ARG PSYCOPG2_BASE_IMAGE
ARG FRONTEND_BASE_IMAGE
ARG JBIG2ENC_VERSION
ARG QPDF_VERSION
ARG PIKEPDF_VERSION
ARG PSYCOPG2_VERSION
ARG FRONTEND_VERSION
FROM ${JBIG2ENC_BASE_IMAGE} AS jbig2enc-builder
FROM ${QPDF_BASE_IMAGE} as qpdf-builder
FROM ${PIKEPDF_BASE_IMAGE} as pikepdf-builder
FROM ${PSYCOPG2_BASE_IMAGE} as psycopg2-builder
FROM ${FRONTEND_BASE_IMAGE} as compile-frontend
FROM ghcr.io/${REPO}/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
FROM ghcr.io/${REPO}/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
FROM ghcr.io/${REPO}/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
FROM ghcr.io/${REPO}/builder/frontend:${FRONTEND_VERSION} as compile-frontend
FROM python:3.9-slim-bullseye as main-app

10
docker-builders/Dockerfile.pikepdf
View File

@@ -1,11 +1,15 @@
# This Dockerfile builds the pikepdf wheel
# Inputs:
# - QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from
# - REPO - Docker repository to pull qpdf from
# - QPDF_VERSION - The image qpdf version to copy .deb files from
# - GIT_TAG - The Git tag to clone and build from
# - VERSION - Used to force the built pikepdf version to match
ARG QPDF_BASE_IMAGE
FROM ${QPDF_BASE_IMAGE} as qpdf-builder
# Default to pulling from the main repo registry when manually building
ARG REPO="paperless-ngx/paperless-ngx"
ARG QPDF_VERSION
FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
# This does nothing, except provide a name for a copy below

52
docker-builders/get-build-json.py
View File

@@ -1,8 +1,18 @@
#!/usr/bin/env python3
"""
This is a helper script to either parse the JSON of the Pipfile.lock
or otherwise return a JSON object detailing versioning and image tags
for the packages we build seperately, then copy into the final Docker image
This is a helper script for the mutli-stage Docker image builder.
It provides a single point of configuration for package version control.
The output JSON object is used by the CI workflow to determine what versions
to build and pull into the final Docker image.
Python package information is obtained from the Pipfile.lock. As this is
kept updated by dependabot, it usually will need no further configuration.
The sole exception currently is pikepdf, which has a dependency on qpdf,
and is configured here to use the latest version of qpdf built by the workflow.
Other package version information is configured directly below, generally by
setting the version and Git information, if any.
"""
import argparse
import json
@@ -14,11 +24,13 @@ CONFIG: Final = {
# All packages need to be in the dict, even if not configured further
# as it is used for the possible choices in the argument
"psycopg2": {},
"frontend": {},
# Most information about Python packages comes from the Pipfile.lock
# Excpetion being pikepdf, which needs a specific qpdf
"pikepdf": {
"qpdf_version": "10.6.3",
},
# For other packages, it is directly configured, for now
# For other packages, version and Git information are directly configured
# These require manual updates to this file for version updates
"qpdf": {
"version": "10.6.3",
@@ -59,11 +71,9 @@ def _main():
output = {"name": args.package}
# Read Pipfile.lock file
pipfile_data = json.loads(pip_lock.read_text())
# Read the version from Pipfile.lock
if args.package in pipfile_data["default"]:
pkg_data = pipfile_data["default"][args.package]
@@ -73,7 +83,6 @@ def _main():
output["version"] = pkg_version
# Based on the package, generate the expected Git tag name
if args.package == "pikepdf":
git_tag_name = f"v{pkg_version}"
elif args.package == "psycopg2":
@@ -81,31 +90,18 @@ def _main():
output["git_tag"] = git_tag_name
# Based on the package and environment, generate the Docker image tag
# Use the basic ref name, minus refs/heads or refs/tags for frontend builder image
elif args.package == "frontend":
output["version"] = os.environ["GITHUB_REF_NAME"]
image_tag = _get_image_tag(repo_name, args.package, pkg_version)
# Add anything special from the config
output.update(CONFIG[args.package])
output["image_tag"] = image_tag
# Check for any special configuration, based on package
if args.package in CONFIG:
output.update(CONFIG[args.package])
elif args.package in CONFIG:
# This is not a Python package
output.update(CONFIG[args.package])
output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
else:
raise NotImplementedError(args.package)
# Based on the package and environment, generate the Docker image tag
output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
# Output the JSON info to stdout
print(json.dumps(output, indent=2))
print(json.dumps(output))
if __name__ == "__main__":