Incorporates the base image building back into the main repo with multi stage building

This commit is contained in:
Trenton Holmes
2022-04-18 08:18:20 -07:00
parent a14d95f6c6
commit d744be97f4
14 changed files with 886 additions and 151 deletions

View File

@@ -0,0 +1,13 @@
# This Dockerfile compiles the frontend
# Inputs: None
FROM node:16-bullseye-slim AS compile-frontend
COPY . /src
WORKDIR /src/src-ui
RUN set -eux \
&& npm update npm -g \
&& npm ci --no-optional
RUN set -eux \
&& ./node_modules/.bin/ng build --configuration production

View File

@@ -0,0 +1,39 @@
# This Dockerfile compiles the jbig2enc library
# Inputs:
# - JBIG2ENC_VERSION - the Git tag to checkout and build
FROM debian:bullseye-slim
LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
ARG DEBIAN_FRONTEND=noninteractive
ARG BUILD_PACKAGES="\
build-essential \
automake \
libtool \
libleptonica-dev \
zlib1g-dev \
git \
ca-certificates"
WORKDIR /usr/src/jbig2enc
# As this is an base image for a multi-stage final image
# the added size of the install is basically irrelevant
RUN apt-get update --quiet \
&& apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
&& rm -rf /var/lib/apt/lists/*
# Layers after this point change according to required version
# For better caching, seperate the basic installs from
# the building
ARG JBIG2ENC_VERSION
RUN set -eux \
&& git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
RUN set -eux \
&& ./autogen.sh
RUN set -eux \
&& ./configure && make

View File

@@ -0,0 +1,65 @@
# This Dockerfile builds the pikepdf wheel
# Inputs:
# - QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from
# - GIT_TAG - The Git tag to clone and build from
# - VERSION - Used to force the built pikepdf version to match
ARG QPDF_BASE_IMAGE
FROM ${QPDF_BASE_IMAGE} as qpdf-builder
# This does nothing, except provide a name for a copy below
FROM python:3.9-slim-bullseye
LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
ARG DEBIAN_FRONTEND=noninteractive
ARG BUILD_PACKAGES="\
build-essential \
git \
libjpeg62-turbo-dev \
zlib1g-dev \
libgnutls28-dev \
libxml2-dev \
libxslt1-dev \
python3-dev \
python3-pip"
WORKDIR /usr/src
COPY --from=qpdf-builder /usr/src/qpdf/*.deb .
# As this is an base image for a multi-stage final image
# the added size of the install is basically irrelevant
RUN set -eux \
&& apt-get update --quiet \
&& apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
&& dpkg --install libqpdf28_*.deb \
&& dpkg --install libqpdf-dev_*.deb \
&& python3 -m pip install --no-cache-dir --upgrade pip wheel pybind11 \
&& rm -rf /var/lib/apt/lists/*
# Layers after this point change according to required version
# For better caching, seperate the basic installs from
# the building
ARG GIT_TAG
ARG VERSION
RUN set -eux \
&& echo "building pikepdf wheel" \
# Note the v in the tag name here
&& git clone --quiet --depth 1 --branch "${GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
&& cd pikepdf \
# pikepdf seems to specifciy either a next version when built OR
# a post release tag.
# In either case, this won't match what we want from requirements.txt
# Directly modify the setup.py to set the version we just checked out of Git
&& sed -i "s/use_scm_version=True/version=\"${VERSION}\"/g" setup.py \
# https://github.com/pikepdf/pikepdf/issues/323
&& rm pyproject.toml \
&& mkdir wheels \
&& python3 -m pip wheel . --wheel-dir wheels \
&& ls -ahl wheels

View File

@@ -0,0 +1,44 @@
# This Dockerfile builds the psycopg2 wheel
# Inputs:
# - GIT_TAG - The Git tag to clone and build from
# - VERSION - Unused, kept for future possible usage
FROM python:3.9-slim-bullseye
LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
ARG DEBIAN_FRONTEND=noninteractive
ARG BUILD_PACKAGES="\
build-essential \
git \
libpq-dev \
python3-dev \
python3-pip"
WORKDIR /usr/src
# As this is an base image for a multi-stage final image
# the added size of the install is basically irrelevant
RUN set -eux \
&& apt-get update --quiet \
&& apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
&& rm -rf /var/lib/apt/lists/* \
&& python3 -m pip install --upgrade pip wheel
# Layers after this point change according to required version
# For better caching, seperate the basic installs from
# the building
ARG GIT_TAG
ARG VERSION
RUN set -eux \
&& echo "Building psycopg2 wheel" \
&& cd /usr/src \
&& git clone --quiet --depth 1 --branch ${GIT_TAG} https://github.com/psycopg/psycopg2.git \
&& cd psycopg2 \
&& mkdir wheels \
&& python3 -m pip wheel . --wheel-dir wheels \
&& ls -ahl wheels/

View File

@@ -0,0 +1,51 @@
FROM debian:bullseye-slim
LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
ARG DEBIAN_FRONTEND=noninteractive
ARG BUILD_PACKAGES="\
build-essential \
debhelper \
debian-keyring \
devscripts \
equivs \
libtool \
libjpeg62-turbo-dev \
libgnutls28-dev \
packaging-dev \
zlib1g-dev"
WORKDIR /usr/src
# As this is an base image for a multi-stage final image
# the added size of the install is basically irrelevant
RUN set -eux \
&& apt-get update --quiet \
&& apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
&& rm -rf /var/lib/apt/lists/*
# Layers after this point change according to required version
# For better caching, seperate the basic installs from
# the building
# This must match to pikepdf's minimum at least
ARG QPDF_VERSION
# In order to get the required version of qpdf, it is backported from bookwork
# and then built from source
RUN set -eux \
&& echo "Building qpdf" \
&& echo "deb-src http://deb.debian.org/debian/ bookworm main" | tee /etc/apt/sources.list.d/bookworm-src.list \
&& apt-get update \
&& mkdir qpdf \
&& cd qpdf \
&& apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
&& rm -rf /var/lib/apt/lists/* \
&& cd qpdf-$QPDF_VERSION \
&& DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
&& export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
&& dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
&& pwd \
&& ls -ahl ../*.deb

112
docker-builders/get-build-json.py Executable file
View File

@@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""
This is a helper script to either parse the JSON of the Pipfile.lock
or otherwise return a JSON object detailing versioning and image tags
for the packages we build seperately, then copy into the final Docker image
"""
import argparse
import json
import os
from pathlib import Path
from typing import Final
CONFIG: Final = {
# All packages need to be in the dict, even if not configured further
# as it is used for the possible choices in the argument
"psycopg2": {},
# Most information about Python packages comes from the Pipfile.lock
"pikepdf": {
"qpdf_version": "10.6.3",
},
# For other packages, it is directly configured, for now
# These require manual updates to this file for version updates
"qpdf": {
"version": "10.6.3",
"git_tag": "N/A",
},
"jbig2enc": {
"version": "0.29",
"git_tag": "0.29",
},
}
def _get_image_tag(
repo_name: str,
pkg_name: str,
pkg_version: str,
) -> str:
return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
def _main():
parser = argparse.ArgumentParser(
description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
)
parser.add_argument(
"package",
help="The name of the package to generate JSON for",
choices=CONFIG.keys(),
)
args = parser.parse_args()
pip_lock = Path("Pipfile.lock")
repo_name = os.environ["GITHUB_REPOSITORY"]
# The JSON object we'll output
output = {"name": args.package}
# Read Pipfile.lock file
pipfile_data = json.loads(pip_lock.read_text())
# Read the version from Pipfile.lock
if args.package in pipfile_data["default"]:
pkg_data = pipfile_data["default"][args.package]
pkg_version = pkg_data["version"].split("==")[-1]
output["version"] = pkg_version
# Based on the package, generate the expected Git tag name
if args.package == "pikepdf":
git_tag_name = f"v{pkg_version}"
elif args.package == "psycopg2":
git_tag_name = pkg_version.replace(".", "_")
output["git_tag"] = git_tag_name
# Based on the package and environment, generate the Docker image tag
image_tag = _get_image_tag(repo_name, args.package, pkg_version)
output["image_tag"] = image_tag
# Check for any special configuration, based on package
if args.package in CONFIG:
output.update(CONFIG[args.package])
elif args.package in CONFIG:
# This is not a Python package
output.update(CONFIG[args.package])
output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
else:
raise NotImplementedError(args.package)
# Output the JSON info to stdout
print(json.dumps(output, indent=2))
if __name__ == "__main__":
_main()