mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-02 13:45:10 -05:00
Updates the installer library to be static in the final image, saving the installers into Git and curl-ing the correct revision
This commit is contained in:
Trenton H
parent
dadd7472fd
commit
f1204d2749
6
.github/workflows/ci.yml
vendored
6
.github/workflows/ci.yml
vendored
@ -212,12 +212,6 @@ jobs:
|
||||
name: Prepare Docker Pipeline Data
|
||||
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v'))
|
||||
runs-on: ubuntu-22.04
|
||||
# If the push triggered the installer library workflow, wait for it to
|
||||
# complete here. This ensures the required versions for the final
|
||||
# image have been built, while not waiting at all if the versions haven't changed
|
||||
concurrency:
|
||||
group: build-installer-library
|
||||
cancel-in-progress: false
|
||||
needs:
|
||||
- documentation
|
||||
- tests-backend
|
||||
|
||||
139
.github/workflows/installer-library.yml
vendored
139
.github/workflows/installer-library.yml
vendored
@ -169,3 +169,142 @@ jobs:
|
||||
PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
|
||||
PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }}
|
||||
LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }}
|
||||
|
||||
commit-binary-files:
|
||||
name: Store installers
|
||||
needs:
|
||||
- prepare-docker-build
|
||||
- build-qpdf-debs
|
||||
- build-jbig2enc
|
||||
- build-psycopg2-wheel
|
||||
- build-pikepdf-wheel
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
ref: binary-library
|
||||
-
|
||||
name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.9"
|
||||
-
|
||||
name: Install system dependencies
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -qq --no-install-recommends tree
|
||||
-
|
||||
name: Extract qpdf files
|
||||
run: |
|
||||
version=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
|
||||
tag=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
|
||||
|
||||
docker pull --quiet ${tag}
|
||||
docker create --name qpdf-extract ${tag}
|
||||
|
||||
mkdir --parents qpdf/${version}/amd64
|
||||
docker cp qpdf-extract:/usr/src/qpdf/${version}/amd64 qpdf/${version}
|
||||
|
||||
mkdir --parents qpdf/${version}/arm64
|
||||
docker cp qpdf-extract:/usr/src/qpdf/${version}/arm64 qpdf/${version}
|
||||
|
||||
mkdir --parents qpdf/${version}/armv7
|
||||
docker cp qpdf-extract:/usr/src/qpdf/${version}/armv7 qpdf/${version}
|
||||
-
|
||||
name: Extract psycopg2 files
|
||||
run: |
|
||||
version=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
|
||||
tag=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
|
||||
|
||||
docker pull --quiet --platform linux/amd64 ${tag}
|
||||
docker create --platform linux/amd64 --name psycopg2-extract ${tag}
|
||||
mkdir --parents psycopg2/${version}/amd64
|
||||
docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/amd64
|
||||
mv psycopg2/${version}/amd64/wheels/* psycopg2/${version}/amd64
|
||||
rm -r psycopg2/${version}/amd64/wheels/
|
||||
docker rm psycopg2-extract
|
||||
|
||||
docker pull --quiet --platform linux/arm64 ${tag}
|
||||
docker create --platform linux/arm64 --name psycopg2-extract ${tag}
|
||||
mkdir --parents psycopg2/${version}/arm64
|
||||
docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/arm64
|
||||
mv psycopg2/${version}/arm64/wheels/* psycopg2/${version}/arm64
|
||||
rm -r psycopg2/${version}/arm64/wheels/
|
||||
docker rm psycopg2-extract
|
||||
|
||||
docker pull --quiet --platform linux/arm/v7 ${tag}
|
||||
docker create --platform linux/arm/v7 --name psycopg2-extract ${tag}
|
||||
mkdir --parents psycopg2/${version}/armv7
|
||||
docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/armv7
|
||||
mv psycopg2/${version}/armv7/wheels/* psycopg2/${version}/armv7
|
||||
rm -r psycopg2/${version}/armv7/wheels/
|
||||
docker rm psycopg2-extract
|
||||
-
|
||||
name: Extract pikepdf files
|
||||
run: |
|
||||
version=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
|
||||
tag=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
|
||||
|
||||
docker pull --quiet --platform linux/amd64 ${tag}
|
||||
docker create --platform linux/amd64 --name pikepdf-extract ${tag}
|
||||
mkdir --parents pikepdf/${version}/amd64
|
||||
docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/amd64
|
||||
mv pikepdf/${version}/amd64/wheels/* pikepdf/${version}/amd64
|
||||
rm -r pikepdf/${version}/amd64/wheels/
|
||||
docker rm pikepdf-extract
|
||||
|
||||
docker pull --quiet --platform linux/arm64 ${tag}
|
||||
docker create --platform linux/arm64 --name pikepdf-extract ${tag}
|
||||
mkdir --parents pikepdf/${version}/arm64
|
||||
docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/arm64
|
||||
mv pikepdf/${version}/arm64/wheels/* pikepdf/${version}/arm64
|
||||
rm -r pikepdf/${version}/arm64/wheels/
|
||||
docker rm pikepdf-extract
|
||||
|
||||
docker pull --quiet --platform linux/arm/v7 ${tag}
|
||||
docker create --platform linux/arm/v7 --name pikepdf-extract ${tag}
|
||||
mkdir --parents pikepdf/${version}/armv7
|
||||
docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/armv7
|
||||
mv pikepdf/${version}/armv7/wheels/* pikepdf/${version}/armv7
|
||||
rm -r pikepdf/${version}/armv7/wheels/
|
||||
docker rm pikepdf-extract
|
||||
-
|
||||
name: Extract jbig2enc files
|
||||
run: |
|
||||
version=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
|
||||
tag=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
|
||||
|
||||
docker pull --quiet --platform linux/amd64 ${tag}
|
||||
docker create --platform linux/amd64 --name jbig2enc-extract ${tag}
|
||||
mkdir --parents jbig2enc/${version}/amd64
|
||||
docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/amd64/
|
||||
mv jbig2enc/${version}/amd64/build/* jbig2enc/${version}/amd64/
|
||||
docker rm jbig2enc-extract
|
||||
|
||||
docker pull --quiet --platform linux/arm64 ${tag}
|
||||
docker create --platform linux/arm64 --name jbig2enc-extract ${tag}
|
||||
mkdir --parents jbig2enc/${version}/arm64
|
||||
docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/arm64
|
||||
mv jbig2enc/${version}/arm64/build/* jbig2enc/${version}/arm64/
|
||||
docker rm jbig2enc-extract
|
||||
|
||||
docker pull --quiet --platform linux/arm/v7 ${tag}
|
||||
docker create --platform linux/arm/v7 --name jbig2enc-extract ${tag}
|
||||
mkdir --parents jbig2enc/${version}/armv7
|
||||
docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/armv7
|
||||
mv jbig2enc/${version}/armv7/build/* jbig2enc/${version}/armv7/
|
||||
docker rm jbig2enc-extract
|
||||
-
|
||||
name: Show file structure
|
||||
run: |
|
||||
tree .
|
||||
-
|
||||
name: Commit files
|
||||
run: |
|
||||
git config --global user.name "github-actions"
|
||||
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add pikepdf/ qpdf/ psycopg2/ jbig2enc/
|
||||
git commit -m "Updating installer packages" || true
|
||||
git push origin || true
|
||||
|
||||
53
Dockerfile
53
Dockerfile
@ -1,19 +1,5 @@
|
||||
# syntax=docker/dockerfile:1.4
|
||||
|
||||
# Pull the installer images from the library
|
||||
# These are all built previously
|
||||
# They provide either a .deb or .whl
|
||||
|
||||
ARG JBIG2ENC_VERSION
|
||||
ARG QPDF_VERSION
|
||||
ARG PIKEPDF_VERSION
|
||||
ARG PSYCOPG2_VERSION
|
||||
|
||||
FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
|
||||
FROM --platform=$BUILDPLATFORM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
|
||||
FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
|
||||
FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
|
||||
|
||||
FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
|
||||
|
||||
# This stage compiles the frontend
|
||||
@ -58,24 +44,21 @@ LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-n
|
||||
LABEL org.opencontainers.image.licenses="GPL-3.0-only"
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
# Buildx provided
|
||||
# Buildx provided, must be defined to use though
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
|
||||
# Workflow provided
|
||||
ARG JBIG2ENC_VERSION
|
||||
ARG QPDF_VERSION
|
||||
ARG PIKEPDF_VERSION
|
||||
ARG PSYCOPG2_VERSION
|
||||
|
||||
#
|
||||
# Begin installation and configuration
|
||||
# Order the steps below from least often changed to most
|
||||
#
|
||||
|
||||
# copy jbig2enc
|
||||
# Basically will never change again
|
||||
COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
|
||||
COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
|
||||
COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
|
||||
|
||||
# Packages need for running
|
||||
ARG RUNTIME_PACKAGES="\
|
||||
# Python
|
||||
@ -198,19 +181,29 @@ RUN set -eux \
|
||||
# Install the built packages from the installer library images
|
||||
# Use mounts to avoid copying installer files into the image
|
||||
# These change sometimes
|
||||
RUN --mount=type=bind,from=qpdf-builder,target=/qpdf \
|
||||
--mount=type=bind,from=psycopg2-builder,target=/psycopg2 \
|
||||
--mount=type=bind,from=pikepdf-builder,target=/pikepdf \
|
||||
set -eux \
|
||||
RUN set -eux \
|
||||
&& echo "Getting binaries" \
|
||||
&& mkdir paperless-ngx \
|
||||
&& curl --fail --silent --show-error --output paperless-ngx.tar.gz --location https://github.com/paperless-ngx/paperless-ngx/archive/41d6e7e407af09a0882736d50c89b6e015997bff.tar.gz \
|
||||
&& tar -xf paperless-ngx.tar.gz --directory paperless-ngx --strip-components=1 \
|
||||
&& cd paperless-ngx \
|
||||
# Setting a specific revision ensures we know what this installed
|
||||
# and ensures cache breaking on changes
|
||||
&& echo "Installing jbig2enc" \
|
||||
&& cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/jbig2 /usr/local/bin/ \
|
||||
&& cp ./jbig2enc/${JBIG2ENC_VERSION}/${TARGETARCH}${TARGETVARIANT}/libjbig2enc* /usr/local/lib/ \
|
||||
&& echo "Installing qpdf" \
|
||||
&& apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
|
||||
&& apt-get install --yes --no-install-recommends /qpdf/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
|
||||
&& apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/libqpdf29_*.deb \
|
||||
&& apt-get install --yes --no-install-recommends ./qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/qpdf_*.deb \
|
||||
&& echo "Installing pikepdf and dependencies" \
|
||||
&& python3 -m pip install --no-cache-dir /pikepdf/usr/src/wheels/*.whl \
|
||||
&& python3 -m pip install --no-cache-dir ./pikepdf/${PIKEPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.whl \
|
||||
&& python3 -m pip list \
|
||||
&& echo "Installing psycopg2" \
|
||||
&& python3 -m pip install --no-cache-dir /psycopg2/usr/src/wheels/psycopg2*.whl \
|
||||
&& python3 -m pip list
|
||||
&& python3 -m pip install --no-cache-dir ./psycopg2/${PSYCOPG2_VERSION}/${TARGETARCH}${TARGETVARIANT}/psycopg2*.whl \
|
||||
&& python3 -m pip list \
|
||||
&& echo "Cleaning up image layer" \
|
||||
&& cd ../ \
|
||||
&& rm -rf paperless-ngx
|
||||
|
||||
WORKDIR /usr/src/paperless/src/
|
||||
|
||||
|
||||
@ -29,7 +29,20 @@ RUN set -eux \
|
||||
&& ./autogen.sh \
|
||||
&& ./configure \
|
||||
&& make \
|
||||
&& echo "Gathering package data" \
|
||||
&& dpkg-query -f '${Package;-40}${Version}\n' -W > ./pkg-list.txt \
|
||||
&& echo "Cleaning up image" \
|
||||
&& apt-get -y purge ${BUILD_PACKAGES} \
|
||||
&& apt-get -y autoremove --purge \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& echo "Moving files around" \
|
||||
&& mkdir build \
|
||||
# Unlink a symlink that causes problems
|
||||
&& unlink ./src/.libs/libjbig2enc.la \
|
||||
# Move what the link pointed to
|
||||
&& mv ./src/libjbig2enc.la ./build/ \
|
||||
# Move the shared library .so files
|
||||
&& mv ./src/.libs/libjbig2enc* ./build/ \
|
||||
# And move the cli binary
|
||||
&& mv ./src/jbig2 ./build/ \
|
||||
&& mv ./pkg-list.txt ./build/
|
||||
|
||||
@ -7,12 +7,17 @@
|
||||
# Default to pulling from the main repo registry when manually building
|
||||
ARG REPO="paperless-ngx/paperless-ngx"
|
||||
|
||||
# This does nothing, except provide a name for a copy below
|
||||
ARG QPDF_VERSION
|
||||
FROM --platform=$BUILDPLATFORM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
|
||||
|
||||
# This does nothing, except provide a name for a copy below
|
||||
|
||||
FROM python:3.9-slim-bullseye as main
|
||||
#
|
||||
# Stage: builder
|
||||
# Purpose:
|
||||
# - Build the pikepdf wheel
|
||||
# - Build any dependent wheels which can't be found
|
||||
#
|
||||
FROM python:3.9-slim-bullseye as builder
|
||||
|
||||
LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
|
||||
|
||||
@ -100,3 +105,14 @@ RUN set -eux \
|
||||
&& apt-get -y purge ${BUILD_PACKAGES} \
|
||||
&& apt-get -y autoremove --purge \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
#
|
||||
# Stage: package
|
||||
# Purpose: Holds the compiled .whl files in a tiny image to pull
|
||||
#
|
||||
FROM alpine:3.17 as package
|
||||
|
||||
WORKDIR /usr/src/wheels/
|
||||
|
||||
COPY --from=builder /usr/src/wheels/*.whl ./
|
||||
COPY --from=builder /usr/src/wheels/pkg-list.txt ./
|
||||
|
||||
@ -2,7 +2,12 @@
|
||||
# Inputs:
|
||||
# - PSYCOPG2_VERSION - Version to build
|
||||
|
||||
FROM python:3.9-slim-bullseye as main
|
||||
#
|
||||
# Stage: builder
|
||||
# Purpose:
|
||||
# - Build the psycopg2 wheel
|
||||
#
|
||||
FROM python:3.9-slim-bullseye as builder
|
||||
|
||||
LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
|
||||
|
||||
@ -48,3 +53,14 @@ RUN set -eux \
|
||||
&& apt-get -y purge ${BUILD_PACKAGES} \
|
||||
&& apt-get -y autoremove --purge \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
#
|
||||
# Stage: package
|
||||
# Purpose: Holds the compiled .whl files in a tiny image to pull
|
||||
#
|
||||
FROM alpine:3.17 as package
|
||||
|
||||
WORKDIR /usr/src/wheels/
|
||||
|
||||
COPY --from=builder /usr/src/wheels/*.whl ./
|
||||
COPY --from=builder /usr/src/wheels/pkg-list.txt ./
|
||||
|
||||
51
docker-builders/README.md
Normal file
51
docker-builders/README.md
Normal file
@ -0,0 +1,51 @@
|
||||
# Installer Library
|
||||
|
||||
This folder contains the Dockerfiles for building certain installers or libraries, which are then pulled into the main image.
|
||||
|
||||
## [jbig2enc](https://github.com/agl/jbig2enc)
|
||||
|
||||
### Why
|
||||
|
||||
JBIG is an image coding which can achieve better compression of images for PDFs.
|
||||
|
||||
### What
|
||||
|
||||
The Docker image builds a shared library file and utility, which is copied into the correct location in the final image.
|
||||
|
||||
See Also:
|
||||
|
||||
- [OCRMyPDF Documentation](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html)
|
||||
|
||||
## [psycopg2](https://www.psycopg.org/)
|
||||
|
||||
### Why
|
||||
|
||||
The pre-built wheels of psycopg2 are built on Debian 9, which provides a quite old version of libpq-dev. This causes issue with authentication methods.
|
||||
|
||||
### What
|
||||
|
||||
The image builds psycopg2 wheels on Debian 10 and places the produced wheels into `/usr/src/wheels/`.
|
||||
|
||||
See Also:
|
||||
|
||||
- [Issue 266](https://github.com/paperless-ngx/paperless-ngx/issues/266)
|
||||
|
||||
## [qpdf](https://qpdf.readthedocs.io/en/stable/index.html)
|
||||
|
||||
### Why
|
||||
|
||||
qpdf and it's library provide tools to read, manipulate and fix up PDFs. Version 11 is also required by `pikepdf` 6+ and Debian 9 does not provide above version 10.
|
||||
|
||||
### What
|
||||
|
||||
The Docker image cross compiles .deb installers for each supported architecture of the main image. The installers are placed in `/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/`
|
||||
|
||||
## [pikepdf](https://pikepdf.readthedocs.io/en/latest/)
|
||||
|
||||
### Why
|
||||
|
||||
Required by OCRMyPdf, this is a general purpose library for PDF manipulation in Python via the qpdf libraries.
|
||||
|
||||
### What
|
||||
|
||||
The built wheels are placed into `/usr/src/wheels/`
|
||||
Loading…
x
Reference in New Issue
Block a user