Incorporates the base image building back into the main repo with multi stage building

.github/workflows/ci.yml

@@ -45,73 +45,158 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  code-checks-backend:
+  ci-backend:
-    name: "Backend Code Checks"
+    uses: ./.github/workflows/reusable-ci-backend.yml
+
+  ci-frontend:
+    uses: ./.github/workflows/reusable-ci-frontend.yml
+
+  prepare-docker-build:
+    name: Prepare Docker Pipeline Data
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
     runs-on: ubuntu-20.04
+    needs:
+      - documentation
+      - ci-backend
+      - ci-frontend
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
       -
-        name: Install checkers
+        name: Get branch name
-        run: |
+        id: branch-name
-          pipx install reorder-python-imports
+        uses: tj-actions/branch-names@v5
-          pipx install yesqa
-          pipx install add-trailing-comma
-          pipx install flake8
       -
-        name: Run reorder-python-imports
+        name: Login to Github Container Registry
-        run: |
+        uses: docker/login-action@v1
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-      -
-        name: Run yesqa
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-      -
-        name: Run add-trailing-comma
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
-      # black is placed after add-trailing-comma because it may format differently
-      # if a trailing comma is added
-      -
-        name: Run black
-        uses: psf/black@stable
         with:
-          options: "--check --diff"
+          registry: ghcr.io
-          version: "22.3.0"
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
-        name: Run flake8 checks
+        name: Set up Python
-        run: |
+        uses: actions/setup-python@v3
-          cd src/
-          flake8 --max-line-length=88 --ignore=E203,W503
-
-  code-checks-frontend:
-    name: "Frontend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          python-version: "3.9"
       -
-        name: Install prettier
+        name: Make script executable
         run: |
-          npm install prettier
+          chmod +x ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py
       -
-        name: Run prettier
+        name: Setup qpdf image
-        run:
+        id: qpdf-setup
-          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py qpdf)
 
-  tests-backend:
+          echo ${build_json}
-    needs: [code-checks-backend]
+
-    name: "Backend Tests (${{ matrix.python-version }})"
+          echo ::set-output name=qpdf-json::${build_json}
-    runs-on: ubuntu-20.04
+      -
-    strategy:
+        name: Setup psycopg2 image
-      matrix:
+        id: psycopg2-setup
-        python-version: ['3.8', '3.9', '3.10']
+        run: |
-      fail-fast: false
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo ::set-output name=psycopg2-json::${build_json}
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=pikepdf-json::${build_json}
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo ::set-output name=jbig2enc-json::${build_json}
+      -
+        name: Setup frontend image
+        id: frontend-setup
+        run: |
+          frontend_image=ghcr.io/${{ github.repository }}/ngx-frontend:${{ steps.branch-name.outputs.current_branch }}
+
+          echo ${frontend_image}
+
+          echo ::set-output name=frontend-image-tag::${frontend_image}
+
+    outputs:
+
+      frontend-image-tag: ${{ steps.frontend-setup.outputs.frontend-image-tag }}
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+
+  build-qpdf-debs:
+    name: qpdf
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+      build-args: |
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+  build-jbig2enc:
+    name: jbig2enc
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.jbig2enc
+      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+      build-args: |
+        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+  build-psycopg2-wheel:
+    name: psycopg2
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.psycopg2
+      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+      build-args: |
+        GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
+        VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+  build-pikepdf-wheel:
+    name: pikepdf
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.pikepdf
+      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+      build-args: |
+        QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+        GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
+        VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+
+  build-frontend:
+    name: Compile frontend
+    concurrency:
+      group: ${{ github.workflow }}-build-frontend-${{ github.ref }}
+      cancel-in-progress: false
+    needs:
+      - prepare-docker-build
+    runs-on: ubuntu-latest
     steps:
       -
         name: Checkout
@@ -119,77 +204,82 @@ jobs:
         with:
           fetch-depth: 2
       -
-        name: Install pipenv
+        name: Get changed frontend files
-        run: pipx install pipenv
-      -
-        name: Set up Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
-      -
-        name: Install system dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
-      -
-        name: Install Python dependencies
-        run: |
-          pipenv sync --dev
-      -
-        name: Tests
-        run: |
-          cd src/
-          pipenv run pytest
-      -
-        name: Get changed files
         id: changed-files-specific
         uses: tj-actions/changed-files@v18.1
         with:
           files: |
-            src/**
+            src-ui/**
       -
-        name: List all changed files
+        name: Login to Github Container Registry
-        run: |
+        uses: docker/login-action@v1
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github
-
-  tests-frontend:
-    needs: [code-checks-frontend]
-    name: "Frontend Tests"
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
         with:
-          node-version: ${{ matrix.node-version }}
+          registry: ghcr.io
-      - run: cd src-ui && npm ci
+          username: ${{ github.actor }}
-      - run: cd src-ui && npm run test
+          password: ${{ secrets.GITHUB_TOKEN }}
-      - run: cd src-ui && npm run e2e:ci
+      -
+        name: Determine if build needed
+        id: build-skip-check
+        # Skip building the frontend if the tag exists and no src-ui files changed
+        run: |
+          if ! docker manifest inspect ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} &> /dev/null ; then
+            echo "Build required, no existing image"
+            echo ::set-output name=frontend-build-needed::true
+          elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then
+            echo "Build required, src-ui changes"
+            echo ::set-output name=frontend-build-needed::true
+          else
+            echo "No build required"
+            echo ::set-output name=frontend-build-needed::false
+          fi
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
+      -
+        name: Compile frontend
+        uses: docker/build-push-action@v2
+        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
+        with:
+          context: .
+          file: ./docker-builders/Dockerfile.frontend
+          tags: ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      -
+        name: Export frontend artifact from docker
+        run: |
+          docker create --name frontend-extract ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+          docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/
+      -
+        name: Upload frontend artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: frontend-compiled
+          path: src/documents/static/frontend/
 
   # build and push image to docker hub.
   build-docker-image:
-    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
     concurrency:
       group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
       cancel-in-progress: true
     runs-on: ubuntu-20.04
-    needs: [tests-backend, tests-frontend]
+    concurrency:
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
+      cancel-in-progress: true
+    needs:
+      - prepare-docker-build
+      - build-psycopg2-wheel
+      - build-jbig2enc
+      - build-qpdf-debs
+      - build-pikepdf-wheel
+      - build-frontend
     steps:
       -
         name: Gather Docker metadata
@@ -226,26 +316,22 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
+          build-args: |
+            JBIG2ENC_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
+            QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+            PIKEPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
+            PSYCOPG2_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
+            FRONTEND_BASE_IMAGE=${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
       -
         name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
-      -
-        name: Export frontend artifact from docker
-        run: |
-          docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
-          docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
-      -
-        name: Upload frontend artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: frontend-compiled
-          path: src/documents/static/frontend/
 
   build-release:
-    needs: [build-docker-image, documentation]
+    needs:
+      - build-docker-image
     runs-on: ubuntu-20.04
     steps:
       -
@@ -313,7 +399,8 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
-    needs: build-release
+    needs:
+      - build-release
     if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
     steps:
       -

.github/workflows/reusable-ci-backend.yml

@@ -0,0 +1,108 @@
+name: Backend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-backend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Install checkers
+        run: |
+          pipx install reorder-python-imports
+          pipx install yesqa
+          pipx install add-trailing-comma
+          pipx install flake8
+      -
+        name: Run reorder-python-imports
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
+      -
+        name: Run yesqa
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
+      -
+        name: Run add-trailing-comma
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
+      # black is placed after add-trailing-comma because it may format differently
+      # if a trailing comma is added
+      -
+        name: Run black
+        uses: psf/black@stable
+        with:
+          options: "--check --diff"
+          version: "22.3.0"
+      -
+        name: Run flake8 checks
+        run: |
+          cd src/
+          flake8 --max-line-length=88 --ignore=E203,W503
+
+  tests-backend:
+    name: "Tests (${{ matrix.python-version }})"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-backend
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+      fail-fast: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      -
+        name: Install pipenv
+        run: pipx install pipenv
+      -
+        name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Tests
+        run: |
+          cd src/
+          pipenv run pytest
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v18.1
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          pipenv run coveralls --service=github

.github/workflows/reusable-ci-frontend.yml

@@ -0,0 +1,42 @@
+name: Frontend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-frontend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      -
+        name: Install prettier
+        run: |
+          npm install prettier
+      -
+        name: Run prettier
+        run:
+          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  tests-frontend:
+    name: "Tests"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-frontend
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci

.github/workflows/reusable-workflow-builder.yml

@@ -0,0 +1,68 @@
+name: Reusable Image Builder
+
+on:
+  workflow_call:
+    inputs:
+      dockerfile:
+        required: true
+        type: string
+      build-json:
+        required: true
+        type: string
+      build-args:
+        required: false
+        default: ""
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
+  cancel-in-progress: false
+
+jobs:
+  build-image:
+    name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Determine if build needed
+        id: build-skip-check
+        run: |
+          if ! docker manifest inspect ${{ fromJSON(inputs.build-json).image_tag }} &> /dev/null ; then
+            echo "Building, no image exists with this version"
+            echo ::set-output name=image-exists::false
+          else
+            echo "Not building, image exists with this version"
+            echo ::set-output name=image-exists::true
+          fi
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+      -
+        name: Build ${{ fromJSON(inputs.build-json).name }}
+        uses: docker/build-push-action@v2
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          tags: ${{ fromJSON(inputs.build-json).image_tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          build-args: ${{ inputs.build-args }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.pre-commit-config.yaml

@@ -63,10 +63,17 @@ repos:
     hooks:
       - id: black
   # Dockerfile hooks
-  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
+  - repo: https://github.com/AleksaC/hadolint-py
-    rev: "v0.1.0"
+    rev: v1.19.0
     hooks:
-      - id: dockerfilelint
+      - id: hadolint
+        args:
+          - --ignore
+          - DL3006      # https://github.com/hadolint/hadolint/wiki/DL3006  (doesn't understand FROM with ARG)
+          - --ignore
+          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
+          - --ignore
+          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1

Dockerfile

@@ -1,12 +1,18 @@
-FROM node:16 AS compile-frontend
+# These are all built previously in the pipeline
+# They provide either a .deb, .whl or whatever npm outputs
+ARG JBIG2ENC_BASE_IMAGE
+ARG QPDF_BASE_IMAGE
+ARG PIKEPDF_BASE_IMAGE
+ARG PSYCOPG2_BASE_IMAGE
+ARG FRONTEND_BASE_IMAGE
 
-COPY . /src
+FROM ${JBIG2ENC_BASE_IMAGE} AS jbig2enc-builder
+FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+FROM ${PIKEPDF_BASE_IMAGE} as pikepdf-builder
+FROM ${PSYCOPG2_BASE_IMAGE} as psycopg2-builder
+FROM ${FRONTEND_BASE_IMAGE} as compile-frontend
 
-WORKDIR /src/src-ui
+FROM python:3.9-slim-bullseye as main-app
-RUN npm update npm -g && npm ci --no-optional
-RUN ./node_modules/.bin/ng build --configuration production
-
-FROM ghcr.io/paperless-ngx/builder/ngx-base:1.7.0 as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
 LABEL org.opencontainers.image.documentation="https://paperless-ngx.readthedocs.io/en/latest/"
@@ -14,27 +20,115 @@ LABEL org.opencontainers.image.source="https://github.com/paperless-ngx/paperles
 LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-ngx"
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Packages needed only for building
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  python3-dev"
+
+# Packages need for running
+ARG RUNTIME_PACKAGES="\
+  curl \
+  file \
+  # fonts for text file thumbnail generation
+  fonts-liberation \
+  gettext \
+  ghostscript \
+  gnupg \
+  gosu \
+  icc-profiles-free \
+  imagemagick \
+  media-types \
+  liblept5 \
+  libpq5 \
+  libxml2 \
+  libxslt1.1 \
+  libgnutls30 \
+  libjpeg62-turbo \
+  optipng \
+  python3 \
+  python3-pip \
+  python3-setuptools \
+  postgresql-client \
+  # For Numpy
+  libatlas3-base \
+  # thumbnail size reduction
+  pngquant \
+  # OCRmyPDF dependencies
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-deu \
+  tesseract-ocr-fra \
+  tesseract-ocr-ita \
+  tesseract-ocr-spa \
+  tzdata \
+  unpaper \
+  # Mime type detection
+  zlib1g \
+  # Barcode splitter
+  libzbar0 \
+  poppler-utils"
+
 WORKDIR /usr/src/paperless/src/
 
+# Copy qpdf and runtime library
+COPY --from=qpdf-builder /usr/src/qpdf/libqpdf28_*.deb .
+COPY --from=qpdf-builder /usr/src/qpdf/qpdf_*.deb .
+
+# Copy pikepdf wheel and dependencies
+COPY --from=pikepdf-builder /usr/src/pikepdf/wheels/*.whl .
+
+# Copy psycopg2 wheel
+COPY --from=psycopg2-builder /usr/src/psycopg2/wheels/psycopg2*.whl .
+
+# copy jbig2enc
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
+
 COPY requirements.txt ../
 
 # Python dependencies
-RUN apt-get update \
+RUN set -eux \
-  # python-Levenshtein still needs to be compiled here
+  && apt-get update \
-  && apt-get -y --no-install-recommends install \
+  && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} ${BUILD_PACKAGES} \
-    build-essential \
+  && python3 -m pip install --no-cache-dir --upgrade wheel \
-    && python3 -m pip install --upgrade --no-cache-dir pip wheel \
+  && echo "Installing qpdf" \
-  && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
+    && apt-get install --yes --no-install-recommends ./libqpdf28_*.deb \
-  && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+    && apt-get install --yes --no-install-recommends ./qpdf_*.deb \
-  && apt-get -y purge build-essential \
+  && echo "Installing pikepdf and dependencies wheel" \
-  && apt-get -y autoremove --purge \
+    && python3 -m pip install --no-cache-dir packaging*.whl \
-  && rm -rf /var/lib/apt/lists/*
+    && python3 -m pip install --no-cache-dir lxml*.whl \
+    && python3 -m pip install --no-cache-dir Pillow*.whl \
+    && python3 -m pip install --no-cache-dir pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir pikepdf*.whl \
+    && python -m pip list \
+  && echo "Installing psycopg2 wheel" \
+    && python3 -m pip install --no-cache-dir psycopg2*.whl \
+    && python -m pip list \
+  && echo "Installing supervisor" \
+    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
+  && echo "Installing Python requirements" \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && apt-get clean --yes \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* \
+    && rm -rf /var/tmp/* \
+    && rm -rf /var/cache/apt/archives/* \
+    && truncate -s 0 /var/log/*log
 
 # setup docker-specific things
 COPY docker/ ./docker/
 
-RUN cd docker \
+WORKDIR /usr/src/paperless/src/docker/
-    && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+
+RUN set -eux \
+  && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
   && mkdir /var/log/supervisord /var/run/supervisord \
   && cp supervisord.conf /etc/supervisord.conf \
   && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
@@ -42,17 +136,18 @@ RUN cd docker \
   && cp docker-prepare.sh /sbin/docker-prepare.sh \
   && chmod 755 /sbin/docker-prepare.sh \
   && chmod +x install_management_commands.sh \
-  && ./install_management_commands.sh \
+  && ./install_management_commands.sh
-  && cd .. \
-  && rm -rf docker/
 
-COPY gunicorn.conf.py ../
+WORKDIR /usr/src/paperless/
+
+COPY gunicorn.conf.py .
 
 # copy app
 COPY --from=compile-frontend /src/src/ ./
 
 # add users, setup scripts
-RUN addgroup --gid 1000 paperless \
+RUN set -eux \
+  && addgroup --gid 1000 paperless \
   && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
   && chown -R paperless:paperless ../ \
   && gosu paperless python3 manage.py collectstatic --clear --no-input \

docker-builders/Dockerfile.frontend

@@ -0,0 +1,13 @@
+# This Dockerfile compiles the frontend
+# Inputs: None
+
+FROM node:16-bullseye-slim AS compile-frontend
+
+COPY . /src
+
+WORKDIR /src/src-ui
+RUN set -eux \
+	&& npm update npm -g \
+	&& npm ci --no-optional
+RUN set -eux \
+	&& ./node_modules/.bin/ng build --configuration production

docker-builders/Dockerfile.jbig2enc

@@ -0,0 +1,39 @@
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+#		- JBIG2ENC_VERSION - the Git tag to checkout and build
+
+FROM debian:bullseye-slim
+
+LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  automake \
+  libtool \
+  libleptonica-dev \
+  zlib1g-dev \
+  git \
+  ca-certificates"
+
+WORKDIR /usr/src/jbig2enc
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+RUN apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG JBIG2ENC_VERSION
+
+RUN set -eux \
+	&& git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
+RUN set -eux \
+	&& ./autogen.sh
+RUN set -eux \
+	&& ./configure && make

docker-builders/Dockerfile.pikepdf

@@ -0,0 +1,65 @@
+# This Dockerfile builds the pikepdf wheel
+# Inputs:
+#		- QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from
+#		- GIT_TAG - The Git tag to clone and build from
+#		- VERSION - Used to force the built pikepdf version to match
+
+ARG QPDF_BASE_IMAGE
+FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+
+# This does nothing, except provide a name for a copy below
+
+FROM python:3.9-slim-bullseye
+
+LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  libjpeg62-turbo-dev \
+  zlib1g-dev \
+  libgnutls28-dev \
+  libxml2-dev \
+  libxslt1-dev \
+  python3-dev \
+  python3-pip"
+
+WORKDIR /usr/src
+
+COPY --from=qpdf-builder /usr/src/qpdf/*.deb .
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && dpkg --install libqpdf28_*.deb \
+  && dpkg --install libqpdf-dev_*.deb \
+  && python3 -m pip install --no-cache-dir --upgrade pip wheel pybind11 \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG GIT_TAG
+ARG VERSION
+
+RUN set -eux \
+	&& echo "building pikepdf wheel" \
+  # Note the v in the tag name here
+  && git clone --quiet --depth 1 --branch "${GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
+  && cd pikepdf \
+  # pikepdf seems to specifciy either a next version when built OR
+  # a post release tag.
+  # In either case, this won't match what we want from requirements.txt
+  # Directly modify the setup.py to set the version we just checked out of Git
+  && sed -i "s/use_scm_version=True/version=\"${VERSION}\"/g" setup.py \
+  # https://github.com/pikepdf/pikepdf/issues/323
+  && rm pyproject.toml \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels

docker-builders/Dockerfile.psycopg2

@@ -0,0 +1,44 @@
+# This Dockerfile builds the psycopg2 wheel
+# Inputs:
+#		- GIT_TAG - The Git tag to clone and build from
+#		- VERSION - Unused, kept for future possible usage
+
+FROM python:3.9-slim-bullseye
+
+LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  libpq-dev \
+  python3-dev \
+  python3-pip"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/* \
+  && python3 -m pip install --upgrade pip wheel
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG GIT_TAG
+ARG VERSION
+
+RUN set -eux \
+	&& echo "Building psycopg2 wheel" \
+  && cd /usr/src \
+  && git clone --quiet --depth 1 --branch ${GIT_TAG} https://github.com/psycopg/psycopg2.git \
+  && cd psycopg2 \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels/

docker-builders/Dockerfile.qpdf

@@ -0,0 +1,51 @@
+FROM debian:bullseye-slim
+
+LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  debhelper \
+  debian-keyring \
+  devscripts \
+  equivs  \
+  libtool \
+  libjpeg62-turbo-dev \
+  libgnutls28-dev \
+  packaging-dev \
+  zlib1g-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
+
+# In order to get the required version of qpdf, it is backported from bookwork
+# and then built from source
+RUN set -eux \
+	&& echo "Building qpdf" \
+  && echo "deb-src http://deb.debian.org/debian/ bookworm main" | tee /etc/apt/sources.list.d/bookworm-src.list \
+  && apt-get update \
+  && mkdir qpdf \
+  && cd qpdf \
+  && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
+  && rm -rf /var/lib/apt/lists/* \
+  && cd qpdf-$QPDF_VERSION \
+  && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
+	&& export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+  && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
+  && pwd \
+  && ls -ahl ../*.deb

docker-builders/get-build-json.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+This is a helper script to either parse the JSON of the Pipfile.lock
+or otherwise return a JSON object detailing versioning and image tags
+for the packages we build seperately, then copy into the final Docker image
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Final
+
+CONFIG: Final = {
+    # All packages need to be in the dict, even if not configured further
+    # as it is used for the possible choices in the argument
+    "psycopg2": {},
+    # Most information about Python packages comes from the Pipfile.lock
+    "pikepdf": {
+        "qpdf_version": "10.6.3",
+    },
+    # For other packages, it is directly configured, for now
+    # These require manual updates to this file for version updates
+    "qpdf": {
+        "version": "10.6.3",
+        "git_tag": "N/A",
+    },
+    "jbig2enc": {
+        "version": "0.29",
+        "git_tag": "0.29",
+    },
+}
+
+
+def _get_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+) -> str:
+    return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+
+
+def _main():
+    parser = argparse.ArgumentParser(
+        description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
+    )
+    parser.add_argument(
+        "package",
+        help="The name of the package to generate JSON for",
+        choices=CONFIG.keys(),
+    )
+
+    args = parser.parse_args()
+
+    pip_lock = Path("Pipfile.lock")
+
+    repo_name = os.environ["GITHUB_REPOSITORY"]
+
+    # The JSON object we'll output
+    output = {"name": args.package}
+
+    # Read Pipfile.lock file
+
+    pipfile_data = json.loads(pip_lock.read_text())
+
+    # Read the version from Pipfile.lock
+
+    if args.package in pipfile_data["default"]:
+
+        pkg_data = pipfile_data["default"][args.package]
+
+        pkg_version = pkg_data["version"].split("==")[-1]
+
+        output["version"] = pkg_version
+
+        # Based on the package, generate the expected Git tag name
+
+        if args.package == "pikepdf":
+            git_tag_name = f"v{pkg_version}"
+        elif args.package == "psycopg2":
+            git_tag_name = pkg_version.replace(".", "_")
+
+        output["git_tag"] = git_tag_name
+
+        # Based on the package and environment, generate the Docker image tag
+
+        image_tag = _get_image_tag(repo_name, args.package, pkg_version)
+
+        output["image_tag"] = image_tag
+
+        # Check for any special configuration, based on package
+
+        if args.package in CONFIG:
+            output.update(CONFIG[args.package])
+
+    elif args.package in CONFIG:
+
+        # This is not a Python package
+
+        output.update(CONFIG[args.package])
+
+        output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
+
+    else:
+        raise NotImplementedError(args.package)
+
+    # Output the JSON info to stdout
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    _main()

docker/docker-prepare.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 wait_for_postgres() {
 	attempt_num=1
 	max_attempts=5

docker/install_management_commands.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -eu
+
 for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
 do
 	echo "installing $command..."