From 6c70db31bd78d166f06ace734761824239aeb3e4 Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Mon, 18 Apr 2022 08:18:20 -0700
Subject: [PATCH 01/11] Incorporates the base image building back into the main
 repo with multi stage building

---
 .github/workflows/ci.yml                      | 335 +++++++++++-------
 .github/workflows/reusable-ci-backend.yml     | 108 ++++++
 .github/workflows/reusable-ci-frontend.yml    |  42 +++
 .../workflows/reusable-workflow-builder.yml   |  68 ++++
 .pre-commit-config.yaml                       |  13 +-
 Dockerfile                                    | 143 ++++++--
 docker-builders/Dockerfile.frontend           |  13 +
 docker-builders/Dockerfile.jbig2enc           |  39 ++
 docker-builders/Dockerfile.pikepdf            |  65 ++++
 docker-builders/Dockerfile.psycopg2           |  44 +++
 docker-builders/Dockerfile.qpdf               |  51 +++
 docker-builders/get-build-json.py             | 112 ++++++
 docker/docker-prepare.sh                      |   2 +
 docker/install_management_commands.sh         |   2 +
 14 files changed, 886 insertions(+), 151 deletions(-)
 create mode 100644 .github/workflows/reusable-ci-backend.yml
 create mode 100644 .github/workflows/reusable-ci-frontend.yml
 create mode 100644 .github/workflows/reusable-workflow-builder.yml
 create mode 100644 docker-builders/Dockerfile.frontend
 create mode 100644 docker-builders/Dockerfile.jbig2enc
 create mode 100644 docker-builders/Dockerfile.pikepdf
 create mode 100644 docker-builders/Dockerfile.psycopg2
 create mode 100644 docker-builders/Dockerfile.qpdf
 create mode 100755 docker-builders/get-build-json.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 10b81bf79..e9ed4306c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -45,73 +45,158 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  code-checks-backend:
-    name: "Backend Code Checks"
+  ci-backend:
+    uses: ./.github/workflows/reusable-ci-backend.yml
+
+  ci-frontend:
+    uses: ./.github/workflows/reusable-ci-frontend.yml
+
+  prepare-docker-build:
+    name: Prepare Docker Pipeline Data
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
     runs-on: ubuntu-20.04
+    needs:
+      - documentation
+      - ci-backend
+      - ci-frontend
     steps:
       -
         name: Checkout
         uses: actions/checkout@v3
       -
-        name: Install checkers
-        run: |
-          pipx install reorder-python-imports
-          pipx install yesqa
-          pipx install add-trailing-comma
-          pipx install flake8
+        name: Get branch name
+        id: branch-name
+        uses: tj-actions/branch-names@v5
       -
-        name: Run reorder-python-imports
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
-      -
-        name: Run yesqa
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
-      -
-        name: Run add-trailing-comma
-        run: |
-          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
-      # black is placed after add-trailing-comma because it may format differently
-      # if a trailing comma is added
-      -
-        name: Run black
-        uses: psf/black@stable
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
         with:
-          options: "--check --diff"
-          version: "22.3.0"
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
-        name: Run flake8 checks
-        run: |
-          cd src/
-          flake8 --max-line-length=88 --ignore=E203,W503
-
-  code-checks-frontend:
-    name: "Frontend Code Checks"
-    runs-on: ubuntu-20.04
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+        name: Set up Python
+        uses: actions/setup-python@v3
         with:
-          node-version: '16'
+          python-version: "3.9"
       -
-        name: Install prettier
+        name: Make script executable
         run: |
-          npm install prettier
+          chmod +x ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py
       -
-        name: Run prettier
-        run:
-          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+        name: Setup qpdf image
+        id: qpdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py qpdf)
 
-  tests-backend:
-    needs: [code-checks-backend]
-    name: "Backend Tests (${{ matrix.python-version }})"
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: ['3.8', '3.9', '3.10']
-      fail-fast: false
+          echo ${build_json}
+
+          echo ::set-output name=qpdf-json::${build_json}
+      -
+        name: Setup psycopg2 image
+        id: psycopg2-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py psycopg2)
+
+          echo ${build_json}
+
+          echo ::set-output name=psycopg2-json::${build_json}
+      -
+        name: Setup pikepdf image
+        id: pikepdf-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py pikepdf)
+
+          echo ${build_json}
+
+          echo ::set-output name=pikepdf-json::${build_json}
+      -
+        name: Setup jbig2enc image
+        id: jbig2enc-setup
+        run: |
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py jbig2enc)
+
+          echo ${build_json}
+
+          echo ::set-output name=jbig2enc-json::${build_json}
+      -
+        name: Setup frontend image
+        id: frontend-setup
+        run: |
+          frontend_image=ghcr.io/${{ github.repository }}/ngx-frontend:${{ steps.branch-name.outputs.current_branch }}
+
+          echo ${frontend_image}
+
+          echo ::set-output name=frontend-image-tag::${frontend_image}
+
+    outputs:
+
+      frontend-image-tag: ${{ steps.frontend-setup.outputs.frontend-image-tag }}
+
+      qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+      pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+      psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+      jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+
+  build-qpdf-debs:
+    name: qpdf
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.qpdf
+      build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+      build-args: |
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+  build-jbig2enc:
+    name: jbig2enc
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.jbig2enc
+      build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+      build-args: |
+        JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+  build-psycopg2-wheel:
+    name: psycopg2
+    needs:
+      - prepare-docker-build
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.psycopg2
+      build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+      build-args: |
+        GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
+        VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+  build-pikepdf-wheel:
+    name: pikepdf
+    needs:
+      - prepare-docker-build
+      - build-qpdf-debs
+    uses: ./.github/workflows/reusable-workflow-builder.yml
+    with:
+      dockerfile: ./docker-builders/Dockerfile.pikepdf
+      build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+      build-args: |
+        QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+        GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
+        VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+
+  build-frontend:
+    name: Compile frontend
+    concurrency:
+      group: ${{ github.workflow }}-build-frontend-${{ github.ref }}
+      cancel-in-progress: false
+    needs:
+      - prepare-docker-build
+    runs-on: ubuntu-latest
     steps:
       -
         name: Checkout
@@ -119,77 +204,82 @@ jobs:
         with:
           fetch-depth: 2
       -
-        name: Install pipenv
-        run: pipx install pipenv
-      -
-        name: Set up Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: "${{ matrix.python-version }}"
-          cache: "pipenv"
-          cache-dependency-path: 'Pipfile.lock'
-      -
-        name: Install system dependencies
-        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
-      -
-        name: Install Python dependencies
-        run: |
-          pipenv sync --dev
-      -
-        name: Tests
-        run: |
-          cd src/
-          pipenv run pytest
-      -
-        name: Get changed files
+        name: Get changed frontend files
         id: changed-files-specific
         uses: tj-actions/changed-files@v18.1
         with:
           files: |
-            src/**
+            src-ui/**
       -
-        name: List all changed files
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
-            echo "${file} was changed"
-          done
-      -
-        name: Publish coverage results
-        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        # https://github.com/coveralls-clients/coveralls-python/issues/251
-        run: |
-          cd src/
-          pipenv run coveralls --service=github
-
-  tests-frontend:
-    needs: [code-checks-frontend]
-    name: "Frontend Tests"
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        node-version: [16.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
         with:
-          node-version: ${{ matrix.node-version }}
-      - run: cd src-ui && npm ci
-      - run: cd src-ui && npm run test
-      - run: cd src-ui && npm run e2e:ci
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Determine if build needed
+        id: build-skip-check
+        # Skip building the frontend if the tag exists and no src-ui files changed
+        run: |
+          if ! docker manifest inspect ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} &> /dev/null ; then
+            echo "Build required, no existing image"
+            echo ::set-output name=frontend-build-needed::true
+          elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then
+            echo "Build required, src-ui changes"
+            echo ::set-output name=frontend-build-needed::true
+          else
+            echo "No build required"
+            echo ::set-output name=frontend-build-needed::false
+          fi
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
+      -
+        name: Compile frontend
+        uses: docker/build-push-action@v2
+        if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
+        with:
+          context: .
+          file: ./docker-builders/Dockerfile.frontend
+          tags: ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      -
+        name: Export frontend artifact from docker
+        run: |
+          docker create --name frontend-extract ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+          docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/
+      -
+        name: Upload frontend artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: frontend-compiled
+          path: src/documents/static/frontend/
 
   # build and push image to docker hub.
   build-docker-image:
-    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
     concurrency:
       group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
       cancel-in-progress: true
     runs-on: ubuntu-20.04
-    needs: [tests-backend, tests-frontend]
+    concurrency:
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
+      cancel-in-progress: true
+    needs:
+      - prepare-docker-build
+      - build-psycopg2-wheel
+      - build-jbig2enc
+      - build-qpdf-debs
+      - build-pikepdf-wheel
+      - build-frontend
     steps:
       -
         name: Gather Docker metadata
@@ -226,26 +316,22 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
+          build-args: |
+            JBIG2ENC_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
+            QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+            PIKEPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
+            PSYCOPG2_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
+            FRONTEND_BASE_IMAGE=${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
       -
         name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
-      -
-        name: Export frontend artifact from docker
-        run: |
-          docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
-          docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
-      -
-        name: Upload frontend artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: frontend-compiled
-          path: src/documents/static/frontend/
 
   build-release:
-    needs: [build-docker-image, documentation]
+    needs:
+      - build-docker-image
     runs-on: ubuntu-20.04
     steps:
       -
@@ -313,7 +399,8 @@ jobs:
 
   publish-release:
     runs-on: ubuntu-20.04
-    needs: build-release
+    needs:
+      - build-release
     if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
     steps:
       -
diff --git a/.github/workflows/reusable-ci-backend.yml b/.github/workflows/reusable-ci-backend.yml
new file mode 100644
index 000000000..28092fcb1
--- /dev/null
+++ b/.github/workflows/reusable-ci-backend.yml
@@ -0,0 +1,108 @@
+name: Backend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-backend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Install checkers
+        run: |
+          pipx install reorder-python-imports
+          pipx install yesqa
+          pipx install add-trailing-comma
+          pipx install flake8
+      -
+        name: Run reorder-python-imports
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
+      -
+        name: Run yesqa
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
+      -
+        name: Run add-trailing-comma
+        run: |
+          find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
+      # black is placed after add-trailing-comma because it may format differently
+      # if a trailing comma is added
+      -
+        name: Run black
+        uses: psf/black@stable
+        with:
+          options: "--check --diff"
+          version: "22.3.0"
+      -
+        name: Run flake8 checks
+        run: |
+          cd src/
+          flake8 --max-line-length=88 --ignore=E203,W503
+
+  tests-backend:
+    name: "Tests (${{ matrix.python-version }})"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-backend
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+      fail-fast: false
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      -
+        name: Install pipenv
+        run: pipx install pipenv
+      -
+        name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "${{ matrix.python-version }}"
+          cache: "pipenv"
+          cache-dependency-path: 'Pipfile.lock'
+      -
+        name: Install system dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+      -
+        name: Install Python dependencies
+        run: |
+          pipenv sync --dev
+      -
+        name: Tests
+        run: |
+          cd src/
+          pipenv run pytest
+      -
+        name: Get changed files
+        id: changed-files-specific
+        uses: tj-actions/changed-files@v18.1
+        with:
+          files: |
+            src/**
+      -
+        name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+            echo "${file} was changed"
+          done
+      -
+        name: Publish coverage results
+        if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # https://github.com/coveralls-clients/coveralls-python/issues/251
+        run: |
+          cd src/
+          pipenv run coveralls --service=github
diff --git a/.github/workflows/reusable-ci-frontend.yml b/.github/workflows/reusable-ci-frontend.yml
new file mode 100644
index 000000000..cc565775a
--- /dev/null
+++ b/.github/workflows/reusable-ci-frontend.yml
@@ -0,0 +1,42 @@
+name: Frontend CI Jobs
+
+on:
+  workflow_call:
+
+jobs:
+
+  code-checks-frontend:
+    name: "Code Style Checks"
+    runs-on: ubuntu-20.04
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      -
+        name: Install prettier
+        run: |
+          npm install prettier
+      -
+        name: Run prettier
+        run:
+          npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+  tests-frontend:
+    name: "Tests"
+    runs-on: ubuntu-20.04
+    needs:
+      - code-checks-frontend
+    strategy:
+      matrix:
+        node-version: [16.x]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - run: cd src-ui && npm ci
+      - run: cd src-ui && npm run test
+      - run: cd src-ui && npm run e2e:ci
diff --git a/.github/workflows/reusable-workflow-builder.yml b/.github/workflows/reusable-workflow-builder.yml
new file mode 100644
index 000000000..543cd3d79
--- /dev/null
+++ b/.github/workflows/reusable-workflow-builder.yml
@@ -0,0 +1,68 @@
+name: Reusable Image Builder
+
+on:
+  workflow_call:
+    inputs:
+      dockerfile:
+        required: true
+        type: string
+      build-json:
+        required: true
+        type: string
+      build-args:
+        required: false
+        default: ""
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
+  cancel-in-progress: false
+
+jobs:
+  build-image:
+    name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Login to Github Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Determine if build needed
+        id: build-skip-check
+        run: |
+          if ! docker manifest inspect ${{ fromJSON(inputs.build-json).image_tag }} &> /dev/null ; then
+            echo "Building, no image exists with this version"
+            echo ::set-output name=image-exists::false
+          else
+            echo "Not building, image exists with this version"
+            echo ::set-output name=image-exists::true
+          fi
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+      -
+        name: Build ${{ fromJSON(inputs.build-json).name }}
+        uses: docker/build-push-action@v2
+        if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+        with:
+          context: .
+          file: ${{ inputs.dockerfile }}
+          tags: ${{ fromJSON(inputs.build-json).image_tag }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          build-args: ${{ inputs.build-args }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 65ecc7980..32998e432 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -63,10 +63,17 @@ repos:
     hooks:
       - id: black
   # Dockerfile hooks
-  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
-    rev: "v0.1.0"
+  - repo: https://github.com/AleksaC/hadolint-py
+    rev: v1.19.0
     hooks:
-      - id: dockerfilelint
+      - id: hadolint
+        args:
+          - --ignore
+          - DL3006      # https://github.com/hadolint/hadolint/wiki/DL3006  (doesn't understand FROM with ARG)
+          - --ignore
+          - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
+          - --ignore
+          - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1
diff --git a/Dockerfile b/Dockerfile
index 8b46d072b..ef162fd38 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,18 @@
-FROM node:16 AS compile-frontend
+# These are all built previously in the pipeline
+# They provide either a .deb, .whl or whatever npm outputs
+ARG JBIG2ENC_BASE_IMAGE
+ARG QPDF_BASE_IMAGE
+ARG PIKEPDF_BASE_IMAGE
+ARG PSYCOPG2_BASE_IMAGE
+ARG FRONTEND_BASE_IMAGE
 
-COPY . /src
+FROM ${JBIG2ENC_BASE_IMAGE} AS jbig2enc-builder
+FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+FROM ${PIKEPDF_BASE_IMAGE} as pikepdf-builder
+FROM ${PSYCOPG2_BASE_IMAGE} as psycopg2-builder
+FROM ${FRONTEND_BASE_IMAGE} as compile-frontend
 
-WORKDIR /src/src-ui
-RUN npm update npm -g && npm ci --no-optional
-RUN ./node_modules/.bin/ng build --configuration production
-
-FROM ghcr.io/paperless-ngx/builder/ngx-base:1.7.0 as main-app
+FROM python:3.9-slim-bullseye as main-app
 
 LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
 LABEL org.opencontainers.image.documentation="https://paperless-ngx.readthedocs.io/en/latest/"
@@ -14,27 +20,115 @@ LABEL org.opencontainers.image.source="https://github.com/paperless-ngx/paperles
 LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-ngx"
 LABEL org.opencontainers.image.licenses="GPL-3.0-only"
 
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Packages needed only for building
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  python3-dev"
+
+# Packages need for running
+ARG RUNTIME_PACKAGES="\
+  curl \
+  file \
+  # fonts for text file thumbnail generation
+  fonts-liberation \
+  gettext \
+  ghostscript \
+  gnupg \
+  gosu \
+  icc-profiles-free \
+  imagemagick \
+  media-types \
+  liblept5 \
+  libpq5 \
+  libxml2 \
+  libxslt1.1 \
+  libgnutls30 \
+  libjpeg62-turbo \
+  optipng \
+  python3 \
+  python3-pip \
+  python3-setuptools \
+  postgresql-client \
+  # For Numpy
+  libatlas3-base \
+  # thumbnail size reduction
+  pngquant \
+  # OCRmyPDF dependencies
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-deu \
+  tesseract-ocr-fra \
+  tesseract-ocr-ita \
+  tesseract-ocr-spa \
+  tzdata \
+  unpaper \
+  # Mime type detection
+  zlib1g \
+  # Barcode splitter
+  libzbar0 \
+  poppler-utils"
+
 WORKDIR /usr/src/paperless/src/
 
+# Copy qpdf and runtime library
+COPY --from=qpdf-builder /usr/src/qpdf/libqpdf28_*.deb .
+COPY --from=qpdf-builder /usr/src/qpdf/qpdf_*.deb .
+
+# Copy pikepdf wheel and dependencies
+COPY --from=pikepdf-builder /usr/src/pikepdf/wheels/*.whl .
+
+# Copy psycopg2 wheel
+COPY --from=psycopg2-builder /usr/src/psycopg2/wheels/psycopg2*.whl .
+
+# copy jbig2enc
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
+
 COPY requirements.txt ../
 
 # Python dependencies
-RUN apt-get update \
-  # python-Levenshtein still needs to be compiled here
-  && apt-get -y --no-install-recommends install \
-    build-essential \
-    && python3 -m pip install --upgrade --no-cache-dir pip wheel \
-  && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
-  && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
-  && apt-get -y purge build-essential \
-  && apt-get -y autoremove --purge \
-  && rm -rf /var/lib/apt/lists/*
+RUN set -eux \
+  && apt-get update \
+  && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} ${BUILD_PACKAGES} \
+  && python3 -m pip install --no-cache-dir --upgrade wheel \
+  && echo "Installing qpdf" \
+    && apt-get install --yes --no-install-recommends ./libqpdf28_*.deb \
+    && apt-get install --yes --no-install-recommends ./qpdf_*.deb \
+  && echo "Installing pikepdf and dependencies wheel" \
+    && python3 -m pip install --no-cache-dir packaging*.whl \
+    && python3 -m pip install --no-cache-dir lxml*.whl \
+    && python3 -m pip install --no-cache-dir Pillow*.whl \
+    && python3 -m pip install --no-cache-dir pyparsing*.whl \
+    && python3 -m pip install --no-cache-dir pikepdf*.whl \
+    && python -m pip list \
+  && echo "Installing psycopg2 wheel" \
+    && python3 -m pip install --no-cache-dir psycopg2*.whl \
+    && python -m pip list \
+  && echo "Installing supervisor" \
+    && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
+  && echo "Installing Python requirements" \
+    && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+  && echo "Cleaning up image" \
+    && apt-get -y purge ${BUILD_PACKAGES} \
+    && apt-get -y autoremove --purge \
+    && apt-get clean --yes \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/* \
+    && rm -rf /var/tmp/* \
+    && rm -rf /var/cache/apt/archives/* \
+    && truncate -s 0 /var/log/*log
 
 # setup docker-specific things
 COPY docker/ ./docker/
 
-RUN cd docker \
-    && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+WORKDIR /usr/src/paperless/src/docker/
+
+RUN set -eux \
+  && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
   && mkdir /var/log/supervisord /var/run/supervisord \
   && cp supervisord.conf /etc/supervisord.conf \
   && cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
@@ -42,17 +136,18 @@ RUN cd docker \
   && cp docker-prepare.sh /sbin/docker-prepare.sh \
   && chmod 755 /sbin/docker-prepare.sh \
   && chmod +x install_management_commands.sh \
-  && ./install_management_commands.sh \
-  && cd .. \
-  && rm -rf docker/
+  && ./install_management_commands.sh
 
-COPY gunicorn.conf.py ../
+WORKDIR /usr/src/paperless/
+
+COPY gunicorn.conf.py .
 
 # copy app
 COPY --from=compile-frontend /src/src/ ./
 
 # add users, setup scripts
-RUN addgroup --gid 1000 paperless \
+RUN set -eux \
+  && addgroup --gid 1000 paperless \
   && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
   && chown -R paperless:paperless ../ \
   && gosu paperless python3 manage.py collectstatic --clear --no-input \
diff --git a/docker-builders/Dockerfile.frontend b/docker-builders/Dockerfile.frontend
new file mode 100644
index 000000000..6e0ee5374
--- /dev/null
+++ b/docker-builders/Dockerfile.frontend
@@ -0,0 +1,13 @@
+# This Dockerfile compiles the frontend
+# Inputs: None
+
+FROM node:16-bullseye-slim AS compile-frontend
+
+COPY . /src
+
+WORKDIR /src/src-ui
+RUN set -eux \
+	&& npm update npm -g \
+	&& npm ci --no-optional
+RUN set -eux \
+	&& ./node_modules/.bin/ng build --configuration production
diff --git a/docker-builders/Dockerfile.jbig2enc b/docker-builders/Dockerfile.jbig2enc
new file mode 100644
index 000000000..72429ec0b
--- /dev/null
+++ b/docker-builders/Dockerfile.jbig2enc
@@ -0,0 +1,39 @@
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+#		- JBIG2ENC_VERSION - the Git tag to checkout and build
+
+FROM debian:bullseye-slim
+
+LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  automake \
+  libtool \
+  libleptonica-dev \
+  zlib1g-dev \
+  git \
+  ca-certificates"
+
+WORKDIR /usr/src/jbig2enc
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+RUN apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG JBIG2ENC_VERSION
+
+RUN set -eux \
+	&& git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
+RUN set -eux \
+	&& ./autogen.sh
+RUN set -eux \
+	&& ./configure && make
diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf
new file mode 100644
index 000000000..3769caf5f
--- /dev/null
+++ b/docker-builders/Dockerfile.pikepdf
@@ -0,0 +1,65 @@
+# This Dockerfile builds the pikepdf wheel
+# Inputs:
+#		- QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from
+#		- GIT_TAG - The Git tag to clone and build from
+#		- VERSION - Used to force the built pikepdf version to match
+
+ARG QPDF_BASE_IMAGE
+FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+
+# This does nothing, except provide a name for a copy below
+
+FROM python:3.9-slim-bullseye
+
+LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  libjpeg62-turbo-dev \
+  zlib1g-dev \
+  libgnutls28-dev \
+  libxml2-dev \
+  libxslt1-dev \
+  python3-dev \
+  python3-pip"
+
+WORKDIR /usr/src
+
+COPY --from=qpdf-builder /usr/src/qpdf/*.deb .
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && dpkg --install libqpdf28_*.deb \
+  && dpkg --install libqpdf-dev_*.deb \
+  && python3 -m pip install --no-cache-dir --upgrade pip wheel pybind11 \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG GIT_TAG
+ARG VERSION
+
+RUN set -eux \
+	&& echo "building pikepdf wheel" \
+  # Note the v in the tag name here
+  && git clone --quiet --depth 1 --branch "${GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
+  && cd pikepdf \
+  # pikepdf seems to specifciy either a next version when built OR
+  # a post release tag.
+  # In either case, this won't match what we want from requirements.txt
+  # Directly modify the setup.py to set the version we just checked out of Git
+  && sed -i "s/use_scm_version=True/version=\"${VERSION}\"/g" setup.py \
+  # https://github.com/pikepdf/pikepdf/issues/323
+  && rm pyproject.toml \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels
diff --git a/docker-builders/Dockerfile.psycopg2 b/docker-builders/Dockerfile.psycopg2
new file mode 100644
index 000000000..0c1cc048b
--- /dev/null
+++ b/docker-builders/Dockerfile.psycopg2
@@ -0,0 +1,44 @@
+# This Dockerfile builds the psycopg2 wheel
+# Inputs:
+#		- GIT_TAG - The Git tag to clone and build from
+#		- VERSION - Unused, kept for future possible usage
+
+FROM python:3.9-slim-bullseye
+
+LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  git \
+  libpq-dev \
+  python3-dev \
+  python3-pip"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/* \
+  && python3 -m pip install --upgrade pip wheel
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG GIT_TAG
+ARG VERSION
+
+RUN set -eux \
+	&& echo "Building psycopg2 wheel" \
+  && cd /usr/src \
+  && git clone --quiet --depth 1 --branch ${GIT_TAG} https://github.com/psycopg/psycopg2.git \
+  && cd psycopg2 \
+  && mkdir wheels \
+  && python3 -m pip wheel . --wheel-dir wheels \
+  && ls -ahl wheels/
diff --git a/docker-builders/Dockerfile.qpdf b/docker-builders/Dockerfile.qpdf
new file mode 100644
index 000000000..c56a515c4
--- /dev/null
+++ b/docker-builders/Dockerfile.qpdf
@@ -0,0 +1,51 @@
+FROM debian:bullseye-slim
+
+LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+  build-essential \
+  debhelper \
+  debian-keyring \
+  devscripts \
+  equivs  \
+  libtool \
+  libjpeg62-turbo-dev \
+  libgnutls28-dev \
+  packaging-dev \
+  zlib1g-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+  && apt-get update --quiet \
+  && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+  && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
+
+# In order to get the required version of qpdf, it is backported from bookwork
+# and then built from source
+RUN set -eux \
+	&& echo "Building qpdf" \
+  && echo "deb-src http://deb.debian.org/debian/ bookworm main" | tee /etc/apt/sources.list.d/bookworm-src.list \
+  && apt-get update \
+  && mkdir qpdf \
+  && cd qpdf \
+  && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
+  && rm -rf /var/lib/apt/lists/* \
+  && cd qpdf-$QPDF_VERSION \
+  && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
+	&& export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+  && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
+  && pwd \
+  && ls -ahl ../*.deb
diff --git a/docker-builders/get-build-json.py b/docker-builders/get-build-json.py
new file mode 100755
index 000000000..f8d4f8701
--- /dev/null
+++ b/docker-builders/get-build-json.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+This is a helper script to either parse the JSON of the Pipfile.lock
+or otherwise return a JSON object detailing versioning and image tags
+for the packages we build seperately, then copy into the final Docker image
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Final
+
+CONFIG: Final = {
+    # All packages need to be in the dict, even if not configured further
+    # as it is used for the possible choices in the argument
+    "psycopg2": {},
+    # Most information about Python packages comes from the Pipfile.lock
+    "pikepdf": {
+        "qpdf_version": "10.6.3",
+    },
+    # For other packages, it is directly configured, for now
+    # These require manual updates to this file for version updates
+    "qpdf": {
+        "version": "10.6.3",
+        "git_tag": "N/A",
+    },
+    "jbig2enc": {
+        "version": "0.29",
+        "git_tag": "0.29",
+    },
+}
+
+
+def _get_image_tag(
+    repo_name: str,
+    pkg_name: str,
+    pkg_version: str,
+) -> str:
+    return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+
+
+def _main():
+    parser = argparse.ArgumentParser(
+        description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
+    )
+    parser.add_argument(
+        "package",
+        help="The name of the package to generate JSON for",
+        choices=CONFIG.keys(),
+    )
+
+    args = parser.parse_args()
+
+    pip_lock = Path("Pipfile.lock")
+
+    repo_name = os.environ["GITHUB_REPOSITORY"]
+
+    # The JSON object we'll output
+    output = {"name": args.package}
+
+    # Read Pipfile.lock file
+
+    pipfile_data = json.loads(pip_lock.read_text())
+
+    # Read the version from Pipfile.lock
+
+    if args.package in pipfile_data["default"]:
+
+        pkg_data = pipfile_data["default"][args.package]
+
+        pkg_version = pkg_data["version"].split("==")[-1]
+
+        output["version"] = pkg_version
+
+        # Based on the package, generate the expected Git tag name
+
+        if args.package == "pikepdf":
+            git_tag_name = f"v{pkg_version}"
+        elif args.package == "psycopg2":
+            git_tag_name = pkg_version.replace(".", "_")
+
+        output["git_tag"] = git_tag_name
+
+        # Based on the package and environment, generate the Docker image tag
+
+        image_tag = _get_image_tag(repo_name, args.package, pkg_version)
+
+        output["image_tag"] = image_tag
+
+        # Check for any special configuration, based on package
+
+        if args.package in CONFIG:
+            output.update(CONFIG[args.package])
+
+    elif args.package in CONFIG:
+
+        # This is not a Python package
+
+        output.update(CONFIG[args.package])
+
+        output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
+
+    else:
+        raise NotImplementedError(args.package)
+
+    # Output the JSON info to stdout
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    _main()
diff --git a/docker/docker-prepare.sh b/docker/docker-prepare.sh
index 681ccf5a0..48f0c6b82 100755
--- a/docker/docker-prepare.sh
+++ b/docker/docker-prepare.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 wait_for_postgres() {
 	attempt_num=1
 	max_attempts=5
diff --git a/docker/install_management_commands.sh b/docker/install_management_commands.sh
index 9da795b50..bf8bbeb93 100755
--- a/docker/install_management_commands.sh
+++ b/docker/install_management_commands.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -eu
+
 for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
 do
 	echo "installing $command..."

From 6179ca5668f7abac8ecf4e8a899b0c53da01adfb Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Mon, 18 Apr 2022 14:48:08 -0700
Subject: [PATCH 02/11] Fixes final WORKDIR directive

---
 Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index ef162fd38..34f4b2c27 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -142,6 +142,8 @@ WORKDIR /usr/src/paperless/
 
 COPY gunicorn.conf.py .
 
+WORKDIR /usr/src/paperless/src/
+
 # copy app
 COPY --from=compile-frontend /src/src/ ./
 

From d19015579c8e74fea118e09ab1d11526683e8873 Mon Sep 17 00:00:00 2001
From: Trenton Holmes <trenton.holmes@psware.com>
Date: Thu, 21 Apr 2022 15:47:50 -0700
Subject: [PATCH 03/11] Instead of using a full image name, use the repo and
 version to build the image to pull from.  Removes building of the frontend
 for multiple platforms

---
 .github/workflows/ci.yml           | 45 +++++++++++++-------------
 Dockerfile                         | 23 +++++++------
 docker-builders/Dockerfile.pikepdf | 10 ++++--
 docker-builders/get-build-json.py  | 52 ++++++++++++++----------------
 4 files changed, 67 insertions(+), 63 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e9ed4306c..fe9afb1a3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -63,10 +63,6 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v3
-      -
-        name: Get branch name
-        id: branch-name
-        uses: tj-actions/branch-names@v5
       -
         name: Login to Github Container Registry
         uses: docker/login-action@v1
@@ -123,16 +119,14 @@ jobs:
         name: Setup frontend image
         id: frontend-setup
         run: |
-          frontend_image=ghcr.io/${{ github.repository }}/ngx-frontend:${{ steps.branch-name.outputs.current_branch }}
+          build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py frontend)
 
-          echo ${frontend_image}
+          echo ${build_json}
 
-          echo ::set-output name=frontend-image-tag::${frontend_image}
+          echo ::set-output name=frontend-json::${build_json}
 
     outputs:
 
-      frontend-image-tag: ${{ steps.frontend-setup.outputs.frontend-image-tag }}
-
       qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
 
       pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
@@ -141,6 +135,8 @@ jobs:
 
       jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
 
+      frontend-json: ${{ steps.frontend-setup.outputs.frontend-json}}
+
   build-qpdf-debs:
     name: qpdf
     needs:
@@ -185,14 +181,15 @@ jobs:
       dockerfile: ./docker-builders/Dockerfile.pikepdf
       build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
       build-args: |
-        QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+        REPO=${{ github.repository }}
+        QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
         GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
         VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
 
   build-frontend:
     name: Compile frontend
     concurrency:
-      group: ${{ github.workflow }}-build-frontend-${{ github.ref }}
+      group: ${{ github.workflow }}-build-frontend-${{ github.ref_name }}
       cancel-in-progress: false
     needs:
       - prepare-docker-build
@@ -222,7 +219,7 @@ jobs:
         id: build-skip-check
         # Skip building the frontend if the tag exists and no src-ui files changed
         run: |
-          if ! docker manifest inspect ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} &> /dev/null ; then
+          if ! docker manifest inspect ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} &> /dev/null ; then
             echo "Build required, no existing image"
             echo ::set-output name=frontend-build-needed::true
           elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then
@@ -247,15 +244,18 @@ jobs:
         with:
           context: .
           file: ./docker-builders/Dockerfile.frontend
-          tags: ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          tags: ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
+          # The compilation is identical between different platforms
+          # The buildx and QEMU setup is left, just in case that ever changes
+          # But the platform is set to the runner's native for speedup
+          platforms: linux/amd64
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
       -
         name: Export frontend artifact from docker
         run: |
-          docker create --name frontend-extract ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+          docker create --name frontend-extract ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
           docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/
       -
         name: Upload frontend artifact
@@ -271,7 +271,7 @@ jobs:
       cancel-in-progress: true
     runs-on: ubuntu-20.04
     concurrency:
-      group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
+      group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}
       cancel-in-progress: true
     needs:
       - prepare-docker-build
@@ -317,11 +317,12 @@ jobs:
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
           build-args: |
-            JBIG2ENC_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
-            QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
-            PIKEPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
-            PSYCOPG2_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
-            FRONTEND_BASE_IMAGE=${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+            REPO=${{ github.repository }}
+            JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+            QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+            PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+            PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+            FRONTEND_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).version }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
       -
@@ -401,7 +402,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs:
       - build-release
-    if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
+    if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'ngx-') || startsWith(github.ref_name, 'beta-'))
     steps:
       -
         name: Download release artifact
diff --git a/Dockerfile b/Dockerfile
index 34f4b2c27..3b71421d9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,16 +1,19 @@
+# Default to pulling from the main repo registry when manually building
+ARG REPO="paperless-ngx/paperless-ngx"
+
 # These are all built previously in the pipeline
 # They provide either a .deb, .whl or whatever npm outputs
-ARG JBIG2ENC_BASE_IMAGE
-ARG QPDF_BASE_IMAGE
-ARG PIKEPDF_BASE_IMAGE
-ARG PSYCOPG2_BASE_IMAGE
-ARG FRONTEND_BASE_IMAGE
+ARG JBIG2ENC_VERSION
+ARG QPDF_VERSION
+ARG PIKEPDF_VERSION
+ARG PSYCOPG2_VERSION
+ARG FRONTEND_VERSION
 
-FROM ${JBIG2ENC_BASE_IMAGE} AS jbig2enc-builder
-FROM ${QPDF_BASE_IMAGE} as qpdf-builder
-FROM ${PIKEPDF_BASE_IMAGE} as pikepdf-builder
-FROM ${PSYCOPG2_BASE_IMAGE} as psycopg2-builder
-FROM ${FRONTEND_BASE_IMAGE} as compile-frontend
+FROM ghcr.io/${REPO}/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
+FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+FROM ghcr.io/${REPO}/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
+FROM ghcr.io/${REPO}/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
+FROM ghcr.io/${REPO}/builder/frontend:${FRONTEND_VERSION} as compile-frontend
 
 FROM python:3.9-slim-bullseye as main-app
 
diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf
index 3769caf5f..be544e282 100644
--- a/docker-builders/Dockerfile.pikepdf
+++ b/docker-builders/Dockerfile.pikepdf
@@ -1,11 +1,15 @@
 # This Dockerfile builds the pikepdf wheel
 # Inputs:
-#		- QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from
+#		- REPO - Docker repository to pull qpdf from
+#		- QPDF_VERSION - The image qpdf version to copy .deb files from
 #		- GIT_TAG - The Git tag to clone and build from
 #		- VERSION - Used to force the built pikepdf version to match
 
-ARG QPDF_BASE_IMAGE
-FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+# Default to pulling from the main repo registry when manually building
+ARG REPO="paperless-ngx/paperless-ngx"
+
+ARG QPDF_VERSION
+FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
 
 # This does nothing, except provide a name for a copy below
 
diff --git a/docker-builders/get-build-json.py b/docker-builders/get-build-json.py
index f8d4f8701..a6d24d3fa 100755
--- a/docker-builders/get-build-json.py
+++ b/docker-builders/get-build-json.py
@@ -1,8 +1,18 @@
 #!/usr/bin/env python3
 """
-This is a helper script to either parse the JSON of the Pipfile.lock
-or otherwise return a JSON object detailing versioning and image tags
-for the packages we build seperately, then copy into the final Docker image
+This is a helper script for the mutli-stage Docker image builder.
+It provides a single point of configuration for package version control.
+The output JSON object is used by the CI workflow to determine what versions
+to build and pull into the final Docker image.
+
+Python package information is obtained from the Pipfile.lock.  As this is
+kept updated by dependabot, it usually will need no further configuration.
+The sole exception currently is pikepdf, which has a dependency on qpdf,
+and is configured here to use the latest version of qpdf built by the workflow.
+
+Other package version information is configured directly below, generally by
+setting the version and Git information, if any.
+
 """
 import argparse
 import json
@@ -14,11 +24,13 @@ CONFIG: Final = {
     # All packages need to be in the dict, even if not configured further
     # as it is used for the possible choices in the argument
     "psycopg2": {},
+    "frontend": {},
     # Most information about Python packages comes from the Pipfile.lock
+    # Excpetion being pikepdf, which needs a specific qpdf
     "pikepdf": {
         "qpdf_version": "10.6.3",
     },
-    # For other packages, it is directly configured, for now
+    # For other packages, version and Git information are directly configured
     # These require manual updates to this file for version updates
     "qpdf": {
         "version": "10.6.3",
@@ -59,11 +71,9 @@ def _main():
     output = {"name": args.package}
 
     # Read Pipfile.lock file
-
     pipfile_data = json.loads(pip_lock.read_text())
 
     # Read the version from Pipfile.lock
-
     if args.package in pipfile_data["default"]:
 
         pkg_data = pipfile_data["default"][args.package]
@@ -73,7 +83,6 @@ def _main():
         output["version"] = pkg_version
 
         # Based on the package, generate the expected Git tag name
-
         if args.package == "pikepdf":
             git_tag_name = f"v{pkg_version}"
         elif args.package == "psycopg2":
@@ -81,31 +90,18 @@ def _main():
 
         output["git_tag"] = git_tag_name
 
-        # Based on the package and environment, generate the Docker image tag
+    # Use the basic ref name, minus refs/heads or refs/tags for frontend builder image
+    elif args.package == "frontend":
+        output["version"] = os.environ["GITHUB_REF_NAME"]
 
-        image_tag = _get_image_tag(repo_name, args.package, pkg_version)
+    # Add anything special from the config
+    output.update(CONFIG[args.package])
 
-        output["image_tag"] = image_tag
-
-        # Check for any special configuration, based on package
-
-        if args.package in CONFIG:
-            output.update(CONFIG[args.package])
-
-    elif args.package in CONFIG:
-
-        # This is not a Python package
-
-        output.update(CONFIG[args.package])
-
-        output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
-
-    else:
-        raise NotImplementedError(args.package)
+    # Based on the package and environment, generate the Docker image tag
+    output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
 
     # Output the JSON info to stdout
-
-    print(json.dumps(output, indent=2))
+    print(json.dumps(output))
 
 
 if __name__ == "__main__":

From ecca51dbdd57901b0a33610c20f424a7534bbb47 Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Sun, 24 Apr 2022 14:18:51 -0700
Subject: [PATCH 04/11] Ports over the improvements to the frontend compilation
 from other PR

---
 docker-builders/Dockerfile.frontend | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker-builders/Dockerfile.frontend b/docker-builders/Dockerfile.frontend
index 6e0ee5374..0caec0030 100644
--- a/docker-builders/Dockerfile.frontend
+++ b/docker-builders/Dockerfile.frontend
@@ -3,7 +3,8 @@
 
 FROM node:16-bullseye-slim AS compile-frontend
 
-COPY . /src
+COPY ./src /src/src
+COPY ./src-ui /src/src-ui
 
 WORKDIR /src/src-ui
 RUN set -eux \

From f1da37dd124e384f7d2ef17e63892e523968e009 Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Sun, 24 Apr 2022 14:50:38 -0700
Subject: [PATCH 05/11] Minor fixes for local building

---
 Dockerfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 3b71421d9..77417383f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -77,14 +77,14 @@ ARG RUNTIME_PACKAGES="\
 WORKDIR /usr/src/paperless/src/
 
 # Copy qpdf and runtime library
-COPY --from=qpdf-builder /usr/src/qpdf/libqpdf28_*.deb .
-COPY --from=qpdf-builder /usr/src/qpdf/qpdf_*.deb .
+COPY --from=qpdf-builder /usr/src/qpdf/libqpdf28_*.deb ./
+COPY --from=qpdf-builder /usr/src/qpdf/qpdf_*.deb ./
 
 # Copy pikepdf wheel and dependencies
-COPY --from=pikepdf-builder /usr/src/pikepdf/wheels/*.whl .
+COPY --from=pikepdf-builder /usr/src/pikepdf/wheels/*.whl ./
 
 # Copy psycopg2 wheel
-COPY --from=psycopg2-builder /usr/src/psycopg2/wheels/psycopg2*.whl .
+COPY --from=psycopg2-builder /usr/src/psycopg2/wheels/psycopg2*.whl ./
 
 # copy jbig2enc
 COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/

From be8ca110f422b0c2d2915e9479aade4c788aa3ef Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Sun, 24 Apr 2022 14:51:06 -0700
Subject: [PATCH 06/11] Fixes spelling

---
 docker-builders/get-build-json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-builders/get-build-json.py b/docker-builders/get-build-json.py
index a6d24d3fa..07a33b4d4 100755
--- a/docker-builders/get-build-json.py
+++ b/docker-builders/get-build-json.py
@@ -26,7 +26,7 @@ CONFIG: Final = {
     "psycopg2": {},
     "frontend": {},
     # Most information about Python packages comes from the Pipfile.lock
-    # Excpetion being pikepdf, which needs a specific qpdf
+    # Exception being pikepdf, which needs a specific qpdf version
     "pikepdf": {
         "qpdf_version": "10.6.3",
     },

From 609b9e3369c844c6f6fd7f425a5c7f100ac3c16a Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Sun, 24 Apr 2022 14:54:44 -0700
Subject: [PATCH 07/11] Adds utility script for building the Docker image
 locally

---
 build-docker-imaage.sh | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100755 build-docker-imaage.sh

diff --git a/build-docker-imaage.sh b/build-docker-imaage.sh
new file mode 100755
index 000000000..7d097fe06
--- /dev/null
+++ b/build-docker-imaage.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Example Usage: ./build-docker-imaage.sh -t paperless-ngx:my-awesome-feature
+
+set -eux
+
+# Parse what we can from Pipfile.lock
+pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
+psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+
+# Get the branch name
+frontend=$(git rev-parse --abbrev-ref HEAD)
+
+# Directly set these
+# Future enhancement: Set this in a single location
+qpdf_version="10.6.3"
+jbig2enc_version="0.29"
+
+docker build . \
+	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
+	--build-arg QPDF_VERSION="${qpdf_version}" \
+	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" \
+	--build-arg FRONTEND_VERSION="${frontend}" "$@"

From 4517692c202ca216603724941c3a660fc5c8e57b Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Sun, 24 Apr 2022 15:20:36 -0700
Subject: [PATCH 08/11] Updates the utility script to allow building of other
 Dockerfiles for building any image locally.  Adds a single point to configure
 non-Python versions

---
 .build-config.json                |  9 ++++
 build-docker-imaage.sh            | 24 ---------
 build-docker-image.sh             | 35 +++++++++++++
 docker-builders/get-build-json.py | 86 ++++++++++++++-----------------
 4 files changed, 84 insertions(+), 70 deletions(-)
 create mode 100644 .build-config.json
 delete mode 100755 build-docker-imaage.sh
 create mode 100755 build-docker-image.sh

diff --git a/.build-config.json b/.build-config.json
new file mode 100644
index 000000000..32cf968d5
--- /dev/null
+++ b/.build-config.json
@@ -0,0 +1,9 @@
+{
+  "qpdf": {
+      "version": "10.6.3"
+    },
+  "jbig2enc": {
+      "version": "0.29",
+      "git_tag": "0.29"
+    }
+}
diff --git a/build-docker-imaage.sh b/build-docker-imaage.sh
deleted file mode 100755
index 7d097fe06..000000000
--- a/build-docker-imaage.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-
-# Example Usage: ./build-docker-imaage.sh -t paperless-ngx:my-awesome-feature
-
-set -eux
-
-# Parse what we can from Pipfile.lock
-pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
-psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
-
-# Get the branch name
-frontend=$(git rev-parse --abbrev-ref HEAD)
-
-# Directly set these
-# Future enhancement: Set this in a single location
-qpdf_version="10.6.3"
-jbig2enc_version="0.29"
-
-docker build . \
-	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
-	--build-arg QPDF_VERSION="${qpdf_version}" \
-	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
-	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" \
-	--build-arg FRONTEND_VERSION="${frontend}" "$@"
diff --git a/build-docker-image.sh b/build-docker-image.sh
new file mode 100755
index 000000000..41c95167c
--- /dev/null
+++ b/build-docker-image.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Helper script for building the Docker image locally.
+# Parses and provides the nessecary versions of other images to Docker
+# before passing in the rest of script args
+
+# First Argument: The Dockerfile to build
+# Other Arguments: Additional arguments to docker build
+
+# Example Usage:
+#	./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature
+#	./build-docker-image.sh docker-builders/Dockerfile.qpdf -t paperless-ngx-build-qpdf:x.y.z
+
+set -eux
+
+# Parse what we can from Pipfile.lock
+pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
+psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
+# Read this from the other config file
+qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
+jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
+# Get the branch name
+frontend=$(git rev-parse --abbrev-ref HEAD)
+
+if [ ! -f "$1" ]; then
+	echo "$1 is not a file, please provide the Dockerfile"
+	exit 1
+fi
+
+docker build --file "$1" \
+	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
+	--build-arg QPDF_VERSION="${qpdf_version}" \
+	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" \
+	--build-arg FRONTEND_VERSION="${frontend}" "${@:2}" .
diff --git a/docker-builders/get-build-json.py b/docker-builders/get-build-json.py
index 07a33b4d4..4b96ac8d7 100755
--- a/docker-builders/get-build-json.py
+++ b/docker-builders/get-build-json.py
@@ -20,28 +20,6 @@ import os
 from pathlib import Path
 from typing import Final
 
-CONFIG: Final = {
-    # All packages need to be in the dict, even if not configured further
-    # as it is used for the possible choices in the argument
-    "psycopg2": {},
-    "frontend": {},
-    # Most information about Python packages comes from the Pipfile.lock
-    # Exception being pikepdf, which needs a specific qpdf version
-    "pikepdf": {
-        "qpdf_version": "10.6.3",
-    },
-    # For other packages, version and Git information are directly configured
-    # These require manual updates to this file for version updates
-    "qpdf": {
-        "version": "10.6.3",
-        "git_tag": "N/A",
-    },
-    "jbig2enc": {
-        "version": "0.29",
-        "git_tag": "0.29",
-    },
-}
-
 
 def _get_image_tag(
     repo_name: str,
@@ -58,47 +36,63 @@ def _main():
     parser.add_argument(
         "package",
         help="The name of the package to generate JSON for",
-        choices=CONFIG.keys(),
     )
 
-    args = parser.parse_args()
+    PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock")
+    BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json")
 
-    pip_lock = Path("Pipfile.lock")
-
-    repo_name = os.environ["GITHUB_REPOSITORY"]
-
-    # The JSON object we'll output
-    output = {"name": args.package}
+    # Read the main config file
+    build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text())
 
     # Read Pipfile.lock file
-    pipfile_data = json.loads(pip_lock.read_text())
+    pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text())
 
-    # Read the version from Pipfile.lock
-    if args.package in pipfile_data["default"]:
+    args: Final = parser.parse_args()
 
+    repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"]
+
+    # Default output values
+    version = None
+    git_tag = None
+    extra_config = {}
+
+    if args.package == "frontend":
+        # Version is just the branch or tag name
+        version = os.environ["GITHUB_REF_NAME"]
+    elif args.package in pipfile_data["default"]:
+        # Read the version from Pipfile.lock
         pkg_data = pipfile_data["default"][args.package]
-
         pkg_version = pkg_data["version"].split("==")[-1]
-
-        output["version"] = pkg_version
+        version = pkg_version
 
         # Based on the package, generate the expected Git tag name
         if args.package == "pikepdf":
-            git_tag_name = f"v{pkg_version}"
+            git_tag = f"v{pkg_version}"
         elif args.package == "psycopg2":
-            git_tag_name = pkg_version.replace(".", "_")
+            git_tag = pkg_version.replace(".", "_")
 
-        output["git_tag"] = git_tag_name
+        # Any extra/special values needed
+        if args.package == "pikepdf":
+            extra_config["qpdf_version"] = build_json["qpdf"]["version"]
 
-    # Use the basic ref name, minus refs/heads or refs/tags for frontend builder image
-    elif args.package == "frontend":
-        output["version"] = os.environ["GITHUB_REF_NAME"]
+    elif args.package in build_json:
+        version = build_json[args.package]["version"]
 
-    # Add anything special from the config
-    output.update(CONFIG[args.package])
+        if "git_tag" in build_json[args.package]:
+            git_tag = build_json[args.package]["git_tag"]
+    else:
+        raise NotImplementedError(args.package)
 
-    # Based on the package and environment, generate the Docker image tag
-    output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
+    # The JSON object we'll output
+    output = {
+        "name": args.package,
+        "version": version,
+        "git_tag": git_tag,
+        "image_tag": _get_image_tag(repo_name, args.package, version),
+    }
+
+    # Add anything special a package may need
+    output.update(extra_config)
 
     # Output the JSON info to stdout
     print(json.dumps(output))

From d9d6b7b151c0d81cd1d24d7d84b47ac9b11021fa Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Mon, 25 Apr 2022 06:31:00 -0700
Subject: [PATCH 09/11] Updates the utlity build script to actually support all
 the images

---
 .github/workflows/ci.yml            |  8 ++++----
 build-docker-image.sh               | 26 +++++++++++++++++++-------
 docker-builders/Dockerfile.pikepdf  | 14 +++++++-------
 docker-builders/Dockerfile.psycopg2 | 10 +++++-----
 4 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fe9afb1a3..c2c25acb4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -168,8 +168,8 @@ jobs:
       dockerfile: ./docker-builders/Dockerfile.psycopg2
       build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
       build-args: |
-        GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
-        VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+        PSYCOPG2_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
+        PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
 
   build-pikepdf-wheel:
     name: pikepdf
@@ -183,8 +183,8 @@ jobs:
       build-args: |
         REPO=${{ github.repository }}
         QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
-        GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
-        VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+        PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
+        PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
 
   build-frontend:
     name: Compile frontend
diff --git a/build-docker-image.sh b/build-docker-image.sh
index 41c95167c..f7ab62ca6 100755
--- a/build-docker-image.sh
+++ b/build-docker-image.sh
@@ -2,7 +2,8 @@
 
 # Helper script for building the Docker image locally.
 # Parses and provides the nessecary versions of other images to Docker
-# before passing in the rest of script args
+# before passing in the rest of script args.  A future enhancement
+# would be to combine this with the CI script
 
 # First Argument: The Dockerfile to build
 # Other Arguments: Additional arguments to docker build
@@ -13,6 +14,14 @@
 
 set -eux
 
+if ! command -v jq;  then
+	echo "jq required"
+	exit 1
+elif [ ! -f "$1" ]; then
+	echo "$1 is not a file, please provide the Dockerfile"
+	exit 1
+fi
+
 # Parse what we can from Pipfile.lock
 pikepdf_version=$(jq ".default.pikepdf.version" Pipfile.lock  | sed 's/=//g' | sed 's/"//g')
 psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | sed 's/"//g')
@@ -20,16 +29,19 @@ psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' |
 qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
 jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
 # Get the branch name
-frontend=$(git rev-parse --abbrev-ref HEAD)
+frontend_version=$(git rev-parse --abbrev-ref HEAD)
 
-if [ ! -f "$1" ]; then
-	echo "$1 is not a file, please provide the Dockerfile"
-	exit 1
-fi
+# Get Git tags for building
+# psycopg2 uses X_Y_Z git tags
+psycopg2_git_tag=${psycopg2_version//./_}
+# pikepdf uses vX.Y.Z
+pikepdf_git_tag="v${pikepdf_version}"
 
 docker build --file "$1" \
 	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
 	--build-arg QPDF_VERSION="${qpdf_version}" \
 	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
+	--build-arg PIKEPDF_GIT_TAG="${pikepdf_git_tag}" \
 	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" \
-	--build-arg FRONTEND_VERSION="${frontend}" "${@:2}" .
+	--build-arg PSYCOPG2_GIT_TAG="${psycopg2_git_tag}" \
+	--build-arg FRONTEND_VERSION="${frontend_version}" "${@:2}" .
diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf
index be544e282..7325bceff 100644
--- a/docker-builders/Dockerfile.pikepdf
+++ b/docker-builders/Dockerfile.pikepdf
@@ -2,8 +2,8 @@
 # Inputs:
 #		- REPO - Docker repository to pull qpdf from
 #		- QPDF_VERSION - The image qpdf version to copy .deb files from
-#		- GIT_TAG - The Git tag to clone and build from
-#		- VERSION - Used to force the built pikepdf version to match
+#		- PIKEPDF_GIT_TAG - The Git tag to clone and build from
+#		- PIKEPDF_VERSION - Used to force the built pikepdf version to match
 
 # Default to pulling from the main repo registry when manually building
 ARG REPO="paperless-ngx/paperless-ngx"
@@ -32,7 +32,7 @@ ARG BUILD_PACKAGES="\
 
 WORKDIR /usr/src
 
-COPY --from=qpdf-builder /usr/src/qpdf/*.deb .
+COPY --from=qpdf-builder /usr/src/qpdf/*.deb ./
 
 # As this is an base image for a multi-stage final image
 # the added size of the install is basically irrelevant
@@ -49,19 +49,19 @@ RUN set -eux \
 # For better caching, seperate the basic installs from
 # the building
 
-ARG GIT_TAG
-ARG VERSION
+ARG PIKEPDF_GIT_TAG
+ARG PIKEPDF_VERSION
 
 RUN set -eux \
 	&& echo "building pikepdf wheel" \
   # Note the v in the tag name here
-  && git clone --quiet --depth 1 --branch "${GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
+  && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
   && cd pikepdf \
   # pikepdf seems to specifciy either a next version when built OR
   # a post release tag.
   # In either case, this won't match what we want from requirements.txt
   # Directly modify the setup.py to set the version we just checked out of Git
-  && sed -i "s/use_scm_version=True/version=\"${VERSION}\"/g" setup.py \
+  && sed -i "s/use_scm_version=True/version=\"${PIKEPDF_VERSION}\"/g" setup.py \
   # https://github.com/pikepdf/pikepdf/issues/323
   && rm pyproject.toml \
   && mkdir wheels \
diff --git a/docker-builders/Dockerfile.psycopg2 b/docker-builders/Dockerfile.psycopg2
index 0c1cc048b..cbfbba7c1 100644
--- a/docker-builders/Dockerfile.psycopg2
+++ b/docker-builders/Dockerfile.psycopg2
@@ -1,7 +1,7 @@
 # This Dockerfile builds the psycopg2 wheel
 # Inputs:
-#		- GIT_TAG - The Git tag to clone and build from
-#		- VERSION - Unused, kept for future possible usage
+#		- PSYCOPG2_GIT_TAG - The Git tag to clone and build from
+#		- PSYCOPG2_VERSION - Unused, kept for future possible usage
 
 FROM python:3.9-slim-bullseye
 
@@ -31,13 +31,13 @@ RUN set -eux \
 # For better caching, seperate the basic installs from
 # the building
 
-ARG GIT_TAG
-ARG VERSION
+ARG PSYCOPG2_GIT_TAG
+ARG PSYCOPG2_VERSION
 
 RUN set -eux \
 	&& echo "Building psycopg2 wheel" \
   && cd /usr/src \
-  && git clone --quiet --depth 1 --branch ${GIT_TAG} https://github.com/psycopg/psycopg2.git \
+  && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \
   && cd psycopg2 \
   && mkdir wheels \
   && python3 -m pip wheel . --wheel-dir wheels \

From bdcc6f861d46d4912698bdd0be497b0f8d1cd0fe Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Mon, 25 Apr 2022 09:15:12 -0700
Subject: [PATCH 10/11] Converts tabs to spaces and update EditorConfig for the
 new files

---
 .editorconfig                       |  2 +-
 .pre-commit-config.yaml             |  8 ++++----
 docker-builders/Dockerfile.frontend |  6 +++---
 docker-builders/Dockerfile.jbig2enc |  8 ++++----
 docker-builders/Dockerfile.pikepdf  | 10 +++++-----
 docker-builders/Dockerfile.psycopg2 |  8 ++++----
 docker-builders/Dockerfile.qpdf     |  9 +++++----
 docs/Dockerfile                     |  6 +++---
 8 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/.editorconfig b/.editorconfig
index 125108a0c..8111f01d8 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -33,5 +33,5 @@ indent_style = space
 [**/test_*.py]
 max_line_length = off
 
-[Dockerfile]
+[Dockerfile*]
 indent_style = space
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 32998e432..f0bf9bace 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -47,7 +47,7 @@ repos:
       - id: yesqa
         exclude: "(migrations)"
   - repo: https://github.com/asottile/add-trailing-comma
-    rev: "v2.2.2"
+    rev: "v2.2.3"
     hooks:
       - id: add-trailing-comma
         exclude: "(migrations)"
@@ -64,16 +64,16 @@ repos:
       - id: black
   # Dockerfile hooks
   - repo: https://github.com/AleksaC/hadolint-py
-    rev: v1.19.0
+    rev: v2.10.0
     hooks:
       - id: hadolint
         args:
-          - --ignore
-          - DL3006      # https://github.com/hadolint/hadolint/wiki/DL3006  (doesn't understand FROM with ARG)
           - --ignore
           - DL3008      # https://github.com/hadolint/hadolint/wiki/DL3008  (should probably do this at some point)
           - --ignore
           - DL3013      # https://github.com/hadolint/hadolint/wiki/DL3013  (should probably do this too at some point)
+          - --ignore
+          - DL3003      # https://github.com/hadolint/hadolint/wiki/DL3003  (seems excessive to use WORKDIR so much)
   # Shell script hooks
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1
diff --git a/docker-builders/Dockerfile.frontend b/docker-builders/Dockerfile.frontend
index 0caec0030..26f6c9e15 100644
--- a/docker-builders/Dockerfile.frontend
+++ b/docker-builders/Dockerfile.frontend
@@ -8,7 +8,7 @@ COPY ./src-ui /src/src-ui
 
 WORKDIR /src/src-ui
 RUN set -eux \
-	&& npm update npm -g \
-	&& npm ci --no-optional
+  && npm update npm -g \
+  && npm ci --no-optional
 RUN set -eux \
-	&& ./node_modules/.bin/ng build --configuration production
+  && ./node_modules/.bin/ng build --configuration production
diff --git a/docker-builders/Dockerfile.jbig2enc b/docker-builders/Dockerfile.jbig2enc
index 72429ec0b..4bc633170 100644
--- a/docker-builders/Dockerfile.jbig2enc
+++ b/docker-builders/Dockerfile.jbig2enc
@@ -1,6 +1,6 @@
 # This Dockerfile compiles the jbig2enc library
 # Inputs:
-#		- JBIG2ENC_VERSION - the Git tag to checkout and build
+#    - JBIG2ENC_VERSION - the Git tag to checkout and build
 
 FROM debian:bullseye-slim
 
@@ -32,8 +32,8 @@ RUN apt-get update --quiet \
 ARG JBIG2ENC_VERSION
 
 RUN set -eux \
-	&& git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
+  && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
 RUN set -eux \
-	&& ./autogen.sh
+  && ./autogen.sh
 RUN set -eux \
-	&& ./configure && make
+  && ./configure && make
diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf
index 7325bceff..3d2e5f235 100644
--- a/docker-builders/Dockerfile.pikepdf
+++ b/docker-builders/Dockerfile.pikepdf
@@ -1,9 +1,9 @@
 # This Dockerfile builds the pikepdf wheel
 # Inputs:
-#		- REPO - Docker repository to pull qpdf from
-#		- QPDF_VERSION - The image qpdf version to copy .deb files from
-#		- PIKEPDF_GIT_TAG - The Git tag to clone and build from
-#		- PIKEPDF_VERSION - Used to force the built pikepdf version to match
+#    - REPO - Docker repository to pull qpdf from
+#    - QPDF_VERSION - The image qpdf version to copy .deb files from
+#    - PIKEPDF_GIT_TAG - The Git tag to clone and build from
+#    - PIKEPDF_VERSION - Used to force the built pikepdf version to match
 
 # Default to pulling from the main repo registry when manually building
 ARG REPO="paperless-ngx/paperless-ngx"
@@ -53,7 +53,7 @@ ARG PIKEPDF_GIT_TAG
 ARG PIKEPDF_VERSION
 
 RUN set -eux \
-	&& echo "building pikepdf wheel" \
+  && echo "building pikepdf wheel" \
   # Note the v in the tag name here
   && git clone --quiet --depth 1 --branch "${PIKEPDF_GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
   && cd pikepdf \
diff --git a/docker-builders/Dockerfile.psycopg2 b/docker-builders/Dockerfile.psycopg2
index cbfbba7c1..cfd7ff56a 100644
--- a/docker-builders/Dockerfile.psycopg2
+++ b/docker-builders/Dockerfile.psycopg2
@@ -1,7 +1,7 @@
 # This Dockerfile builds the psycopg2 wheel
 # Inputs:
-#		- PSYCOPG2_GIT_TAG - The Git tag to clone and build from
-#		- PSYCOPG2_VERSION - Unused, kept for future possible usage
+#    - PSYCOPG2_GIT_TAG - The Git tag to clone and build from
+#    - PSYCOPG2_VERSION - Unused, kept for future possible usage
 
 FROM python:3.9-slim-bullseye
 
@@ -25,7 +25,7 @@ RUN set -eux \
   && apt-get update --quiet \
   && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
   && rm -rf /var/lib/apt/lists/* \
-  && python3 -m pip install --upgrade pip wheel
+  && python3 -m pip install --no-cache-dir --upgrade pip wheel
 
 # Layers after this point change according to required version
 # For better caching, seperate the basic installs from
@@ -35,7 +35,7 @@ ARG PSYCOPG2_GIT_TAG
 ARG PSYCOPG2_VERSION
 
 RUN set -eux \
-	&& echo "Building psycopg2 wheel" \
+  && echo "Building psycopg2 wheel" \
   && cd /usr/src \
   && git clone --quiet --depth 1 --branch ${PSYCOPG2_GIT_TAG} https://github.com/psycopg/psycopg2.git \
   && cd psycopg2 \
diff --git a/docker-builders/Dockerfile.qpdf b/docker-builders/Dockerfile.qpdf
index c56a515c4..770d8c2ee 100644
--- a/docker-builders/Dockerfile.qpdf
+++ b/docker-builders/Dockerfile.qpdf
@@ -36,16 +36,17 @@ ARG QPDF_VERSION
 # In order to get the required version of qpdf, it is backported from bookwork
 # and then built from source
 RUN set -eux \
-	&& echo "Building qpdf" \
-  && echo "deb-src http://deb.debian.org/debian/ bookworm main" | tee /etc/apt/sources.list.d/bookworm-src.list \
+  && echo "Building qpdf" \
+  && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \
   && apt-get update \
   && mkdir qpdf \
   && cd qpdf \
   && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
   && rm -rf /var/lib/apt/lists/* \
   && cd qpdf-$QPDF_VERSION \
+  # We don't need to build the tests (also don't run them)
+  && rm -rf libtests \
   && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
-	&& export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+  && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
   && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
-  && pwd \
   && ls -ahl ../*.deb
diff --git a/docs/Dockerfile b/docs/Dockerfile
index 9fb8bd0cc..bb4b35e2d 100644
--- a/docs/Dockerfile
+++ b/docs/Dockerfile
@@ -1,10 +1,10 @@
 FROM python:3.5.1
 
 # Install Sphinx and Pygments
-RUN pip install Sphinx Pygments
+RUN pip install --no-cache-dir Sphinx Pygments \
+  # Setup directories, copy data
+  && mkdir /build
 
-# Setup directories, copy data
-RUN mkdir /build
 COPY . /build
 WORKDIR /build/docs
 

From cd8d4c357d7c768d3a8b0deffe4dd0de9cf1a621 Mon Sep 17 00:00:00 2001
From: Trenton Holmes <holmes.trenton@gmail.com>
Date: Mon, 25 Apr 2022 11:39:05 -0700
Subject: [PATCH 11/11] Fixes the merge issues

---
 .github/workflows/ci.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c2c25acb4..34bcf21a2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -266,9 +266,6 @@ jobs:
 
   # build and push image to docker hub.
   build-docker-image:
-    concurrency:
-      group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
-      cancel-in-progress: true
     runs-on: ubuntu-20.04
     concurrency:
       group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }}