From f875ae4abf77ffb9c94c574327860faf0942cc98 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Thu, 27 Apr 2023 09:05:28 -0700 Subject: [PATCH] CI cleanup and improvements. Removes the building of installers from the repo, they can now be built elsewhere, on demand, as their building is no longer tied to the Dockerfile --- .build-config.json | 9 - .github/scripts/common.py | 47 --- .github/scripts/get-build-json.py | 91 ----- .github/workflows/ci.yml | 120 ++----- .github/workflows/cleanup-tags.yml | 9 + .github/workflows/installer-library.yml | 310 ------------------ .../workflows/reusable-workflow-builder.yml | 57 ---- Dockerfile | 8 +- Pipfile | 2 +- Pipfile.lock | 147 ++++----- build-docker-image.sh | 81 ----- docker-builders/Dockerfile.jbig2enc | 48 --- docker-builders/Dockerfile.pikepdf | 118 ------- docker-builders/Dockerfile.psycopg2 | 66 ---- docker-builders/Dockerfile.qpdf | 156 --------- docker-builders/README.md | 57 ---- docs/development.md | 9 +- 17 files changed, 112 insertions(+), 1223 deletions(-) delete mode 100644 .build-config.json delete mode 100644 .github/scripts/common.py delete mode 100644 .github/scripts/get-build-json.py delete mode 100644 .github/workflows/installer-library.yml delete mode 100644 .github/workflows/reusable-workflow-builder.yml delete mode 100755 build-docker-image.sh delete mode 100644 docker-builders/Dockerfile.jbig2enc delete mode 100644 docker-builders/Dockerfile.pikepdf delete mode 100644 docker-builders/Dockerfile.psycopg2 delete mode 100644 docker-builders/Dockerfile.qpdf delete mode 100644 docker-builders/README.md diff --git a/.build-config.json b/.build-config.json deleted file mode 100644 index 73e8b7792..000000000 --- a/.build-config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "qpdf": { - "version": "11.3.0" - }, - "jbig2enc": { - "version": "0.29", - "git_tag": "0.29" - } -} diff --git a/.github/scripts/common.py b/.github/scripts/common.py deleted file mode 100644 index f7c91e7d7..000000000 --- a/.github/scripts/common.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging - - -def get_image_tag( - repo_name: str, - pkg_name: str, - pkg_version: str, -) -> str: - """ - Returns a string representing the normal image for a given package - """ - return f"ghcr.io/{repo_name.lower()}/builder/{pkg_name}:{pkg_version}" - - -def get_cache_image_tag( - repo_name: str, - pkg_name: str, - pkg_version: str, - branch_name: str, -) -> str: - """ - Returns a string representing the expected image cache tag for a given package - - Registry type caching is utilized for the builder images, to allow fast - rebuilds, generally almost instant for the same version - """ - return f"ghcr.io/{repo_name.lower()}/builder/cache/{pkg_name}:{pkg_version}" - - -def get_log_level(args) -> int: - """ - Returns a logging level, based - :param args: - :return: - """ - levels = { - "critical": logging.CRITICAL, - "error": logging.ERROR, - "warn": logging.WARNING, - "warning": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG, - } - level = levels.get(args.loglevel.lower()) - if level is None: - level = logging.INFO - return level diff --git a/.github/scripts/get-build-json.py b/.github/scripts/get-build-json.py deleted file mode 100644 index a37b31e90..000000000 --- a/.github/scripts/get-build-json.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -This is a helper script for the mutli-stage Docker image builder. -It provides a single point of configuration for package version control. -The output JSON object is used by the CI workflow to determine what versions -to build and pull into the final Docker image. - -Python package information is obtained from the Pipfile.lock. As this is -kept updated by dependabot, it usually will need no further configuration. -The sole exception currently is pikepdf, which has a dependency on qpdf, -and is configured here to use the latest version of qpdf built by the workflow. - -Other package version information is configured directly below, generally by -setting the version and Git information, if any. - -""" -import argparse -import json -import os -from pathlib import Path -from typing import Final - -from common import get_cache_image_tag -from common import get_image_tag - - -def _main(): - parser = argparse.ArgumentParser( - description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock", - ) - parser.add_argument( - "package", - help="The name of the package to generate JSON for", - ) - - PIPFILE_LOCK_PATH: Final[Path] = Path("Pipfile.lock") - BUILD_CONFIG_PATH: Final[Path] = Path(".build-config.json") - - # Read the main config file - build_json: Final = json.loads(BUILD_CONFIG_PATH.read_text()) - - # Read Pipfile.lock file - pipfile_data: Final = json.loads(PIPFILE_LOCK_PATH.read_text()) - - args: Final = parser.parse_args() - - # Read from environment variables set by GitHub Actions - repo_name: Final[str] = os.environ["GITHUB_REPOSITORY"] - branch_name: Final[str] = os.environ["GITHUB_REF_NAME"] - - # Default output values - version = None - extra_config = {} - - if args.package in pipfile_data["default"]: - # Read the version from Pipfile.lock - pkg_data = pipfile_data["default"][args.package] - pkg_version = pkg_data["version"].split("==")[-1] - version = pkg_version - - # Any extra/special values needed - if args.package == "pikepdf": - extra_config["qpdf_version"] = build_json["qpdf"]["version"] - - elif args.package in build_json: - version = build_json[args.package]["version"] - - else: - raise NotImplementedError(args.package) - - # The JSON object we'll output - output = { - "name": args.package, - "version": version, - "image_tag": get_image_tag(repo_name, args.package, version), - "cache_tag": get_cache_image_tag( - repo_name, - args.package, - version, - branch_name, - ), - } - - # Add anything special a package may need - output.update(extra_config) - - # Output the JSON info to stdout - print(json.dumps(output)) - - -if __name__ == "__main__": - _main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f824b04dc..ae6809c19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ on: env: # This is the version of pipenv all the steps will use # If changing this, change Dockerfile - DEFAULT_PIP_ENV_VERSION: "2023.3.20" + DEFAULT_PIP_ENV_VERSION: "2023.4.20" # This is the default version of Python to use in most steps # If changing this, change Dockerfile DEFAULT_PYTHON_VERSION: "3.9" @@ -197,90 +197,20 @@ jobs: - run: cd src-ui && npm run test - run: cd src-ui && npm run e2e:ci - prepare-docker-build: - name: Prepare Docker Pipeline Data - if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v')) - runs-on: ubuntu-22.04 - needs: - - documentation - - tests-backend - - tests-frontend - steps: - - - name: Set ghcr repository name - id: set-ghcr-repository - run: | - ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }') - echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - - - name: Setup qpdf image - id: qpdf-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf) - - echo ${build_json} - - echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT - - - name: Setup psycopg2 image - id: psycopg2-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2) - - echo ${build_json} - - echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT - - - name: Setup pikepdf image - id: pikepdf-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf) - - echo ${build_json} - - echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT - - - name: Setup jbig2enc image - id: jbig2enc-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc) - - echo ${build_json} - - echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT - - outputs: - - ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }} - - qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }} - - pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }} - - psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }} - - jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}} - - # build and push image to docker hub. build-docker-image: + name: Build Docker image for ${{ github.ref_name }} runs-on: ubuntu-22.04 + if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v')) concurrency: group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }} cancel-in-progress: true needs: - - prepare-docker-build + - tests-backend + - tests-frontend steps: - name: Check pushing to Docker Hub - id: docker-hub + id: push-other-places # Only push to Dockerhub from the main repo AND the ref is either: # main # dev @@ -288,22 +218,29 @@ jobs: # a tag # Otherwise forks would require a Docker Hub account and secrets setup run: | - if [[ ${{ needs.prepare-docker-build.outputs.ghcr-repository }} == "paperless-ngx/paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then + if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( ${{ github.ref_name }} == "main" || ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then echo "Enabling DockerHub image push" echo "enable=true" >> $GITHUB_OUTPUT else echo "Not pushing to DockerHub" echo "enable=false" >> $GITHUB_OUTPUT fi + - + name: Set ghcr repository name + id: set-ghcr-repository + run: | + ghcr_name=$(echo "${{ github.repository }}" | awk '{ print tolower($0) }') + echo "Name is ${ghcr_name}" + echo "ghcr-repository=${ghcr_name}" >> $GITHUB_OUTPUT - name: Gather Docker metadata id: docker-meta uses: docker/metadata-action@v4 with: images: | - ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }} - name=paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }} - name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.docker-hub.outputs.enable }} + ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }} + name=paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }} + name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }} tags: | # Tag branches with branch name type=ref,event=branch @@ -314,8 +251,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 + # If https://github.com/docker/buildx/issues/1044 is resolved, + # the append input with a native arm64 arch could be used to + # significantly speed up building - - name: Set up Docker Buildx + name: Set up Docker Buildx (fork/failure) uses: docker/setup-buildx-action@v2 - name: Set up QEMU @@ -331,15 +271,15 @@ jobs: name: Login to Docker Hub uses: docker/login-action@v2 # Don't attempt to login is not pushing to Docker Hub - if: steps.docker-hub.outputs.enable == 'true' + if: steps.push-other-places.outputs.enable == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Quay.io uses: docker/login-action@v2 - # Don't attempt to login is not pushing to Docker Hub - if: steps.docker-hub.outputs.enable == 'true' + # Don't attempt to login is not pushing to Quay.io + if: steps.push-other-places.outputs.enable == 'true' with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} @@ -354,19 +294,13 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} - build-args: | - JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }} - QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} - PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} # Get cache layers from this branch, then dev, then main # This allows new branches to get at least some cache benefits, generally from dev cache-from: | - type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} - type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:dev - type=registry,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:main + type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} + type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:dev cache-to: | - type=registry,mode=max,ref=ghcr.io/${{ needs.prepare-docker-build.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} + type=registry,mode=max,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} - name: Inspect image run: | diff --git a/.github/workflows/cleanup-tags.yml b/.github/workflows/cleanup-tags.yml index c850b952e..3885720d7 100644 --- a/.github/workflows/cleanup-tags.yml +++ b/.github/workflows/cleanup-tags.yml @@ -59,6 +59,15 @@ jobs: - primary-name: "paperless-ngx/builder/cache/jbig2enc" - primary-name: "paperless-ngx/builder/psycopg2" - primary-name: "paperless-ngx/builder/cache/psycopg2" + # TODO: Remove the above and replace with the below + # - primary-name: "builder/qpdf" + # - primary-name: "builder/cache/qpdf" + # - primary-name: "builder/pikepdf" + # - primary-name: "builder/cache/pikepdf" + # - primary-name: "builder/jbig2enc" + # - primary-name: "builder/cache/jbig2enc" + # - primary-name: "builder/psycopg2" + # - primary-name: "builder/cache/psycopg2" env: # Requires a personal access token with the OAuth scope delete:packages TOKEN: ${{ secrets.GHA_CONTAINER_DELETE_TOKEN }} diff --git a/.github/workflows/installer-library.yml b/.github/workflows/installer-library.yml deleted file mode 100644 index 56064ad86..000000000 --- a/.github/workflows/installer-library.yml +++ /dev/null @@ -1,310 +0,0 @@ -# This workflow will run to update the installer library of -# Docker images. These are the images which provide updated wheels -# .deb installation packages or maybe just some compiled library - -name: Build Image Library - -on: - push: - # Must match one of these branches AND one of the paths - # to be triggered - branches: - - "main" - - "dev" - - "library-*" - - "feature-*" - paths: - # Trigger the workflow if a Dockerfile changed - - "docker-builders/**" - # Trigger if a package was updated - - ".build-config.json" - - "Pipfile.lock" - # Also trigger on workflow changes related to the library - - ".github/workflows/installer-library.yml" - - ".github/workflows/reusable-workflow-builder.yml" - - ".github/scripts/**" - -# Set a workflow level concurrency group so primary workflow -# can wait for this to complete if needed -# DO NOT CHANGE without updating main workflow group -concurrency: - group: build-installer-library - cancel-in-progress: false - -jobs: - prepare-docker-build: - name: Prepare Docker Image Version Data - runs-on: ubuntu-22.04 - steps: - - - name: Set ghcr repository name - id: set-ghcr-repository - run: | - ghcr_name=$(echo "${GITHUB_REPOSITORY}" | awk '{ print tolower($0) }') - echo "repository=${ghcr_name}" >> $GITHUB_OUTPUT - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install jq - run: | - sudo apt-get update - sudo apt-get install jq - - - name: Setup qpdf image - id: qpdf-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py qpdf) - - echo ${build_json} - - echo "qpdf-json=${build_json}" >> $GITHUB_OUTPUT - - - name: Setup psycopg2 image - id: psycopg2-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py psycopg2) - - echo ${build_json} - - echo "psycopg2-json=${build_json}" >> $GITHUB_OUTPUT - - - name: Setup pikepdf image - id: pikepdf-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py pikepdf) - - echo ${build_json} - - echo "pikepdf-json=${build_json}" >> $GITHUB_OUTPUT - - - name: Setup jbig2enc image - id: jbig2enc-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py jbig2enc) - - echo ${build_json} - - echo "jbig2enc-json=${build_json}" >> $GITHUB_OUTPUT - - - name: Setup other versions - id: cache-bust-setup - run: | - pillow_version=$(jq -r '.default.pillow.version | gsub("=";"")' Pipfile.lock) - lxml_version=$(jq -r '.default.lxml.version | gsub("=";"")' Pipfile.lock) - - echo "Pillow is ${pillow_version}" - echo "lxml is ${lxml_version}" - - echo "pillow-version=${pillow_version}" >> $GITHUB_OUTPUT - echo "lxml-version=${lxml_version}" >> $GITHUB_OUTPUT - - outputs: - - ghcr-repository: ${{ steps.set-ghcr-repository.outputs.repository }} - - qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }} - - pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }} - - psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }} - - jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json }} - - pillow-version: ${{ steps.cache-bust-setup.outputs.pillow-version }} - - lxml-version: ${{ steps.cache-bust-setup.outputs.lxml-version }} - - build-qpdf-debs: - name: qpdf - needs: - - prepare-docker-build - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.qpdf - build-platforms: linux/amd64 - build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }} - build-args: | - QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - - build-jbig2enc: - name: jbig2enc - needs: - - prepare-docker-build - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.jbig2enc - build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }} - build-args: | - JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }} - - build-psycopg2-wheel: - name: psycopg2 - needs: - - prepare-docker-build - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.psycopg2 - build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }} - build-args: | - PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} - - build-pikepdf-wheel: - name: pikepdf - needs: - - prepare-docker-build - - build-qpdf-debs - uses: ./.github/workflows/reusable-workflow-builder.yml - with: - dockerfile: ./docker-builders/Dockerfile.pikepdf - build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }} - build-args: | - REPO=${{ needs.prepare-docker-build.outputs.ghcr-repository }} - QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} - PILLOW_VERSION=${{ needs.prepare-docker-build.outputs.pillow-version }} - LXML_VERSION=${{ needs.prepare-docker-build.outputs.lxml-version }} - - commit-binary-files: - name: Store installers - needs: - - prepare-docker-build - - build-qpdf-debs - - build-jbig2enc - - build-psycopg2-wheel - - build-pikepdf-wheel - runs-on: ubuntu-22.04 - steps: - - - name: Checkout - uses: actions/checkout@v3 - with: - ref: binary-library - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install system dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -qq --no-install-recommends tree - - - name: Extract qpdf files - run: | - version=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} - tag=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }} - - docker pull --quiet ${tag} - docker create --name qpdf-extract ${tag} - - mkdir --parents qpdf/${version}/amd64 - docker cp qpdf-extract:/usr/src/qpdf/${version}/amd64 qpdf/${version} - - mkdir --parents qpdf/${version}/arm64 - docker cp qpdf-extract:/usr/src/qpdf/${version}/arm64 qpdf/${version} - - mkdir --parents qpdf/${version}/armv7 - docker cp qpdf-extract:/usr/src/qpdf/${version}/armv7 qpdf/${version} - - - name: Extract psycopg2 files - run: | - version=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} - tag=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }} - - docker pull --quiet --platform linux/amd64 ${tag} - docker create --platform linux/amd64 --name psycopg2-extract ${tag} - mkdir --parents psycopg2/${version}/amd64 - docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/amd64 - mv psycopg2/${version}/amd64/wheels/* psycopg2/${version}/amd64 - rm -r psycopg2/${version}/amd64/wheels/ - docker rm psycopg2-extract - - docker pull --quiet --platform linux/arm64 ${tag} - docker create --platform linux/arm64 --name psycopg2-extract ${tag} - mkdir --parents psycopg2/${version}/arm64 - docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/arm64 - mv psycopg2/${version}/arm64/wheels/* psycopg2/${version}/arm64 - rm -r psycopg2/${version}/arm64/wheels/ - docker rm psycopg2-extract - - docker pull --quiet --platform linux/arm/v7 ${tag} - docker create --platform linux/arm/v7 --name psycopg2-extract ${tag} - mkdir --parents psycopg2/${version}/armv7 - docker cp psycopg2-extract:/usr/src/wheels/ psycopg2/${version}/armv7 - mv psycopg2/${version}/armv7/wheels/* psycopg2/${version}/armv7 - rm -r psycopg2/${version}/armv7/wheels/ - docker rm psycopg2-extract - - - name: Extract pikepdf files - run: | - version=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} - tag=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }} - - docker pull --quiet --platform linux/amd64 ${tag} - docker create --platform linux/amd64 --name pikepdf-extract ${tag} - mkdir --parents pikepdf/${version}/amd64 - docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/amd64 - mv pikepdf/${version}/amd64/wheels/* pikepdf/${version}/amd64 - rm -r pikepdf/${version}/amd64/wheels/ - docker rm pikepdf-extract - - docker pull --quiet --platform linux/arm64 ${tag} - docker create --platform linux/arm64 --name pikepdf-extract ${tag} - mkdir --parents pikepdf/${version}/arm64 - docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/arm64 - mv pikepdf/${version}/arm64/wheels/* pikepdf/${version}/arm64 - rm -r pikepdf/${version}/arm64/wheels/ - docker rm pikepdf-extract - - docker pull --quiet --platform linux/arm/v7 ${tag} - docker create --platform linux/arm/v7 --name pikepdf-extract ${tag} - mkdir --parents pikepdf/${version}/armv7 - docker cp pikepdf-extract:/usr/src/wheels/ pikepdf/${version}/armv7 - mv pikepdf/${version}/armv7/wheels/* pikepdf/${version}/armv7 - rm -r pikepdf/${version}/armv7/wheels/ - docker rm pikepdf-extract - - - name: Extract jbig2enc files - run: | - version=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }} - tag=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }} - - docker pull --quiet --platform linux/amd64 ${tag} - docker create --platform linux/amd64 --name jbig2enc-extract ${tag} - mkdir --parents jbig2enc/${version}/amd64 - docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/amd64/ - mv jbig2enc/${version}/amd64/build/* jbig2enc/${version}/amd64/ - docker rm jbig2enc-extract - - docker pull --quiet --platform linux/arm64 ${tag} - docker create --platform linux/arm64 --name jbig2enc-extract ${tag} - mkdir --parents jbig2enc/${version}/arm64 - docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/arm64 - mv jbig2enc/${version}/arm64/build/* jbig2enc/${version}/arm64/ - docker rm jbig2enc-extract - - docker pull --quiet --platform linux/arm/v7 ${tag} - docker create --platform linux/arm/v7 --name jbig2enc-extract ${tag} - mkdir --parents jbig2enc/${version}/armv7 - docker cp jbig2enc-extract:/usr/src/jbig2enc/build jbig2enc/${version}/armv7 - mv jbig2enc/${version}/armv7/build/* jbig2enc/${version}/armv7/ - docker rm jbig2enc-extract - - - name: Show file structure - run: | - tree . - - - name: Commit files - run: | - git config --global user.name "github-actions" - git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add pikepdf/ qpdf/ psycopg2/ jbig2enc/ - git commit -m "Updating installer packages" || true - git push origin || true diff --git a/.github/workflows/reusable-workflow-builder.yml b/.github/workflows/reusable-workflow-builder.yml deleted file mode 100644 index 012e42b91..000000000 --- a/.github/workflows/reusable-workflow-builder.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Reusable Image Builder - -on: - workflow_call: - inputs: - dockerfile: - required: true - type: string - build-json: - required: true - type: string - build-args: - required: false - default: "" - type: string - build-platforms: - required: false - default: linux/amd64,linux/arm64,linux/arm/v7 - type: string - -concurrency: - group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }} - cancel-in-progress: false - -jobs: - build-image: - name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }} - runs-on: ubuntu-22.04 - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Login to Github Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Build ${{ fromJSON(inputs.build-json).name }} - uses: docker/build-push-action@v4 - with: - context: . - file: ${{ inputs.dockerfile }} - tags: ${{ fromJSON(inputs.build-json).image_tag }} - platforms: ${{ inputs.build-platforms }} - build-args: ${{ inputs.build-args }} - push: true - cache-from: type=registry,ref=${{ fromJSON(inputs.build-json).cache_tag }} - cache-to: type=registry,mode=max,ref=${{ fromJSON(inputs.build-json).cache_tag }} diff --git a/Dockerfile b/Dockerfile index 1d3def67a..88b5d7ad4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN set -eux \ # Comments: # - pipenv dependencies are not left in the final image # - pipenv can't touch the final image somehow -FROM --platform=$BUILDPLATFORM python:3.9-slim-bullseye as pipenv-base +FROM --platform=$BUILDPLATFORM python:3.9-alpine as pipenv-base WORKDIR /usr/src/pipenv @@ -29,7 +29,7 @@ COPY Pipfile* ./ RUN set -eux \ && echo "Installing pipenv" \ - && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.3.20 \ + && python3 -m pip install --no-cache-dir --upgrade pipenv==2023.4.20 \ && echo "Generating requirement.txt" \ && pipenv requirements > requirements.txt @@ -170,7 +170,7 @@ RUN set -eux \ ARG TARGETARCH ARG TARGETVARIANT -# Workflow provided, defaults set for manual building +# Can be workflow provided, defaults set for manual building ARG JBIG2ENC_VERSION=0.29 ARG QPDF_VERSION=11.3.0 ARG PIKEPDF_VERSION=7.1.1 @@ -181,7 +181,7 @@ ARG PSYCOPG2_VERSION=2.9.5 RUN set -eux \ && echo "Getting binaries" \ && mkdir paperless-ngx \ - && curl --fail --silent --show-error --output paperless-ngx.tar.gz --location https://github.com/paperless-ngx/paperless-ngx/archive/ba28a1e16c27d121b644b4f6bdb78855a2850561.tar.gz \ + && curl --fail --silent --show-error --output paperless-ngx.tar.gz --location https://github.com/paperless-ngx/builder/archive/e04666c257ef648716381a99113856f599e66911.tar.gz \ && tar -xf paperless-ngx.tar.gz --directory paperless-ngx --strip-components=1 \ && cd paperless-ngx \ # Setting a specific revision ensures we know what this installed diff --git a/Pipfile b/Pipfile index 1308dc2a3..d1471f52c 100644 --- a/Pipfile +++ b/Pipfile @@ -25,7 +25,7 @@ gunicorn = "*" imap-tools = "*" langdetect = "*" pathvalidate = "*" -pillow = "~=9.4" +pillow = "*" pikepdf = "*" python-gnupg = "*" python-dotenv = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 39805737f..1831d894c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "77248fee6dad10b9e5189e9ba80f7c506c9f49c875bac8b259e90dadecba03f1" + "sha256": "5a1fda76f86696709eca15c024959540ea9ce860b9d67f401e3e3aa7b01eca34" }, "pipfile-spec": 6, "requires": {}, @@ -1125,86 +1125,75 @@ }, "pillow": { "hashes": [ - "sha256:013016af6b3a12a2f40b704677f8b51f72cb007dac785a9933d5c86a72a7fe33", - "sha256:0845adc64fe9886db00f5ab68c4a8cd933ab749a87747555cec1c95acea64b0b", - "sha256:0884ba7b515163a1a05440a138adeb722b8a6ae2c2b33aea93ea3118dd3a899e", - "sha256:09b89ddc95c248ee788328528e6a2996e09eaccddeeb82a5356e92645733be35", - "sha256:0dd4c681b82214b36273c18ca7ee87065a50e013112eea7d78c7a1b89a739153", - "sha256:0e51f608da093e5d9038c592b5b575cadc12fd748af1479b5e858045fff955a9", - "sha256:0f3269304c1a7ce82f1759c12ce731ef9b6e95b6df829dccd9fe42912cc48569", - "sha256:16a8df99701f9095bea8a6c4b3197da105df6f74e6176c5b410bc2df2fd29a57", - "sha256:19005a8e58b7c1796bc0167862b1f54a64d3b44ee5d48152b06bb861458bc0f8", - "sha256:1b4b4e9dda4f4e4c4e6896f93e84a8f0bcca3b059de9ddf67dac3c334b1195e1", - "sha256:28676836c7796805914b76b1837a40f76827ee0d5398f72f7dcc634bae7c6264", - "sha256:2968c58feca624bb6c8502f9564dd187d0e1389964898f5e9e1fbc8533169157", - "sha256:3f4cc516e0b264c8d4ccd6b6cbc69a07c6d582d8337df79be1e15a5056b258c9", - "sha256:3fa1284762aacca6dc97474ee9c16f83990b8eeb6697f2ba17140d54b453e133", - "sha256:43521ce2c4b865d385e78579a082b6ad1166ebed2b1a2293c3be1d68dd7ca3b9", - "sha256:451f10ef963918e65b8869e17d67db5e2f4ab40e716ee6ce7129b0cde2876eab", - "sha256:46c259e87199041583658457372a183636ae8cd56dbf3f0755e0f376a7f9d0e6", - "sha256:46f39cab8bbf4a384ba7cb0bc8bae7b7062b6a11cfac1ca4bc144dea90d4a9f5", - "sha256:519e14e2c49fcf7616d6d2cfc5c70adae95682ae20f0395e9280db85e8d6c4df", - "sha256:53dcb50fbdc3fb2c55431a9b30caeb2f7027fcd2aeb501459464f0214200a503", - "sha256:54614444887e0d3043557d9dbc697dbb16cfb5a35d672b7a0fcc1ed0cf1c600b", - "sha256:575d8912dca808edd9acd6f7795199332696d3469665ef26163cd090fa1f8bfa", - "sha256:5dd5a9c3091a0f414a963d427f920368e2b6a4c2f7527fdd82cde8ef0bc7a327", - "sha256:5f532a2ad4d174eb73494e7397988e22bf427f91acc8e6ebf5bb10597b49c493", - "sha256:60e7da3a3ad1812c128750fc1bc14a7ceeb8d29f77e0a2356a8fb2aa8925287d", - "sha256:653d7fb2df65efefbcbf81ef5fe5e5be931f1ee4332c2893ca638c9b11a409c4", - "sha256:6663977496d616b618b6cfa43ec86e479ee62b942e1da76a2c3daa1c75933ef4", - "sha256:6abfb51a82e919e3933eb137e17c4ae9c0475a25508ea88993bb59faf82f3b35", - "sha256:6c6b1389ed66cdd174d040105123a5a1bc91d0aa7059c7261d20e583b6d8cbd2", - "sha256:6d9dfb9959a3b0039ee06c1a1a90dc23bac3b430842dcb97908ddde05870601c", - "sha256:765cb54c0b8724a7c12c55146ae4647e0274a839fb6de7bcba841e04298e1011", - "sha256:7a21222644ab69ddd9967cfe6f2bb420b460dae4289c9d40ff9a4896e7c35c9a", - "sha256:7ac7594397698f77bce84382929747130765f66406dc2cd8b4ab4da68ade4c6e", - "sha256:7cfc287da09f9d2a7ec146ee4d72d6ea1342e770d975e49a8621bf54eaa8f30f", - "sha256:83125753a60cfc8c412de5896d10a0a405e0bd88d0470ad82e0869ddf0cb3848", - "sha256:847b114580c5cc9ebaf216dd8c8dbc6b00a3b7ab0131e173d7120e6deade1f57", - "sha256:87708d78a14d56a990fbf4f9cb350b7d89ee8988705e58e39bdf4d82c149210f", - "sha256:8a2b5874d17e72dfb80d917213abd55d7e1ed2479f38f001f264f7ce7bae757c", - "sha256:8f127e7b028900421cad64f51f75c051b628db17fb00e099eb148761eed598c9", - "sha256:94cdff45173b1919350601f82d61365e792895e3c3a3443cf99819e6fbf717a5", - "sha256:99d92d148dd03fd19d16175b6d355cc1b01faf80dae93c6c3eb4163709edc0a9", - "sha256:9a3049a10261d7f2b6514d35bbb7a4dfc3ece4c4de14ef5876c4b7a23a0e566d", - "sha256:9d9a62576b68cd90f7075876f4e8444487db5eeea0e4df3ba298ee38a8d067b0", - "sha256:9e5f94742033898bfe84c93c831a6f552bb629448d4072dd312306bab3bd96f1", - "sha256:a1c2d7780448eb93fbcc3789bf3916aa5720d942e37945f4056680317f1cd23e", - "sha256:a2e0f87144fcbbe54297cae708c5e7f9da21a4646523456b00cc956bd4c65815", - "sha256:a4dfdae195335abb4e89cc9762b2edc524f3c6e80d647a9a81bf81e17e3fb6f0", - "sha256:a96e6e23f2b79433390273eaf8cc94fec9c6370842e577ab10dabdcc7ea0a66b", - "sha256:aabdab8ec1e7ca7f1434d042bf8b1e92056245fb179790dc97ed040361f16bfd", - "sha256:b222090c455d6d1a64e6b7bb5f4035c4dff479e22455c9eaa1bdd4c75b52c80c", - "sha256:b52ff4f4e002f828ea6483faf4c4e8deea8d743cf801b74910243c58acc6eda3", - "sha256:b70756ec9417c34e097f987b4d8c510975216ad26ba6e57ccb53bc758f490dab", - "sha256:b8c2f6eb0df979ee99433d8b3f6d193d9590f735cf12274c108bd954e30ca858", - "sha256:b9b752ab91e78234941e44abdecc07f1f0d8f51fb62941d32995b8161f68cfe5", - "sha256:ba6612b6548220ff5e9df85261bddc811a057b0b465a1226b39bfb8550616aee", - "sha256:bd752c5ff1b4a870b7661234694f24b1d2b9076b8bf337321a814c612665f343", - "sha256:c3c4ed2ff6760e98d262e0cc9c9a7f7b8a9f61aa4d47c58835cdaf7b0b8811bb", - "sha256:c5c1362c14aee73f50143d74389b2c158707b4abce2cb055b7ad37ce60738d47", - "sha256:cb362e3b0976dc994857391b776ddaa8c13c28a16f80ac6522c23d5257156bed", - "sha256:d197df5489004db87d90b918033edbeee0bd6df3848a204bca3ff0a903bef837", - "sha256:d3b56206244dc8711f7e8b7d6cad4663917cd5b2d950799425076681e8766286", - "sha256:d5b2f8a31bd43e0f18172d8ac82347c8f37ef3e0b414431157718aa234991b28", - "sha256:d7081c084ceb58278dd3cf81f836bc818978c0ccc770cbbb202125ddabec6628", - "sha256:db74f5562c09953b2c5f8ec4b7dfd3f5421f31811e97d1dbc0a7c93d6e3a24df", - "sha256:df41112ccce5d47770a0c13651479fbcd8793f34232a2dd9faeccb75eb5d0d0d", - "sha256:e1339790c083c5a4de48f688b4841f18df839eb3c9584a770cbd818b33e26d5d", - "sha256:e621b0246192d3b9cb1dc62c78cfa4c6f6d2ddc0ec207d43c0dedecb914f152a", - "sha256:e8c5cf126889a4de385c02a2c3d3aba4b00f70234bfddae82a5eaa3ee6d5e3e6", - "sha256:e9d7747847c53a16a729b6ee5e737cf170f7a16611c143d95aa60a109a59c336", - "sha256:eaef5d2de3c7e9b21f1e762f289d17b726c2239a42b11e25446abf82b26ac132", - "sha256:ed3e4b4e1e6de75fdc16d3259098de7c6571b1a6cc863b1a49e7d3d53e036070", - "sha256:ef21af928e807f10bf4141cad4746eee692a0dd3ff56cfb25fce076ec3cc8abe", - "sha256:f09598b416ba39a8f489c124447b007fe865f786a89dbfa48bb5cf395693132a", - "sha256:f0caf4a5dcf610d96c3bd32932bfac8aee61c96e60481c2a0ea58da435e25acd", - "sha256:f6e78171be3fb7941f9910ea15b4b14ec27725865a73c15277bc39f5ca4f8391", - "sha256:f715c32e774a60a337b2bb8ad9839b4abf75b267a0f18806f6f4f5f1688c4b5a", - "sha256:fb5c1ad6bad98c57482236a21bf985ab0ef42bd51f7ad4e4538e89a997624e12" + "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1", + "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba", + "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a", + "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799", + "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51", + "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb", + "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5", + "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270", + "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6", + "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47", + "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf", + "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e", + "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b", + "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66", + "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865", + "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec", + "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c", + "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1", + "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38", + "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906", + "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705", + "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef", + "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc", + "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f", + "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf", + "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392", + "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d", + "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe", + "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32", + "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5", + "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7", + "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44", + "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d", + "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3", + "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625", + "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e", + "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829", + "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089", + "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3", + "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78", + "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96", + "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964", + "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597", + "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99", + "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a", + "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140", + "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7", + "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16", + "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903", + "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1", + "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296", + "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572", + "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115", + "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a", + "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd", + "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4", + "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1", + "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb", + "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa", + "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a", + "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569", + "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c", + "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf", + "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082", + "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062", + "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579" ], "index": "pypi", - "version": "==9.4.0" + "version": "==9.5.0" }, "pluggy": { "hashes": [ diff --git a/build-docker-image.sh b/build-docker-image.sh deleted file mode 100755 index 01e2251a0..000000000 --- a/build-docker-image.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# Helper script for building the Docker image locally. -# Parses and provides the nessecary versions of other images to Docker -# before passing in the rest of script args. - -# First Argument: The Dockerfile to build -# Other Arguments: Additional arguments to docker build - -# Example Usage: -# ./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature - -set -eu - -if ! command -v jq &> /dev/null ; then - echo "jq required" - exit 1 -elif [ ! -f "$1" ]; then - echo "$1 is not a file, please provide the Dockerfile" - exit 1 -fi - -# Get the branch name (used for caching) -branch_name=$(git rev-parse --abbrev-ref HEAD) - -# Parse eithe Pipfile.lock or the .build-config.json -jbig2enc_version=$(jq -r '.jbig2enc.version' .build-config.json) -qpdf_version=$(jq -r '.qpdf.version' .build-config.json) -psycopg2_version=$(jq -r '.default.psycopg2.version | gsub("=";"")' Pipfile.lock) -pikepdf_version=$(jq -r '.default.pikepdf.version | gsub("=";"")' Pipfile.lock) -pillow_version=$(jq -r '.default.pillow.version | gsub("=";"")' Pipfile.lock) -lxml_version=$(jq -r '.default.lxml.version | gsub("=";"")' Pipfile.lock) - -base_filename="$(basename -- "${1}")" -build_args_str="" -cache_from_str="" - -case "${base_filename}" in - - *.jbig2enc) - build_args_str="--build-arg JBIG2ENC_VERSION=${jbig2enc_version}" - cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/jbig2enc:${jbig2enc_version}" - ;; - - *.psycopg2) - build_args_str="--build-arg PSYCOPG2_VERSION=${psycopg2_version}" - cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/psycopg2:${psycopg2_version}" - ;; - - *.qpdf) - build_args_str="--build-arg QPDF_VERSION=${qpdf_version}" - cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/qpdf:${qpdf_version}" - ;; - - *.pikepdf) - build_args_str="--build-arg QPDF_VERSION=${qpdf_version} --build-arg PIKEPDF_VERSION=${pikepdf_version} --build-arg PILLOW_VERSION=${pillow_version} --build-arg LXML_VERSION=${lxml_version}" - cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/pikepdf:${pikepdf_version}" - ;; - - Dockerfile) - build_args_str="--build-arg QPDF_VERSION=${qpdf_version} --build-arg PIKEPDF_VERSION=${pikepdf_version} --build-arg PSYCOPG2_VERSION=${psycopg2_version} --build-arg JBIG2ENC_VERSION=${jbig2enc_version}" - cache_from_str="--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:${branch_name} --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev" - ;; - - *) - echo "Unable to match ${base_filename}" - exit 1 - ;; -esac - -read -r -a build_args_arr <<< "${build_args_str}" -read -r -a cache_from_arr <<< "${cache_from_str}" - -set -eux - -docker buildx build --file "${1}" \ - --progress=plain \ - --output=type=docker \ - "${cache_from_arr[@]}" \ - "${build_args_arr[@]}" \ - "${@:2}" . diff --git a/docker-builders/Dockerfile.jbig2enc b/docker-builders/Dockerfile.jbig2enc deleted file mode 100644 index 388bdd1f7..000000000 --- a/docker-builders/Dockerfile.jbig2enc +++ /dev/null @@ -1,48 +0,0 @@ -# This Dockerfile compiles the jbig2enc library -# Inputs: -# - JBIG2ENC_VERSION - the Git tag to checkout and build - -FROM debian:bullseye-slim as main - -LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built" - -ARG DEBIAN_FRONTEND=noninteractive -ARG JBIG2ENC_VERSION - -ARG BUILD_PACKAGES="\ - build-essential \ - automake \ - libtool \ - libleptonica-dev \ - zlib1g-dev \ - git \ - ca-certificates" - -WORKDIR /usr/src/jbig2enc - -RUN set -eux \ - && echo "Installing build tools" \ - && apt-get update --quiet \ - && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \ - && echo "Building jbig2enc" \ - && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc . \ - && ./autogen.sh \ - && ./configure \ - && make \ - && echo "Gathering package data" \ - && dpkg-query -f '${Package;-40}${Version}\n' -W > ./pkg-list.txt \ - && echo "Cleaning up image" \ - && apt-get -y purge ${BUILD_PACKAGES} \ - && apt-get -y autoremove --purge \ - && rm -rf /var/lib/apt/lists/* \ - && echo "Moving files around" \ - && mkdir build \ - # Unlink a symlink that causes problems - && unlink ./src/.libs/libjbig2enc.la \ - # Move what the link pointed to - && mv ./src/libjbig2enc.la ./build/ \ - # Move the shared library .so files - && mv ./src/.libs/libjbig2enc* ./build/ \ - # And move the cli binary - && mv ./src/jbig2 ./build/ \ - && mv ./pkg-list.txt ./build/ diff --git a/docker-builders/Dockerfile.pikepdf b/docker-builders/Dockerfile.pikepdf deleted file mode 100644 index e4181c538..000000000 --- a/docker-builders/Dockerfile.pikepdf +++ /dev/null @@ -1,118 +0,0 @@ -# This Dockerfile builds the pikepdf wheel -# Inputs: -# - REPO - Docker repository to pull qpdf from -# - QPDF_VERSION - The image qpdf version to copy .deb files from -# - PIKEPDF_VERSION - Version of pikepdf to build wheel for - -# Default to pulling from the main repo registry when manually building -ARG REPO="paperless-ngx/paperless-ngx" - -# This does nothing, except provide a name for a copy below -ARG QPDF_VERSION -FROM --platform=$BUILDPLATFORM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder - -# -# Stage: builder -# Purpose: -# - Build the pikepdf wheel -# - Build any dependent wheels which can't be found -# -FROM python:3.9-slim-bullseye as builder - -LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built" - -# Buildx provided -ARG TARGETARCH -ARG TARGETVARIANT - -ARG DEBIAN_FRONTEND=noninteractive -# Workflow provided -ARG QPDF_VERSION -ARG PIKEPDF_VERSION -# These are not used, but will still bust the cache if one changes -# Otherwise, the main image will try to build thing (and fail) -ARG PILLOW_VERSION -ARG LXML_VERSION - -ARG BUILD_PACKAGES="\ - build-essential \ - python3-dev \ - python3-pip \ - # qpdf requirement - https://github.com/qpdf/qpdf#crypto-providers - libgnutls28-dev \ - # lxml requrements - https://lxml.de/installation.html - libxml2-dev \ - libxslt1-dev \ - # Pillow requirements - https://pillow.readthedocs.io/en/stable/installation.html#external-libraries - # JPEG functionality - libjpeg62-turbo-dev \ - # conpressed PNG - zlib1g-dev \ - # compressed TIFF - libtiff-dev \ - # type related services - libfreetype-dev \ - # color management - liblcms2-dev \ - # WebP format - libwebp-dev \ - # JPEG 2000 - libopenjp2-7-dev \ - # improved color quantization - libimagequant-dev \ - # complex text layout support - libraqm-dev" - -WORKDIR /usr/src - -COPY --from=qpdf-builder /usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/*.deb ./ - -# As this is an base image for a multi-stage final image -# the added size of the install is basically irrelevant - -RUN set -eux \ - && echo "Installing build tools" \ - && apt-get update --quiet \ - && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \ - && echo "Installing qpdf" \ - && dpkg --install libqpdf29_*.deb \ - && dpkg --install libqpdf-dev_*.deb \ - && echo "Installing Python tools" \ - && python3 -m pip install --no-cache-dir --upgrade \ - pip \ - wheel \ - # https://pikepdf.readthedocs.io/en/latest/installation.html#requirements - pybind11 \ - && echo "Building pikepdf wheel ${PIKEPDF_VERSION}" \ - && mkdir wheels \ - && python3 -m pip wheel \ - # Build the package at the required version - pikepdf==${PIKEPDF_VERSION} \ - # Look to piwheels for additional pre-built wheels - --extra-index-url https://www.piwheels.org/simple \ - # Output the *.whl into this directory - --wheel-dir wheels \ - # Do not use a binary packge for the package being built - --no-binary=pikepdf \ - # Do use binary packages for dependencies - --prefer-binary \ - # Don't cache build files - --no-cache-dir \ - && ls -ahl wheels \ - && echo "Gathering package data" \ - && dpkg-query -f '${Package;-40}${Version}\n' -W > ./wheels/pkg-list.txt \ - && echo "Cleaning up image" \ - && apt-get -y purge ${BUILD_PACKAGES} \ - && apt-get -y autoremove --purge \ - && rm -rf /var/lib/apt/lists/* - -# -# Stage: package -# Purpose: Holds the compiled .whl files in a tiny image to pull -# -FROM alpine:3.17 as package - -WORKDIR /usr/src/wheels/ - -COPY --from=builder /usr/src/wheels/*.whl ./ -COPY --from=builder /usr/src/wheels/pkg-list.txt ./ diff --git a/docker-builders/Dockerfile.psycopg2 b/docker-builders/Dockerfile.psycopg2 deleted file mode 100644 index e3f182435..000000000 --- a/docker-builders/Dockerfile.psycopg2 +++ /dev/null @@ -1,66 +0,0 @@ -# This Dockerfile builds the psycopg2 wheel -# Inputs: -# - PSYCOPG2_VERSION - Version to build - -# -# Stage: builder -# Purpose: -# - Build the psycopg2 wheel -# -FROM python:3.9-slim-bullseye as builder - -LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built" - -ARG PSYCOPG2_VERSION -ARG DEBIAN_FRONTEND=noninteractive - -ARG BUILD_PACKAGES="\ - build-essential \ - python3-dev \ - python3-pip \ - # https://www.psycopg.org/docs/install.html#prerequisites - libpq-dev" - -WORKDIR /usr/src - -# As this is an base image for a multi-stage final image -# the added size of the install is basically irrelevant - -RUN set -eux \ - && echo "Installing build tools" \ - && apt-get update --quiet \ - && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \ - && echo "Installing Python tools" \ - && python3 -m pip install --no-cache-dir --upgrade pip wheel \ - && echo "Building psycopg2 wheel ${PSYCOPG2_VERSION}" \ - && cd /usr/src \ - && mkdir wheels \ - && python3 -m pip wheel \ - # Build the package at the required version - psycopg2==${PSYCOPG2_VERSION} \ - # Output the *.whl into this directory - --wheel-dir wheels \ - # Do not use a binary packge for the package being built - --no-binary=psycopg2 \ - # Do use binary packages for dependencies - --prefer-binary \ - # Don't cache build files - --no-cache-dir \ - && ls -ahl wheels/ \ - && echo "Gathering package data" \ - && dpkg-query -f '${Package;-40}${Version}\n' -W > ./wheels/pkg-list.txt \ - && echo "Cleaning up image" \ - && apt-get -y purge ${BUILD_PACKAGES} \ - && apt-get -y autoremove --purge \ - && rm -rf /var/lib/apt/lists/* - -# -# Stage: package -# Purpose: Holds the compiled .whl files in a tiny image to pull -# -FROM alpine:3.17 as package - -WORKDIR /usr/src/wheels/ - -COPY --from=builder /usr/src/wheels/*.whl ./ -COPY --from=builder /usr/src/wheels/pkg-list.txt ./ diff --git a/docker-builders/Dockerfile.qpdf b/docker-builders/Dockerfile.qpdf deleted file mode 100644 index 3ad11d8b1..000000000 --- a/docker-builders/Dockerfile.qpdf +++ /dev/null @@ -1,156 +0,0 @@ -# -# Stage: pre-build -# Purpose: -# - Installs common packages -# - Sets common environment variables related to dpkg -# - Aquires the qpdf source from bookwork -# Useful Links: -# - https://qpdf.readthedocs.io/en/stable/installation.html#system-requirements -# - https://wiki.debian.org/Multiarch/HOWTO -# - https://wiki.debian.org/CrossCompiling -# - -FROM debian:bullseye-slim as pre-build - -ARG QPDF_VERSION - -ARG COMMON_BUILD_PACKAGES="\ - cmake \ - debhelper\ - debian-keyring \ - devscripts \ - dpkg-dev \ - equivs \ - packaging-dev \ - libtool" - -ENV DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" - -WORKDIR /usr/src - -RUN set -eux \ - && echo "Installing common packages" \ - && apt-get update --quiet \ - && apt-get install --yes --quiet --no-install-recommends ${COMMON_BUILD_PACKAGES} \ - && echo "Getting qpdf source" \ - && echo "deb-src http://deb.debian.org/debian/ bookworm main" > /etc/apt/sources.list.d/bookworm-src.list \ - && apt-get update --quiet \ - && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm - -# -# Stage: amd64-builder -# Purpose: Builds qpdf for x86_64 (native build) -# -FROM pre-build as amd64-builder - -ARG AMD64_BUILD_PACKAGES="\ - build-essential \ - libjpeg62-turbo-dev:amd64 \ - libgnutls28-dev:amd64 \ - zlib1g-dev:amd64" - -WORKDIR /usr/src/qpdf-${QPDF_VERSION} - -RUN set -eux \ - && echo "Beginning amd64" \ - && echo "Install amd64 packages" \ - && apt-get update --quiet \ - && apt-get install --yes --quiet --no-install-recommends ${AMD64_BUILD_PACKAGES} \ - && echo "Building amd64" \ - && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean \ - && echo "Removing debug files" \ - && rm -f ../libqpdf29-dbgsym* \ - && rm -f ../qpdf-dbgsym* \ - && echo "Gathering package data" \ - && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt -# -# Stage: armhf-builder -# Purpose: -# - Sets armhf specific environment -# - Builds qpdf for armhf (cross compile) -# -FROM pre-build as armhf-builder - -ARG ARMHF_PACKAGES="\ - crossbuild-essential-armhf \ - libjpeg62-turbo-dev:armhf \ - libgnutls28-dev:armhf \ - zlib1g-dev:armhf" - -WORKDIR /usr/src/qpdf-${QPDF_VERSION} - -ENV CXX="/usr/bin/arm-linux-gnueabihf-g++" \ - CC="/usr/bin/arm-linux-gnueabihf-gcc" - -RUN set -eux \ - && echo "Beginning armhf" \ - && echo "Install armhf packages" \ - && dpkg --add-architecture armhf \ - && apt-get update --quiet \ - && apt-get install --yes --quiet --no-install-recommends ${ARMHF_PACKAGES} \ - && echo "Building armhf" \ - && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean --host-arch armhf \ - && echo "Removing debug files" \ - && rm -f ../libqpdf29-dbgsym* \ - && rm -f ../qpdf-dbgsym* \ - && echo "Gathering package data" \ - && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt - -# -# Stage: aarch64-builder -# Purpose: -# - Sets aarch64 specific environment -# - Builds qpdf for aarch64 (cross compile) -# -FROM pre-build as aarch64-builder - -ARG ARM64_PACKAGES="\ - crossbuild-essential-arm64 \ - libjpeg62-turbo-dev:arm64 \ - libgnutls28-dev:arm64 \ - zlib1g-dev:arm64" - -ENV CXX="/usr/bin/aarch64-linux-gnu-g++" \ - CC="/usr/bin/aarch64-linux-gnu-gcc" - -WORKDIR /usr/src/qpdf-${QPDF_VERSION} - -RUN set -eux \ - && echo "Beginning arm64" \ - && echo "Install arm64 packages" \ - && dpkg --add-architecture arm64 \ - && apt-get update --quiet \ - && apt-get install --yes --quiet --no-install-recommends ${ARM64_PACKAGES} \ - && echo "Building arm64" \ - && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes --post-clean --host-arch arm64 \ - && echo "Removing debug files" \ - && rm -f ../libqpdf29-dbgsym* \ - && rm -f ../qpdf-dbgsym* \ - && echo "Gathering package data" \ - && dpkg-query -f '${Package;-40}${Version}\n' -W > ../pkg-list.txt - -# -# Stage: package -# Purpose: Holds the compiled .deb files in arch/variant specific folders -# -FROM alpine:3.17 as package - -LABEL org.opencontainers.image.description="A image with qpdf installers stored in architecture & version specific folders" - -ARG QPDF_VERSION - -WORKDIR /usr/src/qpdf/${QPDF_VERSION}/amd64 - -COPY --from=amd64-builder /usr/src/*.deb ./ -COPY --from=amd64-builder /usr/src/pkg-list.txt ./ - -# Note this is ${TARGETARCH}${TARGETVARIANT} for armv7 -WORKDIR /usr/src/qpdf/${QPDF_VERSION}/armv7 - -COPY --from=armhf-builder /usr/src/*.deb ./ -COPY --from=armhf-builder /usr/src/pkg-list.txt ./ - -WORKDIR /usr/src/qpdf/${QPDF_VERSION}/arm64 - -COPY --from=aarch64-builder /usr/src/*.deb ./ -COPY --from=aarch64-builder /usr/src/pkg-list.txt ./ diff --git a/docker-builders/README.md b/docker-builders/README.md deleted file mode 100644 index 6202719c6..000000000 --- a/docker-builders/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Installer Library - -This folder contains the Dockerfiles for building certain installers or libraries, which are then pulled into the main image. - -## [jbig2enc](https://github.com/agl/jbig2enc) - -### Why - -JBIG is an image coding which can achieve better compression of images for PDFs. - -### What - -The Docker image builds a shared library file and utility, which is copied into the correct location in the final image. - -### Updating - -1. Ensure the given qpdf version is present in [Debian bookworm](https://packages.debian.org/bookworm/qpdf) -2. Update `.build-config.json` to the given version -3. If the Debian specific version has incremented, update `Dockerfile.qpdf` - -See Also: - -- [OCRMyPDF Documentation](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) - -## [psycopg2](https://www.psycopg.org/) - -### Why - -The pre-built wheels of psycopg2 are built on Debian 9, which provides a quite old version of libpq-dev. This causes issue with authentication methods. - -### What - -The image builds psycopg2 wheels on Debian 10 and places the produced wheels into `/usr/src/wheels/`. - -See Also: - -- [Issue 266](https://github.com/paperless-ngx/paperless-ngx/issues/266) - -## [qpdf](https://qpdf.readthedocs.io/en/stable/index.html) - -### Why - -qpdf and it's library provide tools to read, manipulate and fix up PDFs. Version 11 is also required by `pikepdf` 6+ and Debian 9 does not provide above version 10. - -### What - -The Docker image cross compiles .deb installers for each supported architecture of the main image. The installers are placed in `/usr/src/qpdf/${QPDF_VERSION}/${TARGETARCH}${TARGETVARIANT}/` - -## [pikepdf](https://pikepdf.readthedocs.io/en/latest/) - -### Why - -Required by OCRMyPdf, this is a general purpose library for PDF manipulation in Python via the qpdf libraries. - -### What - -The built wheels are placed into `/usr/src/wheels/` diff --git a/docs/development.md b/docs/development.md index 82d2d4c16..ddbd9c40e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -374,13 +374,10 @@ If you want to build the documentation locally, this is how you do it: The docker image is primarily built by the GitHub actions workflow, but it can be faster when developing to build and tag an image locally. -To provide the build arguments automatically, build the image using the -helper script `build-docker-image.sh`. +Building the image works as with any image: -Building the docker image from source: - -```bash -./build-docker-image.sh Dockerfile -t +``` +docker build --file Dockerfile --tag paperless:local --progress simple . ``` ## Extending Paperless-ngx