diff --git a/.github/scripts/get-build-json.py b/.github/scripts/get-build-json.py index 42c37d695..42299abae 100755 --- a/.github/scripts/get-build-json.py +++ b/.github/scripts/get-build-json.py @@ -53,10 +53,7 @@ def _main(): git_tag = None extra_config = {} - if args.package == "frontend": - # Version is just the branch or tag name - version = branch_name - elif args.package in pipfile_data["default"]: + if args.package in pipfile_data["default"]: # Read the version from Pipfile.lock pkg_data = pipfile_data["default"][args.package] pkg_version = pkg_data["version"].split("==")[-1] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe52da55d..a637dc78a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,15 +104,6 @@ jobs: echo ${build_json} echo ::set-output name=jbig2enc-json::${build_json} - - - name: Setup frontend image - id: frontend-setup - run: | - build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py frontend) - - echo ${build_json} - - echo ::set-output name=frontend-json::${build_json} outputs: @@ -124,8 +115,6 @@ jobs: jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}} - frontend-json: ${{ steps.frontend-setup.outputs.frontend-json}} - build-qpdf-debs: name: qpdf needs: @@ -175,57 +164,6 @@ jobs: PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }} PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} - build-frontend: - name: Compile frontend - concurrency: - group: ${{ github.workflow }}-build-frontend-${{ github.ref_name }} - cancel-in-progress: false - needs: - - prepare-docker-build - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Login to Github Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Compile frontend - uses: docker/build-push-action@v2 - with: - context: . - file: ./docker-builders/Dockerfile.frontend - tags: ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} - # The compilation is identical between different platforms - # The buildx and QEMU setup is left, just in case that ever changes - # But the platform is set to the runner's native for speedup - platforms: linux/amd64 - push: true - cache-from: type=registry,ref=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).cache_tag }} - cache-to: type=registry,mode=max,ref=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).cache_tag }} - - - name: Export frontend artifact from docker - run: | - docker create --name frontend-extract ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }} - docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/ - - - name: Upload frontend artifact - uses: actions/upload-artifact@v3 - with: - name: frontend-compiled - path: src/documents/static/frontend/ - # build and push image to docker hub. build-docker-image: runs-on: ubuntu-20.04 @@ -238,7 +176,6 @@ jobs: - build-jbig2enc - build-qpdf-debs - build-pikepdf-wheel - - build-frontend steps: - name: Check pushing to Docker Hub @@ -297,18 +234,33 @@ jobs: tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} build-args: | - REPO=${{ github.repository }} JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }} QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }} PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }} PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }} - FRONTEND_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).version }} - cache-from: type=gha - cache-to: type=gha,mode=max + # Get cache layers from this branch, then dev, then main + # This allows new branches to get at least some cache benefits, generally from dev + cache-from: | + type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }} + type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:dev + type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:main + cache-to: | + type=registry,mode=max,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }} - name: Inspect image run: | docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }} + - + name: Export frontend artifact from docker + run: | + docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }} + docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/ + - + name: Upload frontend artifact + uses: actions/upload-artifact@v3 + with: + name: frontend-compiled + path: src/documents/static/frontend/ build-release: needs: diff --git a/Dockerfile b/Dockerfile index 8d2efca6b..779b6e251 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,32 @@ -# Default to pulling from the main repo registry when manually building -ARG REPO="paperless-ngx/paperless-ngx" +# Pull the installer images from the library +# These are all built previously +# They provide either a .deb or .whl -# These are all built previously in the pipeline -# They provide either a .deb, .whl or whatever npm outputs ARG JBIG2ENC_VERSION ARG QPDF_VERSION ARG PIKEPDF_VERSION ARG PSYCOPG2_VERSION -ARG FRONTEND_VERSION -FROM ghcr.io/${REPO}/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder -FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder -FROM ghcr.io/${REPO}/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder -FROM ghcr.io/${REPO}/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder -FROM ghcr.io/${REPO}/builder/frontend:${FRONTEND_VERSION} as compile-frontend +FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder +FROM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder +FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder +FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder + +FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend + +# This stage compiles the frontend +# This stage runs once for the native platform, as the outputs are not +# dependent on target arch +# Inputs: None + +COPY ./src-ui /src/src-ui + +WORKDIR /src/src-ui +RUN set -eux \ + && npm update npm -g \ + && npm ci --no-optional +RUN set -eux \ + && ./node_modules/.bin/ng build --configuration production FROM python:3.9-slim-bullseye as main-app @@ -156,8 +169,11 @@ COPY gunicorn.conf.py . WORKDIR /usr/src/paperless/src/ -# copy app -COPY --from=compile-frontend /src/src/ ./ +# copy backend +COPY ./src ./ + +# copy frontend +COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/ # add users, setup scripts RUN set -eux \ diff --git a/build-docker-image.sh b/build-docker-image.sh index e3bdd7998..ed20589f4 100755 --- a/build-docker-image.sh +++ b/build-docker-image.sh @@ -2,15 +2,13 @@ # Helper script for building the Docker image locally. # Parses and provides the nessecary versions of other images to Docker -# before passing in the rest of script args. A future enhancement -# would be to combine this with the CI script +# before passing in the rest of script args. # First Argument: The Dockerfile to build # Other Arguments: Additional arguments to docker build # Example Usage: # ./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature -# ./build-docker-image.sh docker-builders/Dockerfile.qpdf -t paperless-ngx-build-qpdf:x.y.z set -eux @@ -28,23 +26,17 @@ psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' | # Read this from the other config file qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g') jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g') -# Get the branch name -frontend_version=$(git rev-parse --abbrev-ref HEAD) - -# Get Git tags for building -# psycopg2 uses X_Y_Z git tags -psycopg2_git_tag=${psycopg2_version//./_} -# pikepdf uses vX.Y.Z -pikepdf_git_tag="v${pikepdf_version}" +# Get the branch name (used for caching) +branch_name=$(git rev-parse --abbrev-ref HEAD) # https://docs.docker.com/develop/develop-images/build_enhancements/ +# Required to use cache-from export DOCKER_BUILDKIT=1 docker build --file "$1" \ + --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \ + --cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \ --build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \ --build-arg QPDF_VERSION="${qpdf_version}" \ --build-arg PIKEPDF_VERSION="${pikepdf_version}" \ - --build-arg PIKEPDF_GIT_TAG="${pikepdf_git_tag}" \ - --build-arg PSYCOPG2_VERSION="${psycopg2_version}" \ - --build-arg PSYCOPG2_GIT_TAG="${psycopg2_git_tag}" \ - --build-arg FRONTEND_VERSION="${frontend_version}" "${@:2}" . + --build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" . diff --git a/docs/extending.rst b/docs/extending.rst index 8b276dacc..199cd78ce 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -334,11 +334,17 @@ directory. Building the Docker image ========================= +The docker image is primarily built by the GitHub actions workflow, but it can be +faster when developing to build and tag an image locally. + +To provide the build arguments automatically, build the image using the helper +script ``build-docker-image.sh``. + Building the docker image from source: .. code:: shell-session - docker build . -t + ./build-docker-image.sh Dockerfile -t Extending Paperless ===================