Merge pull request #849 from paperless-ngx/feature-fix-image-local-build

Fix local Docker image building

.github/scripts/get-build-json.py

@@ -53,10 +53,7 @@ def _main():
     git_tag = None
     extra_config = {}
 
-    if args.package == "frontend":
+    if args.package in pipfile_data["default"]:
-        # Version is just the branch or tag name
-        version = branch_name
-    elif args.package in pipfile_data["default"]:
         # Read the version from Pipfile.lock
         pkg_data = pipfile_data["default"][args.package]
         pkg_version = pkg_data["version"].split("==")[-1]

.github/workflows/ci.yml

@@ -104,15 +104,6 @@ jobs:
           echo ${build_json}
 
           echo ::set-output name=jbig2enc-json::${build_json}
-      -
-        name: Setup frontend image
-        id: frontend-setup
-        run: |
-          build_json=$(python ${GITHUB_WORKSPACE}/.github/scripts/get-build-json.py frontend)
-
-          echo ${build_json}
-
-          echo ::set-output name=frontend-json::${build_json}
 
     outputs:
 
@@ -124,8 +115,6 @@ jobs:
 
       jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
 
-      frontend-json: ${{ steps.frontend-setup.outputs.frontend-json}}
-
   build-qpdf-debs:
     name: qpdf
     needs:
@@ -175,57 +164,6 @@ jobs:
         PIKEPDF_GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
         PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
 
-  build-frontend:
-    name: Compile frontend
-    concurrency:
-      group: ${{ github.workflow }}-build-frontend-${{ github.ref_name }}
-      cancel-in-progress: false
-    needs:
-      - prepare-docker-build
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v3
-      -
-        name: Login to Github Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Compile frontend
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./docker-builders/Dockerfile.frontend
-          tags: ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
-          # The compilation is identical between different platforms
-          # The buildx and QEMU setup is left, just in case that ever changes
-          # But the platform is set to the runner's native for speedup
-          platforms: linux/amd64
-          push: true
-          cache-from: type=registry,ref=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).cache_tag }}
-          cache-to: type=registry,mode=max,ref=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).cache_tag }}
-      -
-        name: Export frontend artifact from docker
-        run: |
-          docker create --name frontend-extract ${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).image_tag }}
-          docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/
-      -
-        name: Upload frontend artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: frontend-compiled
-          path: src/documents/static/frontend/
-
   # build and push image to docker hub.
   build-docker-image:
     runs-on: ubuntu-20.04
@@ -238,7 +176,6 @@ jobs:
       - build-jbig2enc
       - build-qpdf-debs
       - build-pikepdf-wheel
-      - build-frontend
     steps:
       -
         name: Check pushing to Docker Hub
@@ -297,18 +234,33 @@ jobs:
           tags: ${{ steps.docker-meta.outputs.tags }}
           labels: ${{ steps.docker-meta.outputs.labels }}
           build-args: |
-            REPO=${{ github.repository }}
             JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
             QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
             PIKEPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
             PSYCOPG2_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
-            FRONTEND_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.frontend-json).version }}
+          # Get cache layers from this branch, then dev, then main
-          cache-from: type=gha
+          # This allows new branches to get at least some cache benefits, generally from dev
-          cache-to: type=gha,mode=max
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:dev
+            type=registry,ref=ghcr.io/${{ github.repository }}/builder/cache/app:main
+          cache-to: |
+            type=registry,mode=max,ref=ghcr.io/${{ github.repository }}/builder/cache/app:${{ github.ref_name }}
       -
         name: Inspect image
         run: |
           docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
+      -
+        name: Export frontend artifact from docker
+        run: |
+          docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
+          docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
+      -
+        name: Upload frontend artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: frontend-compiled
+          path: src/documents/static/frontend/
 
   build-release:
     needs:

Dockerfile

@@ -1,19 +1,32 @@
-# Default to pulling from the main repo registry when manually building
+# Pull the installer images from the library
-ARG REPO="paperless-ngx/paperless-ngx"
+# These are all built previously
+# They provide either a .deb or .whl
 
-# These are all built previously in the pipeline
-# They provide either a .deb, .whl or whatever npm outputs
 ARG JBIG2ENC_VERSION
 ARG QPDF_VERSION
 ARG PIKEPDF_VERSION
 ARG PSYCOPG2_VERSION
-ARG FRONTEND_VERSION
 
-FROM ghcr.io/${REPO}/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/jbig2enc:${JBIG2ENC_VERSION} as jbig2enc-builder
-FROM ghcr.io/${REPO}/builder/qpdf:${QPDF_VERSION} as qpdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/qpdf:${QPDF_VERSION} as qpdf-builder
-FROM ghcr.io/${REPO}/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/pikepdf:${PIKEPDF_VERSION} as pikepdf-builder
-FROM ghcr.io/${REPO}/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
+FROM ghcr.io/paperless-ngx/paperless-ngx/builder/psycopg2:${PSYCOPG2_VERSION} as psycopg2-builder
-FROM ghcr.io/${REPO}/builder/frontend:${FRONTEND_VERSION} as compile-frontend
+
+FROM --platform=$BUILDPLATFORM node:16-bullseye-slim AS compile-frontend
+
+# This stage compiles the frontend
+# This stage runs once for the native platform, as the outputs are not
+# dependent on target arch
+# Inputs: None
+
+COPY ./src-ui /src/src-ui
+
+WORKDIR /src/src-ui
+RUN set -eux \
+  && npm update npm -g \
+  && npm ci --no-optional
+RUN set -eux \
+  && ./node_modules/.bin/ng build --configuration production
 
 FROM python:3.9-slim-bullseye as main-app
 
@@ -156,8 +169,11 @@ COPY gunicorn.conf.py .
 
 WORKDIR /usr/src/paperless/src/
 
-# copy app
+# copy backend
-COPY --from=compile-frontend /src/src/ ./
+COPY ./src ./
+
+# copy frontend
+COPY --from=compile-frontend /src/src/documents/static/frontend/ ./documents/static/frontend/
 
 # add users, setup scripts
 RUN set -eux \

build-docker-image.sh

@@ -2,15 +2,13 @@
 
 # Helper script for building the Docker image locally.
 # Parses and provides the nessecary versions of other images to Docker
-# before passing in the rest of script args.  A future enhancement
+# before passing in the rest of script args.
-# would be to combine this with the CI script
 
 # First Argument: The Dockerfile to build
 # Other Arguments: Additional arguments to docker build
 
 # Example Usage:
 #	./build-docker-image.sh Dockerfile -t paperless-ngx:my-awesome-feature
-#	./build-docker-image.sh docker-builders/Dockerfile.qpdf -t paperless-ngx-build-qpdf:x.y.z
 
 set -eux
 
@@ -28,23 +26,17 @@ psycopg2_version=$(jq ".default.psycopg2.version" Pipfile.lock | sed 's/=//g' |
 # Read this from the other config file
 qpdf_version=$(jq ".qpdf.version" .build-config.json | sed 's/"//g')
 jbig2enc_version=$(jq ".jbig2enc.version" .build-config.json | sed 's/"//g')
-# Get the branch name
+# Get the branch name (used for caching)
-frontend_version=$(git rev-parse --abbrev-ref HEAD)
+branch_name=$(git rev-parse --abbrev-ref HEAD)
-
-# Get Git tags for building
-# psycopg2 uses X_Y_Z git tags
-psycopg2_git_tag=${psycopg2_version//./_}
-# pikepdf uses vX.Y.Z
-pikepdf_git_tag="v${pikepdf_version}"
 
 # https://docs.docker.com/develop/develop-images/build_enhancements/
+# Required to use cache-from
 export DOCKER_BUILDKIT=1
 
 docker build --file "$1" \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:"${branch_name}" \
+	--cache-from ghcr.io/paperless-ngx/paperless-ngx/builder/cache/app:dev \
 	--build-arg JBIG2ENC_VERSION="${jbig2enc_version}" \
 	--build-arg QPDF_VERSION="${qpdf_version}" \
 	--build-arg PIKEPDF_VERSION="${pikepdf_version}" \
-	--build-arg PIKEPDF_GIT_TAG="${pikepdf_git_tag}" \
+	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" "${@:2}" .
-	--build-arg PSYCOPG2_VERSION="${psycopg2_version}" \
-	--build-arg PSYCOPG2_GIT_TAG="${psycopg2_git_tag}" \
-	--build-arg FRONTEND_VERSION="${frontend_version}" "${@:2}" .

docs/extending.rst

@@ -334,11 +334,17 @@ directory.
 Building the Docker image
 =========================
 
+The docker image is primarily built by the GitHub actions workflow, but it can be
+faster when developing to build and tag an image locally.
+
+To provide the build arguments automatically, build the image using the helper
+script ``build-docker-image.sh``.
+
 Building the docker image from source:
 
     .. code:: shell-session
 
-        docker build . -t <your-tag>
+        ./build-docker-image.sh Dockerfile -t <your-tag>
 
 Extending Paperless
 ===================