From a6144e9692b12197af7f9625000da9b29732a56d Mon Sep 17 00:00:00 2001 From: Toxix Date: Sun, 4 Jul 2021 00:15:29 +0200 Subject: [PATCH 1/6] Refector: Do the npm compile as a Build step This removes the requirement of npm on the host system and removes an additional manual step during the building of the docker image. Updated the documentation accordingly and fixed a typo. --- Dockerfile | 11 ++++++++++- compile-frontend.sh | 7 ------- docs/setup.rst | 6 ++---- 3 files changed, 12 insertions(+), 12 deletions(-) delete mode 100755 compile-frontend.sh diff --git a/Dockerfile b/Dockerfile index 6c0caa049..6bd64f508 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,12 @@ +FROM node AS compile-frontend + +COPY . /src + +WORKDIR /src/src-ui +RUN npm install +RUN ./node_modules/.bin/ng build --prod + + FROM ubuntu:20.04 AS jbig2enc WORKDIR /usr/src/jbig2enc @@ -88,7 +97,7 @@ RUN cd docker \ COPY gunicorn.conf.py ../ # copy app -COPY src/ ./ +COPY --from=compile-frontend /src/src/ ./ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/compile-frontend.sh b/compile-frontend.sh deleted file mode 100755 index 98b88d033..000000000 --- a/compile-frontend.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -cd src-ui -npm install -./node_modules/.bin/ng build --prod diff --git a/docs/setup.rst b/docs/setup.rst index f8b93f570..ee2cbe77b 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -260,9 +260,7 @@ Build the Docker image yourself webserver: build: . -4. Run the ``compile-frontend.sh`` script. This requires ``node`` and ``npm >= v15``. - -5. Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run +4. Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run ``docker-compose pull`` to pull the image, do .. code:: shell-session @@ -684,7 +682,7 @@ configuring some options in paperless can help improve performance immensely: your documents before feeding them into paperless. Some scanners are able to do this! You might want to even specify ``skip_noarchive`` to skip archive file generation for already ocr'ed documents entirely. -* If you want to perform OCR on the the device, consider using ``PAPERLESS_OCR_CLEAN=none``. +* If you want to perform OCR on the device, consider using ``PAPERLESS_OCR_CLEAN=none``. This will speed up OCR times and use less memory at the expense of slightly worse OCR results. * Set ``PAPERLESS_OPTIMIZE_THUMBNAILS`` to 'false' if you want faster consumption From 84f96d7a984be7b6dd2a8438cff184cdc506f291 Mon Sep 17 00:00:00 2001 From: Toxix Date: Mon, 5 Jul 2021 02:32:31 +0200 Subject: [PATCH 2/6] Feat adjust github pipeline for docker frontend artifacts This should allow to use docker building the frontend artifacts and reusing them in the bare metal release. --- .github/workflows/ci.yml | 47 ++++++++++------------------------------ 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13dc2c74c..2d452f3bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,35 +144,8 @@ jobs: cd src/ coveralls --service=github - frontend: - runs-on: ubuntu-20.04 - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - uses: actions/setup-node@v2 - with: - node-version: '16' - - - name: Configure version on dev branches - if: startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' - run: | - git_hash=$(git rev-parse --short "$GITHUB_SHA") - git_branch=${GITHUB_REF#refs/heads/} - sed -i -E "s/version: \"(.*)\"/version: \"${git_branch} ${git_hash}\"/g" src-ui/src/environments/environment.prod.ts - - - name: Build frontend - run: ./compile-frontend.sh - - - name: Upload artifact - uses: actions/upload-artifact@v2 - with: - name: frontend-compiled - path: src/documents/static/frontend/ - build-release: - needs: [frontend, documentation, tests, whitespace, codestyle] + needs: [build-docker-image, documentation, tests, whitespace, codestyle] runs-on: ubuntu-20.04 steps: - @@ -283,7 +256,7 @@ jobs: build-docker-image: if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-')) runs-on: ubuntu-latest - needs: [frontend, tests, whitespace, codestyle] + needs: [tests, whitespace, codestyle] steps: - name: Prepare @@ -304,12 +277,6 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - - name: Download frontend artifact - uses: actions/download-artifact@v2 - with: - name: frontend-compiled - path: src/documents/static/frontend/ - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -346,3 +313,13 @@ jobs: name: Inspect image run: | docker buildx imagetools inspect ${{ steps.prepare.outputs.inspect_tag }} + - + name: Export frontend artifact from docker + run: | + docker cp ${{ steps.prepare.outputs.tags }}:/src/src/documents/static/frontend/ src/documents/static/frontend/ + - + name: Upload frontend artifact + uses: actions/upload-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ From 9f895fe65f0ceade1a9708ede8e1a805eb140b6e Mon Sep 17 00:00:00 2001 From: Toxix Date: Mon, 5 Jul 2021 02:43:15 +0200 Subject: [PATCH 3/6] Doc Update docs for frontend-compile As frontend compile step is done now inside docker container, we do not need to do it before. This is updating the documentation so it reflects the code changes made. --- docs/administration.rst | 1 - docs/extending.rst | 15 +++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 734a51756..3becf750e 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -92,7 +92,6 @@ B. If you built the image yourself, do the following: .. code:: shell-session $ git pull - $ ./compile-frontend.sh $ docker-compose build $ docker-compose up diff --git a/docs/extending.rst b/docs/extending.rst index f51ecc3a7..e085fd706 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -59,12 +59,11 @@ To do the setup you need to perform the steps from the following chapters in a c pipenv install --dev * Make sure you're using python 3.9.x or lower. Otherwise you might get issues with building dependencies. You can use `pyenv `_ to install a specific python version. - -7. Generate the static UI so you can perform a login to get session that is required for frontend development (this needs to be done one time only). From root folder: +7. Generate the static UI so you can perform a login to get session that is required for frontend development (this needs to be done one time only). From src-ui directory: .. code:: shell-session - compile-frontend.sh + ./node_modules/.bin/ng build --prod 8. Apply migrations and create a superuser for your dev instance: @@ -273,15 +272,7 @@ directory. Building the Docker image ========================= -Building the docker image from source requires the following two steps: - -1. Build the front end. - - .. code:: shell-session - - ./compile-frontend.sh - -2. Build the docker image. +Building the docker image from source: .. code:: shell-session From 31d96ea85b997aac26e310a74f5c26278a62f54d Mon Sep 17 00:00:00 2001 From: Toxix Date: Mon, 5 Jul 2021 03:10:37 +0200 Subject: [PATCH 4/6] Fix node version From automatic codereview. We should use a specific node version. We are using the same version that was used before in the github pipeline. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6bd64f508..d8cca0e3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node AS compile-frontend +FROM node:15 AS compile-frontend COPY . /src From 83e40f3e8768cbf4ed9d2ba5be23b4c327fdcde3 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Fri, 25 Feb 2022 13:51:00 -0800 Subject: [PATCH 5/6] Updates node version to 16 for the builder image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d8cca0e3d..b938bef82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:15 AS compile-frontend +FROM node:16 AS compile-frontend COPY . /src From 6852127c33673f98042e24a346cb003574378a0e Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Fri, 25 Feb 2022 13:59:29 -0800 Subject: [PATCH 6/6] Adds missing step to run npm install --- docs/extending.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/extending.rst b/docs/extending.rst index e085fd706..6d2216b38 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -59,10 +59,12 @@ To do the setup you need to perform the steps from the following chapters in a c pipenv install --dev * Make sure you're using python 3.9.x or lower. Otherwise you might get issues with building dependencies. You can use `pyenv `_ to install a specific python version. + 7. Generate the static UI so you can perform a login to get session that is required for frontend development (this needs to be done one time only). From src-ui directory: .. code:: shell-session + npm install . ./node_modules/.bin/ng build --prod 8. Apply migrations and create a superuser for your dev instance: