From 4467ef054d9c8903ceb939a1acb9d083696721a9 Mon Sep 17 00:00:00 2001 From: Toxix Date: Sun, 4 Jul 2021 00:15:29 +0200 Subject: [PATCH 1/6] Refector: Do the npm compile as a Build step This removes the requirement of npm on the host system and removes an additional manual step during the building of the docker image. Updated the documentation accordingly and fixed a typo. --- Dockerfile | 11 ++++++++++- compile-frontend.sh | 7 ------- docs/setup.rst | 6 ++---- 3 files changed, 12 insertions(+), 12 deletions(-) delete mode 100755 compile-frontend.sh diff --git a/Dockerfile b/Dockerfile index 6c0caa049..6bd64f508 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,12 @@ +FROM node AS compile-frontend + +COPY . /src + +WORKDIR /src/src-ui +RUN npm install +RUN ./node_modules/.bin/ng build --prod + + FROM ubuntu:20.04 AS jbig2enc WORKDIR /usr/src/jbig2enc @@ -88,7 +97,7 @@ RUN cd docker \ COPY gunicorn.conf.py ../ # copy app -COPY src/ ./ +COPY --from=compile-frontend /src/src/ ./ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/compile-frontend.sh b/compile-frontend.sh deleted file mode 100755 index 98b88d033..000000000 --- a/compile-frontend.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -cd src-ui -npm install -./node_modules/.bin/ng build --prod diff --git a/docs/setup.rst b/docs/setup.rst index f8b93f570..ee2cbe77b 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -260,9 +260,7 @@ Build the Docker image yourself webserver: build: . -4. Run the ``compile-frontend.sh`` script. This requires ``node`` and ``npm >= v15``. - -5. Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run +4. Follow steps 3 to 8 of :ref:`setup-docker_hub`. When asked to run ``docker-compose pull`` to pull the image, do .. code:: shell-session @@ -684,7 +682,7 @@ configuring some options in paperless can help improve performance immensely: your documents before feeding them into paperless. Some scanners are able to do this! You might want to even specify ``skip_noarchive`` to skip archive file generation for already ocr'ed documents entirely. -* If you want to perform OCR on the the device, consider using ``PAPERLESS_OCR_CLEAN=none``. +* If you want to perform OCR on the device, consider using ``PAPERLESS_OCR_CLEAN=none``. This will speed up OCR times and use less memory at the expense of slightly worse OCR results. * Set ``PAPERLESS_OPTIMIZE_THUMBNAILS`` to 'false' if you want faster consumption From e2809a48c27539d1fd7001dd7abebdb83ed23cf1 Mon Sep 17 00:00:00 2001 From: Toxix Date: Mon, 5 Jul 2021 02:32:31 +0200 Subject: [PATCH 2/6] Feat adjust github pipeline for docker frontend artifacts This should allow to use docker building the frontend artifacts and reusing them in the bare metal release. --- .github/workflows/ci.yml | 47 ++++++++++------------------------------ 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13dc2c74c..2d452f3bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,35 +144,8 @@ jobs: cd src/ coveralls --service=github - frontend: - runs-on: ubuntu-20.04 - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - uses: actions/setup-node@v2 - with: - node-version: '16' - - - name: Configure version on dev branches - if: startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' - run: | - git_hash=$(git rev-parse --short "$GITHUB_SHA") - git_branch=${GITHUB_REF#refs/heads/} - sed -i -E "s/version: \"(.*)\"/version: \"${git_branch} ${git_hash}\"/g" src-ui/src/environments/environment.prod.ts - - - name: Build frontend - run: ./compile-frontend.sh - - - name: Upload artifact - uses: actions/upload-artifact@v2 - with: - name: frontend-compiled - path: src/documents/static/frontend/ - build-release: - needs: [frontend, documentation, tests, whitespace, codestyle] + needs: [build-docker-image, documentation, tests, whitespace, codestyle] runs-on: ubuntu-20.04 steps: - @@ -283,7 +256,7 @@ jobs: build-docker-image: if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-')) runs-on: ubuntu-latest - needs: [frontend, tests, whitespace, codestyle] + needs: [tests, whitespace, codestyle] steps: - name: Prepare @@ -304,12 +277,6 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - - name: Download frontend artifact - uses: actions/download-artifact@v2 - with: - name: frontend-compiled - path: src/documents/static/frontend/ - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -346,3 +313,13 @@ jobs: name: Inspect image run: | docker buildx imagetools inspect ${{ steps.prepare.outputs.inspect_tag }} + - + name: Export frontend artifact from docker + run: | + docker cp ${{ steps.prepare.outputs.tags }}:/src/src/documents/static/frontend/ src/documents/static/frontend/ + - + name: Upload frontend artifact + uses: actions/upload-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ From bb507529db300ce066c7eac79a9134443b3f358a Mon Sep 17 00:00:00 2001 From: Toxix Date: Mon, 5 Jul 2021 02:43:15 +0200 Subject: [PATCH 3/6] Doc Update docs for frontend-compile As frontend compile step is done now inside docker container, we do not need to do it before. This is updating the documentation so it reflects the code changes made. --- docs/administration.rst | 1 - docs/extending.rst | 15 +++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 734a51756..3becf750e 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -92,7 +92,6 @@ B. If you built the image yourself, do the following: .. code:: shell-session $ git pull - $ ./compile-frontend.sh $ docker-compose build $ docker-compose up diff --git a/docs/extending.rst b/docs/extending.rst index f51ecc3a7..e085fd706 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -59,12 +59,11 @@ To do the setup you need to perform the steps from the following chapters in a c pipenv install --dev * Make sure you're using python 3.9.x or lower. Otherwise you might get issues with building dependencies. You can use `pyenv `_ to install a specific python version. - -7. Generate the static UI so you can perform a login to get session that is required for frontend development (this needs to be done one time only). From root folder: +7. Generate the static UI so you can perform a login to get session that is required for frontend development (this needs to be done one time only). From src-ui directory: .. code:: shell-session - compile-frontend.sh + ./node_modules/.bin/ng build --prod 8. Apply migrations and create a superuser for your dev instance: @@ -273,15 +272,7 @@ directory. Building the Docker image ========================= -Building the docker image from source requires the following two steps: - -1. Build the front end. - - .. code:: shell-session - - ./compile-frontend.sh - -2. Build the docker image. +Building the docker image from source: .. code:: shell-session From 3287ddf6d7dd4ed5981b0665845a375bee36dacb Mon Sep 17 00:00:00 2001 From: Toxix Date: Mon, 5 Jul 2021 03:10:37 +0200 Subject: [PATCH 4/6] Fix node version From automatic codereview. We should use a specific node version. We are using the same version that was used before in the github pipeline. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6bd64f508..d8cca0e3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node AS compile-frontend +FROM node:15 AS compile-frontend COPY . /src From 89d0a232cfca8b50552e24c09ec022d9ffd204cb Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Fri, 25 Feb 2022 13:51:00 -0800 Subject: [PATCH 5/6] Updates node version to 16 for the builder image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d8cca0e3d..b938bef82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:15 AS compile-frontend +FROM node:16 AS compile-frontend COPY . /src From 1a0d6b332ac558c291bbd914b1e1a8f8c0495683 Mon Sep 17 00:00:00 2001 From: Trenton Holmes Date: Fri, 25 Feb 2022 13:59:29 -0800 Subject: [PATCH 6/6] Adds missing step to run npm install --- docs/extending.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/extending.rst b/docs/extending.rst index e085fd706..6d2216b38 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -59,10 +59,12 @@ To do the setup you need to perform the steps from the following chapters in a c pipenv install --dev * Make sure you're using python 3.9.x or lower. Otherwise you might get issues with building dependencies. You can use `pyenv `_ to install a specific python version. + 7. Generate the static UI so you can perform a login to get session that is required for frontend development (this needs to be done one time only). From src-ui directory: .. code:: shell-session + npm install . ./node_modules/.bin/ng build --prod 8. Apply migrations and create a superuser for your dev instance: