From d702147b77b3f2716c8d69b31e5fea5371506c04 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 15:58:29 +0100 Subject: [PATCH 001/148] reorganized docker build. --- docker/local/Dockerfile => Dockerfile | 17 ++- ...ostgres.yml => docker-compose.postgres.yml | 0 ...se.sqlite.yml => docker-compose.sqlite.yml | 0 docker/docker-compose.env | 34 ----- scripts/make-release.sh | 125 ------------------ scripts/push-release.sh | 23 ---- 6 files changed, 14 insertions(+), 185 deletions(-) rename docker/local/Dockerfile => Dockerfile (78%) rename docker/local/docker-compose.postgres.yml => docker-compose.postgres.yml (100%) rename docker/local/docker-compose.sqlite.yml => docker-compose.sqlite.yml (100%) delete mode 100644 docker/docker-compose.env delete mode 100755 scripts/make-release.sh delete mode 100755 scripts/push-release.sh diff --git a/docker/local/Dockerfile b/Dockerfile similarity index 78% rename from docker/local/Dockerfile rename to Dockerfile index 461b9e4fc..732b4183c 100644 --- a/docker/local/Dockerfile +++ b/Dockerfile @@ -1,8 +1,18 @@ +FROM node:15 AS frontend + +WORKDIR /usr/src/paperless/src-ui/ + +COPY src-ui/package* ./ +RUN npm install + +COPY src-ui . +RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false + FROM python:3.7-slim WORKDIR /usr/src/paperless/ -COPY requirements.txt ./ +COPY Pipfile* ./ #Dependencies RUN apt-get update \ @@ -33,8 +43,8 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools \ - && pip install --no-cache-dir -r requirements.txt \ + && pip3 install --upgrade supervisor setuptools pipenv \ + && pipenv install --system --deploy --ignore-pipfile --clear \ && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ @@ -49,6 +59,7 @@ COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh # copy app COPY src/ ./src/ +COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/docker/local/docker-compose.postgres.yml b/docker-compose.postgres.yml similarity index 100% rename from docker/local/docker-compose.postgres.yml rename to docker-compose.postgres.yml diff --git a/docker/local/docker-compose.sqlite.yml b/docker-compose.sqlite.yml similarity index 100% rename from docker/local/docker-compose.sqlite.yml rename to docker-compose.sqlite.yml diff --git a/docker/docker-compose.env b/docker/docker-compose.env deleted file mode 100644 index 4271bce6e..000000000 --- a/docker/docker-compose.env +++ /dev/null @@ -1,34 +0,0 @@ -# The UID and GID of the user used to run paperless in the container. Set this -# to your UID and GID on the host so that you have write access to the -# consumption directory. -#USERMAP_UID=1000 -#USERMAP_GID=1000 - -# Additional languages to install for text recognition, separated by a -# whitespace. Note that this is -# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the -# default language used when guessing the language from the OCR output. -# The container installs English, German, Italian, Spanish and French by -# default. -# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster -# for available languages. -#PAPERLESS_OCR_LANGUAGES=tur ces - -############################################################################### -# Paperless-specific settings # -############################################################################### - -# All settings defined in the paperless.conf.example can be used here. The -# Docker setup does not use the configuration file. -# A few commonly adjusted settings are provided below. - -# Adjust this key if you plan to make paperless available publicly. It should -# be a very long sequence of random characters. You don't need to remember it. -#PAPERLESS_SECRET_KEY=change-me - -# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. -#PAPERLESS_TIME_ZONE=America/Los_Angeles - -# The default language to use for OCR. Set this to the language most of your -# documents are written in. -#PAPERLESS_OCR_LANGUAGE=eng diff --git a/scripts/make-release.sh b/scripts/make-release.sh deleted file mode 100755 index f5c9028fa..000000000 --- a/scripts/make-release.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -# Release checklist -# - wait for travis build. -# adjust src/paperless/version.py -# changelog in the documentation -# adjust versions in docker/hub/* -# adjust version in src-ui/src/environments/prod -# If docker-compose was modified: all compose files are the same. - -# Steps: -# run release script "dev", push -# if it works: new tag, merge into master -# on master: make release "lastest", push -# on master: make release "version-tag", push -# publish release files - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" -PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles" - -if [ -d "$PAPERLESS_DIST" ] -then - echo "Removing $PAPERLESS_DIST" - rm "$PAPERLESS_DIST" -r -fi - -mkdir "$PAPERLESS_DIST" -mkdir "$PAPERLESS_DIST_APP" -mkdir "$PAPERLESS_DIST_APP/docker" -mkdir "$PAPERLESS_DIST_APP/scripts" -mkdir "$PAPERLESS_DIST_DOCKERFILES" - -# setup dependencies. - -cd "$PAPERLESS_ROOT" - -pipenv clean -pipenv install --dev -pipenv lock --keep-outdated -r > "$PAPERLESS_DIST_APP/requirements.txt" - -# test if the application works. - -cd "$PAPERLESS_ROOT/src" -pipenv run pytest --cov -pipenv run pycodestyle - -# make the documentation. - -cd "$PAPERLESS_ROOT/docs" -make clean html - -# copy stuff into place - -# the application itself - -cp "$PAPERLESS_ROOT/.env" \ - "$PAPERLESS_ROOT/.dockerignore" \ - "$PAPERLESS_ROOT/CONTRIBUTING.md" \ - "$PAPERLESS_ROOT/LICENSE" \ - "$PAPERLESS_ROOT/Pipfile" \ - "$PAPERLESS_ROOT/Pipfile.lock" \ - "$PAPERLESS_ROOT/README.md" "$PAPERLESS_DIST_APP" - -cp "$PAPERLESS_ROOT/paperless.conf.example" "$PAPERLESS_DIST_APP/paperless.conf" - -# copy python source, templates and static files. -cd "$PAPERLESS_ROOT" -find src -wholename '*/templates/*' -o -wholename '*/static/*' -o -name '*.py' | cpio -pdm "$PAPERLESS_DIST_APP" - -# build the front end. - -cd "$PAPERLESS_ROOT/src-ui" -ng build --prod --output-hashing none --sourceMap=false --output-path "$PAPERLESS_DIST_APP/src/documents/static/frontend" - -# documentation -cp "$PAPERLESS_ROOT/docs/_build/html/" "$PAPERLESS_DIST_APP/docs" -r - -# docker files for building the image yourself -cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP" - -# docker files for pulling from docker hub -cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES" - -# auxiliary files required for the docker image -cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/gunicorn.conf.py" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/imagemagick-policy.xml" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/" - -# auxiliary files for bare metal installs -cp "$PAPERLESS_ROOT/scripts/paperless-webserver.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-consumer.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-scheduler.service" "$PAPERLESS_DIST_APP/scripts/" - -# try to make the docker build. - -cd "$PAPERLESS_DIST_APP" - -docker build . -t "jonaswinkler/paperless-ng:$VERSION" - -# works. package the app! - -cd "$PAPERLESS_DIST" - -tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/ -tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/ diff --git a/scripts/push-release.sh b/scripts/push-release.sh deleted file mode 100755 index cfa63f5cf..000000000 --- a/scripts/push-release.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" - -cd "$PAPERLESS_DIST_APP" - -docker push "jonaswinkler/paperless-ng:$VERSION" From 7326a246225baa4840f4827e3469488c8365d938 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 16:01:16 +0100 Subject: [PATCH 002/148] added missing file --- docker-compose.example.env | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docker-compose.example.env diff --git a/docker-compose.example.env b/docker-compose.example.env new file mode 100644 index 000000000..4271bce6e --- /dev/null +++ b/docker-compose.example.env @@ -0,0 +1,34 @@ +# The UID and GID of the user used to run paperless in the container. Set this +# to your UID and GID on the host so that you have write access to the +# consumption directory. +#USERMAP_UID=1000 +#USERMAP_GID=1000 + +# Additional languages to install for text recognition, separated by a +# whitespace. Note that this is +# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the +# default language used when guessing the language from the OCR output. +# The container installs English, German, Italian, Spanish and French by +# default. +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# for available languages. +#PAPERLESS_OCR_LANGUAGES=tur ces + +############################################################################### +# Paperless-specific settings # +############################################################################### + +# All settings defined in the paperless.conf.example can be used here. The +# Docker setup does not use the configuration file. +# A few commonly adjusted settings are provided below. + +# Adjust this key if you plan to make paperless available publicly. It should +# be a very long sequence of random characters. You don't need to remember it. +#PAPERLESS_SECRET_KEY=change-me + +# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. +#PAPERLESS_TIME_ZONE=America/Los_Angeles + +# The default language to use for OCR. Set this to the language most of your +# documents are written in. +#PAPERLESS_OCR_LANGUAGE=eng From 2fff2277476c58adb03e856207082780c5b393c8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 13:13:52 +0100 Subject: [PATCH 003/148] Added multi arch docker builds to travis ci --- .travis.yml | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.travis.yml b/.travis.yml index b745d6bd7..307a6073a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,102 @@ jobs: - ng build --prod after_success: true + - stage: build_docker + name: amd64 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 + on: + condition: '"${BUILD_DOCKER}" = 1' + - stage: build_docker + name: arm64v8 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 + arch: arm64 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: build_docker + name: arm32v7 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: publish_manifest + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} + - | + if [ "${TRAVIS_BRANCH}" = "master" ]; then + echo "Master branch detected" + DOCKER_TAG="latest" + else + DOCKER_TAG=${TRAVIS_TAG} + fi + - | + if [ "${DOCKER_TAG}" != "" ]; then + echo "Create Tag ${DOCKER_TAG}" + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} + else + echo "Not a tag and not on master, so not pushing tag/master specific manifest" + fi + on: + condition: '"${BUILD_DOCKER}" = 1' + before_install: - sudo apt-get update -qq From 35523bfcaa960d4e305d731c83ea3df94fb5e4c4 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 16:19:24 +0100 Subject: [PATCH 004/148] Add libxslt-dev to dependencies. At least one arm based image does not pull in dependencies the same way the amd64 one does, resulting in a missing libxslt shared lib. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..016bf7da9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ + libxslt-dev \ optipng \ pngquant \ qpdf \ From e3e826e906e4ca1ba8f4f0028594523af307bc61 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 17:08:01 +0100 Subject: [PATCH 005/148] Adding some echos to bette runderstand where the build hangs --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 307a6073a..203861391 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,9 +60,12 @@ jobs: after_success: - true script: + - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64 on: From 133c1bf2a2c760b646a7d200d2976ae78e41cc29 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 21:53:39 +0100 Subject: [PATCH 006/148] Use qemu also for arm64v8 build - something might be causing trouble on the native hosts --- .travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 203861391..d2534c152 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,14 +60,17 @@ jobs: after_success: - true script: - - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64 + arch: amd64 on: condition: '"${BUILD_DOCKER}" = 1' From bc30d28cbbf595e80d0eb9720515b150a7d2e425 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 23:02:26 +0100 Subject: [PATCH 007/148] Adds jbig2 to the build, fixes #93 --- Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..6c04a1d07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,14 @@ +FROM ubuntu:20.04 AS jbig2 + +WORKDIR /usr/src/jbig2enc + +RUN apt-get update && apt-get install -y --no-install-recommends build-essential automake libtool libleptonica-dev zlib1g-dev git ca-certificates + +RUN git clone https://github.com/agl/jbig2enc . +RUN ./autogen.sh +RUN ./configure && make + + FROM node:15 AS frontend WORKDIR /usr/src/paperless/src-ui/ @@ -57,6 +68,12 @@ COPY docker/gunicorn.conf.py ./ COPY docker/supervisord.conf /etc/supervisord.conf COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh +# copy jbic +COPY --from=jbig2 /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ +COPY --from=jbig2 /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ +COPY --from=jbig2 /usr/src/jbig2enc/src/*.h /usr/local/include/ + + # copy app COPY src/ ./src/ COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ From b2accdbde78ef801c5733407ea623bc94e010a0f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:49:36 +0100 Subject: [PATCH 008/148] Use arm64-gravis instead of regular arm64. See if that is fast enough. --- .travis.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2534c152..b91e43a28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,17 +60,12 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: amd64 + arch: arm64-graviton2 + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From 30984765e4e49a988fdbf32c0d42903561fab87a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:58:35 +0100 Subject: [PATCH 009/148] Remove accidentially added quote --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b91e43a28..5f838755d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 From e13e202e3d7cec54f89f7dd9e016c3ef3c735011 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:27:19 +0100 Subject: [PATCH 010/148] Tell me, what kind of arm64 is it that produces amd64 docker images? It's weird --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f838755d..f99a52eba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,12 +60,13 @@ jobs: after_success: - true script: + - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: vm + virt: lxc on: condition: '"${BUILD_DOCKER}" = 1' @@ -85,8 +86,8 @@ jobs: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile + - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 From 12a8f96f28f2b7f5825e884a25b6802eedbfdfd5 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:42:21 +0100 Subject: [PATCH 011/148] Enable manifests support --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f99a52eba..6d26d6ccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,8 @@ jobs: # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: condition: '"${BUILD_DOCKER}" = 1' From 8c07c8833cccb2ac7487ffb1100677b886a22592 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:43:51 +0100 Subject: [PATCH 012/148] Remove steps not necessary to build docker images --- .travis.yml | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d26d6ccb..c04f765d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: - - name: "Paperless on Python 3.6" - python: "3.6" - - - name: "Paperless on Python 3.7" - python: "3.7" - - - name: "Paperless on Python 3.8" - python: "3.8" - - - name: "Documentation" - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true +# - name: "Paperless on Python 3.6" +# python: "3.6" +# +# - name: "Paperless on Python 3.7" +# python: "3.7" +# +# - name: "Paperless on Python 3.8" +# python: "3.8" +# +# - name: "Documentation" +# script: +# - cd docs/ +# - make html +# after_success: true +# +# - name: "Front end" +# language: node_js +# node_js: +# - 15 +# before_install: true +# install: +# - cd src-ui/ +# - npm install -g @angular/cli +# - npm install +# script: +# - ng build --prod +# after_success: true - stage: build_docker name: amd64 docker build From d9c1c15dd6478e52fbb9482050a77056da854ed8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:15:06 +0100 Subject: [PATCH 013/148] Switching back to virt: vm as there seems to be a prob with the lxc ones --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c04f765d0..9803a4d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ jobs: - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: lxc + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From 5b584670d6e642ad02da42d3bcbe050b02d2080b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:18:11 +0100 Subject: [PATCH 014/148] arm64-graviton2 build requires group:edge --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9803a4d3b..8422229c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 virt: vm + group: edge on: condition: '"${BUILD_DOCKER}" = 1' From 571736816ebda82675edd416845c72c3f809fa4b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 09:14:40 +0100 Subject: [PATCH 015/148] Remove arm32v6 from docker manifest , it is not getting created --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8422229c7..281635ace 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ jobs: - true script: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} @@ -124,7 +124,7 @@ jobs: - | if [ "${DOCKER_TAG}" != "" ]; then echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} From e541992c1216f9dd2ab4d3c5cf146428a3bfd89d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 12:13:12 +0100 Subject: [PATCH 016/148] Re-enable test steps --- .travis.yml | 55 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 281635ace..e5f3cd91e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: -# - name: "Paperless on Python 3.6" -# python: "3.6" -# -# - name: "Paperless on Python 3.7" -# python: "3.7" -# -# - name: "Paperless on Python 3.8" -# python: "3.8" -# -# - name: "Documentation" -# script: -# - cd docs/ -# - make html -# after_success: true -# -# - name: "Front end" -# language: node_js -# node_js: -# - 15 -# before_install: true -# install: -# - cd src-ui/ -# - npm install -g @angular/cli -# - npm install -# script: -# - ng build --prod -# after_success: true + - name: "Paperless on Python 3.6" + python: "3.6" + + - name: "Paperless on Python 3.7" + python: "3.7" + + - name: "Paperless on Python 3.8" + python: "3.8" + + - name: "Documentation" + script: + - cd docs/ + - make html + after_success: true + + - name: "Front end" + language: node_js + node_js: + - 15 + before_install: true + install: + - cd src-ui/ + - npm install -g @angular/cli + - npm install + script: + - ng build --prod + after_success: true - stage: build_docker name: amd64 docker build @@ -60,7 +60,6 @@ jobs: after_success: - true script: - - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . From c72c489cdcc1e532459f1b7eea59a4f546dbd669 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 23 Dec 2020 17:03:00 +0100 Subject: [PATCH 017/148] update Dockerfile --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f0112e529..686496475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ - libxslt-dev \ + libxslt1-dev \ optipng \ pngquant \ qpdf \ @@ -55,9 +55,11 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools pipenv \ - && pipenv install --system --deploy --ignore-pipfile --clear \ - && apt-get -y purge build-essential libqpdf-dev \ + && pip3 install --upgrade supervisor pipenv \ + && pipenv install --system --deploy --ignore-pipfile \ + && pipenv --clear \ + && pip3 uninstall -y pipenv \ + && apt-get -y purge build-essential libqpdf-dev libxslt1-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord From 22ae74d409b6c62148849757e7f615be17457da1 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sun, 27 Dec 2020 20:01:21 +0100 Subject: [PATCH 018/148] Use conditions to prevent builds on branches / tags other than master, dev and ng-* --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5f3cd91e..cf18825e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ jobs: - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -68,8 +68,7 @@ jobs: virt: vm group: edge on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -94,8 +93,7 @@ jobs: env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -131,7 +129,7 @@ jobs: echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From d258fdc783110b8abf0c5e1e5d57fbbdb03a8b73 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:22:35 +0100 Subject: [PATCH 019/148] Move build condition into separate stages list --- .travis.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf18825e8..26153588e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,12 @@ language: python dist: focal os: linux +stages: + - name: build_docker + if: branch =~ /^(master|dev|ng-.*)$/ + - name: publish_manifest + if: branch =~ /^(master|dev|ng-.*)$/ + jobs: include: - name: "Paperless on Python 3.6" @@ -47,8 +53,6 @@ jobs: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -67,8 +71,6 @@ jobs: arch: arm64-graviton2 virt: vm group: edge - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -92,8 +94,6 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -128,8 +128,6 @@ jobs: else echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From 80df0cc06a64984a8ddd344b314d45d6c3e07d63 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:24:39 +0100 Subject: [PATCH 020/148] Add stage definitions of test jobs and add test to list of stages Defining the stages instead of having travis guess them and implicitly build an order ensures that the test stage is executed before the docker images are built. --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 26153588e..f4873cf3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ dist: focal os: linux stages: + - name: test - name: build_docker if: branch =~ /^(master|dev|ng-.*)$/ - name: publish_manifest @@ -12,21 +13,26 @@ stages: jobs: include: - name: "Paperless on Python 3.6" + stage: test python: "3.6" - name: "Paperless on Python 3.7" + stage: test python: "3.7" - name: "Paperless on Python 3.8" + stage: test python: "3.8" - name: "Documentation" + stage: test script: - cd docs/ - make html after_success: true - name: "Front end" + stage: test language: node_js node_js: - 15 From 9aed0aceea2a780191a8be970dc83949367ac50a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:24:57 +0100 Subject: [PATCH 021/148] Removing travis in favor of github actions --- .travis.yml | 153 ---------------------------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f4873cf3c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,153 +0,0 @@ -language: python - -dist: focal -os: linux - -stages: - - name: test - - name: build_docker - if: branch =~ /^(master|dev|ng-.*)$/ - - name: publish_manifest - if: branch =~ /^(master|dev|ng-.*)$/ - -jobs: - include: - - name: "Paperless on Python 3.6" - stage: test - python: "3.6" - - - name: "Paperless on Python 3.7" - stage: test - python: "3.7" - - - name: "Paperless on Python 3.8" - stage: test - python: "3.8" - - - name: "Documentation" - stage: test - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - stage: test - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true - - - stage: build_docker - name: amd64 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - - stage: build_docker - name: arm64v8 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64-graviton2 - virt: vm - group: edge - - stage: build_docker - name: arm32v7 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - - stage: publish_manifest - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} - - | - if [ "${TRAVIS_BRANCH}" = "master" ]; then - echo "Master branch detected" - DOCKER_TAG="latest" - else - DOCKER_TAG=${TRAVIS_TAG} - fi - - | - if [ "${DOCKER_TAG}" != "" ]; then - echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} - else - echo "Not a tag and not on master, so not pushing tag/master specific manifest" - fi - - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - -install: - - pip install --upgrade pipenv - - pipenv install --system --dev - -script: - - cd src/ - - pipenv run pytest --cov - - pipenv run pycodestyle - -after_success: - - pipenv run coveralls From 645ba2d425c9f1be8852fa6edb434b774abc2080 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:27:22 +0100 Subject: [PATCH 022/148] Experimenting with github actions --- .github/workflows/docker-publish.yml | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..f2da6b3b8 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,63 @@ +name: Docker + +on: + push: + branches: travis-multiarch-builds + +jobs: + # Build and push image to docker hub. + buildx: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + fi + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./Dockerfile . + - name: install buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v3.1.0 + + - name: Docker Buildx (build) + run: | + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Login + if: success() && github.event_name != 'pull_request' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + - name: Docker Buildx (push) + if: success() && github.event_name != 'pull_request' + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Check Manifest + if: always() && github.event_name != 'pull_request' + run: | + docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + - name: Clear + if: always() && github.event_name != 'pull_request' + run: | + rm -f ${HOME}/.docker/config.json \ No newline at end of file From 94ab1dd5c4971a98c7859c4d2eda0843d4759e76 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 19:32:04 +0100 Subject: [PATCH 023/148] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f2da6b3b8..5ee210ef2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,7 +15,7 @@ jobs: - name: Prepare id: prepare run: | - DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_IMAGE=moztr/paperless-ng DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then @@ -60,4 +60,4 @@ jobs: - name: Clear if: always() && github.event_name != 'pull_request' run: | - rm -f ${HOME}/.docker/config.json \ No newline at end of file + rm -f ${HOME}/.docker/config.json From d9992fb318466d1f1d8477f27815fd4f970ed9ad Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 1 Jan 2021 12:53:16 +0100 Subject: [PATCH 024/148] Removed separate build step It looks like part of the build step is repeated in the push step due to bad caching possibly. --- .github/workflows/docker-publish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5ee210ef2..53d53e25c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,10 +38,6 @@ jobs: - name: install buildx id: buildx uses: crazy-max/ghaction-docker-buildx@v3.1.0 - - - name: Docker Buildx (build) - run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} - name: Docker Login if: success() && github.event_name != 'pull_request' env: From f9430e1bd24ce6ec81bf502f4d7a9e3728e6d6d0 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:04:46 +0100 Subject: [PATCH 025/148] Execute python tests --- .github/workflows/docker-publish.yml | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 53d53e25c..a2897c05b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -5,6 +5,47 @@ on: branches: travis-multiarch-builds jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python-version }} + - name: Update system, install system requirements + run: | + apt-get update -qq + apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + - name: Upgrade pip, Install dependencies + run: | + pip install --upgrade pip pipenv sphinx + pipenv lock -r > requirements.txt + pip install -r requirements.txt + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W + - name: Execute Tests + run: | + cd src/ + pytest --cov + pycodestyle + - name: Execute coveralls + run: | + coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 004ecc94b0fd9d6764d8f44dfe1a58f362e8f7b5 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:08:19 +0100 Subject: [PATCH 026/148] Use sudo for apt --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a2897c05b..f6ac55c1e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,8 +28,8 @@ jobs: key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | - apt-get update -qq - apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip, Install dependencies run: | pip install --upgrade pip pipenv sphinx From a83d5c39797178edd171d412c405fbdfbec8029f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:15:10 +0100 Subject: [PATCH 027/148] Execute pip commands one after the other to find issue --- .github/workflows/docker-publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f6ac55c1e..a810f134f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -30,10 +30,14 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip, Install dependencies + - name: Upgrade pip run: | pip install --upgrade pip pipenv sphinx + - name: Gather pip requirements + run: | pipenv lock -r > requirements.txt + - name: Insall pip requirements + run: | pip install -r requirements.txt - name: Run sphinx run: | From 192d27dd3c80de4adf23201e472a2f83b010daf2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:19:58 +0100 Subject: [PATCH 028/148] Move sphinx step below tests step as it used to be --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a810f134f..e020fb66b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,17 +36,17 @@ jobs: - name: Gather pip requirements run: | pipenv lock -r > requirements.txt - - name: Insall pip requirements + - name: Install pip requirements run: | pip install -r requirements.txt - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - name: Execute Tests run: | cd src/ pytest --cov pycodestyle + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W - name: Execute coveralls run: | coveralls From 3bb351186a13977b981d1b062b2218daf253255a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:23:49 +0100 Subject: [PATCH 029/148] Add pytest to install step --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e020fb66b..c074b1d2f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 0beba2eb1ed97ae5d41b041e48eeb9066fb8a6bd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:31:20 +0100 Subject: [PATCH 030/148] Add pytest-cov to install --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c074b1d2f..9bba2e031 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest + pip install --upgrade pip pipenv sphinx pytest pytest-cov - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 00c0634dcbd547a25a823881832bdb32b7dcea48 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:41:21 +0100 Subject: [PATCH 031/148] Remove cache to see if that helps with pytest-cov issues --- .github/workflows/docker-publish.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9bba2e031..090cfa937 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,15 +17,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | sudo apt-get update -qq From 7026f4143839083035f68a43e6adc1beadb95e8d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:44:08 +0100 Subject: [PATCH 032/148] Try o make pipenv also install dev dependencies --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 090cfa937..24c3fe062 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov + pip install --upgrade pip pipenv sphinx - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 9563cf576b572ed1ac0dd60126a66dee900f4735 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:23:30 +0100 Subject: [PATCH 033/148] github-actions: readding explicit install explicitly install pytest, pytest-cov and coveralls. I think travis already did that and we need to explicitly do so now. removed installing dev-dependencies, do it explicitly. (just trying to reproduce the travis-ci build) --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 24c3fe062..afbb2571f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | - pipenv lock --dev -r > requirements.txt + pipenv lock -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 8d2d040346e4b396858ec509a2413397a40c64b3 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:27:38 +0100 Subject: [PATCH 034/148] github-actions: removing unsupported parameter I don't understand what the "-n auto" parameter does, pytest doesn't either (in the version installed) - so removing it to see if that produces something usable. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index 2a1a348bd..f43c9adf6 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html -n auto +addopts = --pythonwarnings=all --cov --cov-report=html env = PAPERLESS_DISABLE_DBHANDLER=true From 07556450300ebea030506b49d847d1f74952a286 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:30:12 +0100 Subject: [PATCH 035/148] github-actions: also install wheel There's an error about pip wheel not being available and falling back to some legacy version. Let's see if this helps. --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index afbb2571f..a7a399c3f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,7 +23,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 0e73e56312259288d54eb8a9cd899e46bba2dd98 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:33:56 +0100 Subject: [PATCH 036/148] github-actions: readding previously removed "-n auto" possibly this is fixed by the wheel dependency that was missing before. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index f43c9adf6..2a1a348bd 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html +addopts = --pythonwarnings=all --cov --cov-report=html -n auto env = PAPERLESS_DISABLE_DBHANDLER=true From be9f04cf6cb77a3a17a3d6043183f12775e8a2e7 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:18:56 +0100 Subject: [PATCH 037/148] Explicitly set ubuntu-20.04 as OS and install dev dependencies --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a7a399c3f..d29fd7740 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,8 +9,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] - python-version: [3.7, 3.8, 3.9] + os: [ubuntu-20.04] + python-version: 3.7 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 2562f1b44397c56c3eeb1200a4ba74fde1843b08 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:22:35 +0100 Subject: [PATCH 038/148] Revert job strategy matrix change --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d29fd7740..cfda3d10d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: 3.7 + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From afdcc0908fb598d5f325aeac2f0d2eac0f8e8614 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:24:48 +0100 Subject: [PATCH 039/148] Only build python 3.7 and say so in the Pipfile too --- .github/workflows/docker-publish.yml | 2 +- Pipfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cfda3d10d..0d2c8bd68 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7, 3.8, 3.9] + python-version: [3.7] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/Pipfile b/Pipfile index 48759307c..e0f1d83d8 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.6" +python_version = "3.7" [packages] dateparser = "~=0.7.6" From a0cac680bc2e96845d71572104cbad01d25a764d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:00:35 +0100 Subject: [PATCH 040/148] Revert python version in Pipfile to 3.6 --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index e0f1d83d8..48759307c 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.7" +python_version = "3.6" [packages] dateparser = "~=0.7.6" From 89bec6de1ba4e9db43d049da9b03589eedc0ebd1 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:10:16 +0100 Subject: [PATCH 041/148] Try to reproduce exactly what's done in the travis build to test --- .github/workflows/docker-publish.yml | 30 ++++++++-------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0d2c8bd68..66bc73ea8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,37 +10,23 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Update system, install system requirements + - name: Tests run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip - run: | - pip install --upgrade pip pipenv - - name: Gather pip requirements - run: | - pipenv lock --dev -r > requirements.txt - - name: Install pip requirements - run: | - pip install -r requirements.txt - - name: Execute Tests - run: | + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev cd src/ - pytest --cov - pycodestyle - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - - name: Execute coveralls - run: | - coveralls + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 94083f8f195b9539fc8fcd0145f6c5d0135c7d91 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:17:13 +0100 Subject: [PATCH 042/148] Try manually replacing the python-version --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 66bc73ea8..264552168 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,6 +21,8 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + echo "Replacing python version with ${matrix.python-version}" + sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From a8fb681bd514bc16803ef3e8b79758417065c5cd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:19:00 +0100 Subject: [PATCH 043/148] Print the env --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 264552168..9904d8623 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -19,6 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | + env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng echo "Replacing python version with ${matrix.python-version}" From a59570e2d87ffa4e29875a20616a6895ebee3849 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:28:47 +0100 Subject: [PATCH 044/148] Only build for 3.6 --- .github/workflows/docker-publish.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9904d8623..23c8aef20 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6, 3.7, 3.8] + python-version: [3.6] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -19,11 +19,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | - env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - echo "Replacing python version with ${matrix.python-version}" - sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From 61fc77c49cd718daa192ec065f64370d12cd8c33 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:44:02 +0100 Subject: [PATCH 045/148] Add GITHUB_TOKEN to python tests --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 23c8aef20..01bf72595 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From b9c55eb80cbabacce7b208aa9a9e47b0cfa66453 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:45:12 +0100 Subject: [PATCH 046/148] Fix yaml --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 01bf72595..c0a1565c6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,7 +18,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From 15bd99f325328ba3e8270d1593d23d5dc693523a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 09:05:14 +0100 Subject: [PATCH 047/148] Try to make pip and pipenv use the installed python version --- .github/workflows/docker-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c0a1565c6..a61c570ce 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6] + python-version: [3.6,3.7,3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -20,10 +20,13 @@ jobs: - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MATRIX_PYTHON_VERSION: ${{strategy.matrix.python-version}} run: | + echo "Selected python version from matrix is: ${MATRIX_PYTHON_VERSION}" sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv + pyenv global system ${MATRIX_PYTHON_VERSION} pipenv install --system --dev cd src/ pipenv run pytest --cov From ee017d273c306f911312cd6b6b73156783f8b84e Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 09:10:45 +0100 Subject: [PATCH 048/148] Use another python action instead --- .github/workflows/docker-publish.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a61c570ce..ec60880c9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -12,21 +12,20 @@ jobs: os: [ubuntu-20.04] python-version: [3.6,3.7,3.8] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v5 + with: + default: "${{ matrix.python-version }}" + command: pip install -U pip # upgrade pip after installing python - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MATRIX_PYTHON_VERSION: ${{strategy.matrix.python-version}} run: | echo "Selected python version from matrix is: ${MATRIX_PYTHON_VERSION}" sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv - pyenv global system ${MATRIX_PYTHON_VERSION} pipenv install --system --dev cd src/ pipenv run pytest --cov From 092ad2c6f8217bdf136683d64eb8b5fd704a0d79 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 10:20:31 +0100 Subject: [PATCH 049/148] Remove comment from yaml --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ec60880c9..81eba7e4f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,7 +17,7 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python-version }}" - command: pip install -U pip # upgrade pip after installing python + command: pip install -U pip - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 81ac158bfb22952a2997d1b2893a4f98d47bb074 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 10:22:24 +0100 Subject: [PATCH 050/148] Remove command - should be unnecessary --- .github/workflows/docker-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 81eba7e4f..f13b7399e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,7 +17,6 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python-version }}" - command: pip install -U pip - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 551488f62c12e7b95563ddbf5f385283493394f4 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:48:03 +0100 Subject: [PATCH 051/148] Trying to use pyenv action step --- .github/workflows/docker-publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f13b7399e..490a5edec 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,13 +10,14 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6,3.7,3.8] + python: [3.6,3.7,3.8] steps: - uses: actions/checkout@v2 - name: Install python version uses: gabrielfalcao/pyenv-action@v5 with: - default: "${{ matrix.python-version }}" + default: "${{ matrix.python }}" + command: pip install -U pip # upgrade pip after installing python - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8f2492ea20a993b29950c3b72930741d6013cfe8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:50:11 +0100 Subject: [PATCH 052/148] List python versions differently so the pyenv action might light it better --- .github/workflows/docker-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 490a5edec..6507a6203 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,10 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python: [3.6,3.7,3.8] + python: + - 3.6 + - 3.7 + - 3.8 steps: - uses: actions/checkout@v2 - name: Install python version From 8a51f6e23622e247fe72581caef774d24a410c54 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:52:40 +0100 Subject: [PATCH 053/148] Specify exact version --- .github/workflows/docker-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6507a6203..203133335 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,9 +11,9 @@ jobs: matrix: os: [ubuntu-20.04] python: - - 3.6 - - 3.7 - - 3.8 + - 3.6.8 + - 3.7.5 + - 3.8.2 steps: - uses: actions/checkout@v2 - name: Install python version From 7e5fc1c2d3be5f1f5d7104bdfcb3e144cefb1b90 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:53:32 +0100 Subject: [PATCH 054/148] Remove comment --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 203133335..b0dc46ba5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -20,7 +20,7 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python }}" - command: pip install -U pip # upgrade pip after installing python + command: pip install -U pip - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 79e95dff8a297304cc7e7c211b48dd159b86ca05 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:54:11 +0100 Subject: [PATCH 055/148] Quote command --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b0dc46ba5..558ebea69 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -20,7 +20,7 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python }}" - command: pip install -U pip + command: "pip install -U pip" - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From ca9b05cd76fc69bb92212db27fc0c68c48b33a19 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:55:41 +0100 Subject: [PATCH 056/148] Update to a possibly working version of pyenv-action According to https://github.com/gabrielfalcao/pyenv-action/issues/136 this version should work --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 558ebea69..6b43b8710 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install python version - uses: gabrielfalcao/pyenv-action@v5 + uses: gabrielfalcao/pyenv-action@v7 with: default: "${{ matrix.python }}" command: "pip install -U pip" From 435a638dc266ee6fd3ffe0798dc0b4162dabedec Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:57:34 +0100 Subject: [PATCH 057/148] Fix yaml: indentation was wrong --- .github/workflows/docker-publish.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6b43b8710..1ec70f146 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,12 +15,12 @@ jobs: - 3.7.5 - 3.8.2 steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "${{ matrix.python }}" - command: "pip install -U pip" + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "${{ matrix.python }}" + command: "pip install -U pip" - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 65ee707de48c6f9978249422b7552cab30e48004 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 23:06:40 +0100 Subject: [PATCH 058/148] Remove debugging line and specify release versions --- .github/workflows/docker-publish.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1ec70f146..5e9ed8632 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,9 +11,9 @@ jobs: matrix: os: [ubuntu-20.04] python: - - 3.6.8 - - 3.7.5 - - 3.8.2 + - 3.6 + - 3.7 + - 3.8 steps: - uses: actions/checkout@v2 - name: Install python version @@ -25,7 +25,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo "Selected python version from matrix is: ${MATRIX_PYTHON_VERSION}" sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv From 51d29f998afc0083a38e219edc28a9a8e470fa80 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 23:10:06 +0100 Subject: [PATCH 059/148] Looks like we need to specify the exact version --- .github/workflows/docker-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5e9ed8632..042d64f24 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,9 +11,9 @@ jobs: matrix: os: [ubuntu-20.04] python: - - 3.6 - - 3.7 - - 3.8 + - 3.6.8 + - 3.7.9 + - 3.8.7 steps: - uses: actions/checkout@v2 - name: Install python version From 7987a278cf80f5fcb050719c5c838711707711ec Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 23:11:54 +0100 Subject: [PATCH 060/148] Use 3.8.6 --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 042d64f24..8a199abcd 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,7 +13,7 @@ jobs: python: - 3.6.8 - 3.7.9 - - 3.8.7 + - 3.8.6 steps: - uses: actions/checkout@v2 - name: Install python version From dd4d908d44c519a41e5fa664ea906966936f7179 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:32:43 +0100 Subject: [PATCH 061/148] Docker build step depends on tests --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8a199abcd..649fba14d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,7 +36,7 @@ jobs: # Build and push image to docker hub. buildx: runs-on: ubuntu-latest - + needs: tests steps: - uses: actions/checkout@v2 From bc1f479e1c7db65f8498f1855a510099ad6500d6 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:42:25 +0100 Subject: [PATCH 062/148] Adding documentation and frontend steps from travis build --- .github/workflows/docker-publish.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 649fba14d..50fdbfd31 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -33,6 +33,33 @@ jobs: pipenv run pytest --cov pipenv run pycodestyle pipenv run coveralls + documentation: + steps: + - uses: actions/checkout@v2 + - name: dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + - name: make + run: | + cd docs/ + make html + frontend: + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: dependencies + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + - name: build + run: | + ng build --prod # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From f7cf842342004f337bd96fbb4671df54c50583b0 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:43:45 +0100 Subject: [PATCH 063/148] Fix identation --- .github/workflows/docker-publish.yml | 54 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 50fdbfd31..74e5c0ac7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -33,33 +33,33 @@ jobs: pipenv run pytest --cov pipenv run pycodestyle pipenv run coveralls - documentation: - steps: - - uses: actions/checkout@v2 - - name: dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - - name: make - run: | - cd docs/ - make html - frontend: - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '15' - - name: dependencies - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - - name: build - run: | - ng build --prod + documentation: + steps: + - uses: actions/checkout@v2 + - name: dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + - name: make + run: | + cd docs/ + make html + frontend: + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: dependencies + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + - name: build + run: | + ng build --prod # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From f7c6cd4e8ae147feef82edab13140c53325a38f6 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:44:36 +0100 Subject: [PATCH 064/148] Add missing runs-on --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 74e5c0ac7..76a7f18e2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -34,6 +34,7 @@ jobs: pipenv run pycodestyle pipenv run coveralls documentation: + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: dependencies @@ -47,6 +48,7 @@ jobs: cd docs/ make html frontend: + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 From ba629ea726ef0e41f41c3105b6c5d3be391159e4 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:45:17 +0100 Subject: [PATCH 065/148] ADd documentation and frontend to dependency for docker build --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 76a7f18e2..f2467cbf9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -65,7 +65,7 @@ jobs: # Build and push image to docker hub. buildx: runs-on: ubuntu-latest - needs: tests + needs: [tests, documentation, frontend] steps: - uses: actions/checkout@v2 From b14cbd4963b14ad108040f46ec5f5ccf2fbe892b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:47:42 +0100 Subject: [PATCH 066/148] ALso install python before using pip in the documentation job --- .github/workflows/docker-publish.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f2467cbf9..6c3bf87f9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -37,6 +37,11 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "3.6.8" + command: "pip install -U pip" - name: dependencies run: | sudo apt-get update -qq From 3fb26fab9527c5bd1f5d6cfc2e858f5d7a051115 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:48:32 +0100 Subject: [PATCH 067/148] Simplify node build --- .github/workflows/docker-publish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6c3bf87f9..ab93fd8e7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -59,13 +59,11 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '15' - - name: dependencies + - name: build run: | cd src-ui/ npm install -g @angular/cli npm install - - name: build - run: | ng build --prod # Build and push image to docker hub. buildx: From a3f58634979e5eefef4dab760ec9956acbcaf28c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:57:12 +0100 Subject: [PATCH 068/148] Try installing sphinx simply with apt --- .github/workflows/docker-publish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ab93fd8e7..3d62a9f18 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -45,9 +45,7 @@ jobs: - name: dependencies run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev + sudo apt-get install -qq python3-sphinx - name: make run: | cd docs/ From 62c0dc55384ce5508f0f9adf4f51977bb544178f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:37:25 +0100 Subject: [PATCH 069/148] Use pip to install sphinx Not sure why pipenv install doesn't do the trick .. let's try this way first --- .github/workflows/docker-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3d62a9f18..54281492e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -44,8 +44,7 @@ jobs: command: "pip install -U pip" - name: dependencies run: | - sudo apt-get update -qq - sudo apt-get install -qq python3-sphinx + pip install 'sphinx~=3.3' sphinx_rtd_theme - name: make run: | cd docs/ From 033264013176d9f0e034a92e789dab184377bd98 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:44:05 +0100 Subject: [PATCH 070/148] Add pip cache Try to be a good citizen and use caches instead of hitting the repos all the time --- .github/workflows/docker-publish.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 54281492e..bce52b2e0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,6 +21,15 @@ jobs: with: default: "${{ matrix.python }}" command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python }} - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 3181eac6dfa270f15e402182b1ee414538aaa054 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:50:04 +0100 Subject: [PATCH 071/148] Try to insall sphinx from pipfile --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index bce52b2e0..479f52a2c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -53,7 +53,8 @@ jobs: command: "pip install -U pip" - name: dependencies run: | - pip install 'sphinx~=3.3' sphinx_rtd_theme + pip install --upgrade pipenv + pipenv install --system --dev --ignore-pipfile - name: make run: | cd docs/ From 7118890f856bb08798c44f06f7edcea30fd0948c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:56:48 +0100 Subject: [PATCH 072/148] Install pip --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 479f52a2c..fcdf93c8f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -54,7 +54,7 @@ jobs: - name: dependencies run: | pip install --upgrade pipenv - pipenv install --system --dev --ignore-pipfile + pipenv install --system --dev - name: make run: | cd docs/ From 205bb6c6ce07f226cb3adaf6056f56a484093174 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:02:29 +0100 Subject: [PATCH 073/148] Try to use pip cache and install dependnecies like in tests --- .github/workflows/docker-publish.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index fcdf93c8f..a3329b7ba 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,9 +50,20 @@ jobs: uses: gabrielfalcao/pyenv-action@v7 with: default: "3.6.8" - command: "pip install -U pip" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip3.6.8} - name: dependencies run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv pipenv install --system --dev - name: make From 7e7fc47ea2ac140cfbdc7cac9901e2ee55193821 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 7 Jan 2021 18:42:46 +0100 Subject: [PATCH 074/148] update docker ignore --- .dockerignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.dockerignore b/.dockerignore index 065d2a7e6..a332c2ee7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +**/__pycache__ /src-ui/.vscode /src-ui/node_modules /src-ui/dist @@ -10,3 +11,9 @@ .pytest_cache /dist /scripts +/resources +**/tests +**/*.spec.ts +**/htmlcov +/src/.pytest_cache +.idea From 89f97069111918d997fa504dedd8522201e0e417 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 7 Jan 2021 21:16:16 +0100 Subject: [PATCH 075/148] update installation instructions --- docker/hub/docker-compose.postgres.yml | 2 +- docker/hub/docker-compose.sqlite.yml | 2 +- docker/hub/docker-compose.tika.yml | 2 +- docs/administration.rst | 44 ++------ docs/setup.rst | 133 +++++++++++++------------ docs/troubleshooting.rst | 2 +- 6 files changed, 81 insertions(+), 104 deletions(-) diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml index d7673d891..f821a6781 100644 --- a/docker/hub/docker-compose.postgres.yml +++ b/docker/hub/docker-compose.postgres.yml @@ -15,7 +15,7 @@ services: POSTGRES_PASSWORD: paperless webserver: - image: jonaswinkler/paperless-ng:0.9.12 + image: jonaswinkler/paperless-ng:latest restart: always depends_on: - db diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml index 054eb52d3..cc26dc46c 100644 --- a/docker/hub/docker-compose.sqlite.yml +++ b/docker/hub/docker-compose.sqlite.yml @@ -5,7 +5,7 @@ services: restart: always webserver: - image: jonaswinkler/paperless-ng:0.9.12 + image: jonaswinkler/paperless-ng:latest restart: always depends_on: - broker diff --git a/docker/hub/docker-compose.tika.yml b/docker/hub/docker-compose.tika.yml index 47c477bef..7ad7685d3 100644 --- a/docker/hub/docker-compose.tika.yml +++ b/docker/hub/docker-compose.tika.yml @@ -5,7 +5,7 @@ services: restart: always webserver: - image: jonaswinkler/paperless-ng:0.9.12 + image: jonaswinkler/paperless-ng:latest restart: always depends_on: - broker diff --git a/docs/administration.rst b/docs/administration.rst index 727d9a9d5..bc728943e 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -53,7 +53,7 @@ Restoring .. _administration-updating: -Updating paperless +Updating Paperless ################## If a new release of paperless-ng is available, upgrading depends on how you @@ -69,50 +69,26 @@ First of all, ensure that paperless is stopped. After that, :ref:`make a backup `. -A. If you used the dockerfiles archive, simply download the files of the new release, - adjust the settings in the files (i.e., the path to your consumption directory), - and replace your existing docker-compose files. Then start paperless as usual, - which will pull the new image, and update your database, if necessary: +A. If you pull the image from the docker hub, all you need to do is: .. code:: shell-session - $ cd /path/to/paperless + $ docker-compose pull $ docker-compose up - - If you see everything working, you can start paperless-ng with "-d" to have it - run in the background. - - .. hint:: - - The released docker-compose files specify exact versions to be pulled from the hub. - This is to ensure that if the docker-compose files should change at some point - (i.e., services updates/configured differently), you wont run into trouble due to - docker pulling the ``latest`` image and running it in an older environment. + + The docker-compose files refer to the ``latest`` version, which is always the latest + stable release. -B. If you built the image yourself, grab the new archive and replace your current - paperless folder with the new contents. - - After that, make the necessary adjustments to the docker-compose.yml (i.e., - adjust your consumption directory). - - Build and start the new image with: +B. If you built the image yourself, do the following: .. code:: shell-session - $ cd /path/to/paperless + $ git pull $ docker-compose build $ docker-compose up - If you see everything working, you can start paperless-ng with "-d" to have it - run in the background. - -.. hint:: - - You can usually keep your ``docker-compose.env`` file, since this file will - never include mandatory configuration options. However, it is worth checking - out the new version of this file, since it might have new recommendations - on what to configure. - +If you see everything working, you can start paperless-ng with "-d" to have it +run in the background. Updating paperless without docker ================================= diff --git a/docs/setup.rst b/docs/setup.rst index ddd246b16..5a38c3241 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -3,35 +3,6 @@ Setup ***** -Download -######## - -Go to the project page on GitHub and download the -`latest release `_. -There are multiple options available. - -* Download the dockerfiles archive if you want to pull paperless from - Docker Hub. - -* Download the dist archive and extract it if you want to build the docker image - yourself or want to install paperless without docker. - -.. hint:: - - In contrast to paperless, the recommended way to get and update paperless-ng - is not to pull the entire git repository. Paperless-ng includes artifacts - that need to be compiled, and that's already done for you in the release. - -.. admonition:: Want to try out paperless-ng before migrating? - - The release contains a file ``.env`` which sets the docker-compose project - name to "paperless", which is the same as before and instructs docker-compose - to reuse and upgrade your paperless volumes. - - Just rename the project name in that file to anything else and docker-compose - will create fresh volumes for you! - - Overview of Paperless-ng ######################## @@ -110,22 +81,33 @@ Installation You can go multiple routes with setting up and running Paperless: -* The `docker route`_ -* The `bare metal route`_ +* :ref:`Pull the image from Docker Hub ` +* :ref:`Build the Docker image yourself ` +* :ref:`Install Paperless directly on your system (bare metal) ` -The `docker route`_ is quick & easy. This is the recommended route. This configures all the stuff +The Docker routes are quick & easy. These are the recommended routes. This configures all the stuff from above automatically so that it just works and uses sensible defaults for all configuration options. -The `bare metal route`_ is more complicated to setup but makes it easier +The bare metal route is more complicated to setup but makes it easier should you want to contribute some code back. You need to configure and run the above mentioned components yourself. -.. _setup-docker_route: +.. _setup-docker_hub: -Docker Route -============ +Install Paperless from Docker Hub +================================= -1. Install `Docker`_ and `docker-compose`_. [#compose]_ +1. Go to the `project page `_ + and download one of the ``docker-compose.*.yml`` files, depending on which database backend you + want to use. Download the ``docker-compose.env`` file and the ``.env`` file as well and store them + in the same directory. + + .. hint:: + + For new installations, it is recommended to use PostgreSQL as the database + backend. + +2. Install `Docker`_ and `docker-compose`_. .. caution:: @@ -142,15 +124,8 @@ Docker Route .. _Docker installation guide: https://docs.docker.com/engine/installation/ .. _docker-compose installation guide: https://docs.docker.com/compose/install/ -2. Copy either ``docker-compose.sqlite.yml`` or ``docker-compose.postgres.yml`` to - ``docker-compose.yml``, depending on which database backend you want to use. - .. hint:: - - For new installations, it is recommended to use PostgreSQL as the database - backend. - -2. Modify ``docker-compose.yml`` to your preferences. You may want to change the path +3. Modify ``docker-compose.yml`` to your preferences. You may want to change the path to the consumption directory in this file. Find the line that specifies where to mount the consumption directory: @@ -167,7 +142,7 @@ Docker Route Don't change the part after the colon or paperless wont find your documents. -3. Modify ``docker-compose.env``, following the comments in the file. The +4. Modify ``docker-compose.env``, following the comments in the file. The most important change is to set ``USERMAP_UID`` and ``USERMAP_GID`` to the uid and gid of your user on the host system. This ensures that both the docker container and you on the host machine have write access @@ -177,7 +152,7 @@ Docker Route .. note:: - You can use any settings from the file ``paperless.conf`` in this file. + You can use any settings from the file ``paperless.conf.example`` in this file. Have a look at :ref:`configuration` to see whats available. .. caution:: @@ -188,11 +163,10 @@ Docker Route with the default configuration. You will need to use ``PAPERLESS_CONSUMER_POLLING``, which will disable inotify. See :ref:`here `. -4. Run ``docker-compose up -d``. This will create and start the necessary - containers. This will also build the image of paperless if you grabbed the - source archive. +5. Run ``docker-compose up -d``. This will create and start the necessary + containers. -5. To be able to login, you will need a super user. To create it, execute the +6. To be able to login, you will need a super user. To create it, execute the following command: .. code-block:: shell-session @@ -202,7 +176,7 @@ Docker Route This will prompt you to set a username, an optional e-mail address and finally a password. -6. The default ``docker-compose.yml`` exports the webserver on your local port +7. The default ``docker-compose.yml`` exports the webserver on your local port 8000. If you haven't adapted this, you should now be able to visit your Paperless instance at ``http://127.0.0.1:8000``. You can login with the user and password you just created. @@ -210,11 +184,33 @@ Docker Route .. _Docker: https://www.docker.com/ .. _docker-compose: https://docs.docker.com/compose/install/ -.. [#compose] You of course don't have to use docker-compose, but it - simplifies deployment immensely. If you know your way around Docker, feel - free to tinker around without using compose! +.. _setup-docker_build: -.. _`setup-bare_metal`: +Build the docker image yourself +=============================== + +1. Clone the entire repository of paperless: + + .. code:: shell-session + + git clone https://github.com/jonaswinkler/paperless-ng + + The master branch always reflects the latest stable version. + +2. Copy one of the ``docker-compose.*.yml`` to ``docker-compose.yml``, + depending on which database backend you want to use. Copy + ``docker-compose.env.example`` to ``docker-compose.env``. + +3. Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run + ``docker-compose up -d`` to start the containers, do + + .. code:: shell-session + + $ docker-compose build + + before that to build the image. + +.. _setup-bare_metal: Bare Metal Route ================ @@ -229,6 +225,7 @@ writing. Windows is not and will never be supported. * ``python3-pip``, optionally ``pipenv`` for package installation * ``python3-dev`` + * ``gettext`` for localization * ``fonts-liberation`` for generating thumbnails for plain text files * ``imagemagick`` >= 6 for PDF conversion * ``optipng`` for optimizing thumbnails @@ -258,8 +255,9 @@ writing. Windows is not and will never be supported. 3. Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish to use PostgreSQL, SQLite is avialable as well. -4. Get the release archive. If you pull the git repo as it is, you also have to compile the front end by yourself. - Extract the frontend to a place from where you wish to execute it, such as ``/opt/paperless``. +4. Get the release archive from ``_. + If you clone the git repo as it is, you also have to compile the front end by yourself. + Extract the archive to a place from where you wish to execute it, such as ``/opt/paperless``. 5. Configure paperless. See :ref:`configuration` for details. Edit the included ``paperless.conf`` and adjust the settings to your needs. Required settings for getting paperless running are: @@ -372,6 +370,11 @@ writing. Windows is not and will never be supported. .. code:: + +13. Optional: Install the `jbig2enc `_ + encoder. This will reduce the size of generated PDF documents. You'll most likely need + to compile this by yourself, because this software has been patented until around 2017 and + binary packages are not available for most distributions. Migration to paperless-ng ######################### @@ -406,13 +409,14 @@ Migration to paperless-ng is then performed in a few simple steps: paperless. 3. Download the latest release of paperless-ng. You can either go with the - docker-compose files or use the archive to build the image yourself. + docker-compose files from `here `_ + or clone the repository to build the image yourself. You can either replace your current paperless folder or put paperless-ng in a different location. .. caution:: - The release include a ``.env`` file. This will set the + Paperless includes a ``.env`` file. This will set the project name for docker compose to ``paperless`` so that paperless-ng will automatically reuse your existing paperless volumes. When you start it, it will migrate your existing data. After that, your old paperless installation @@ -422,9 +426,8 @@ Migration to paperless-ng is then performed in a few simple steps: If you want to switch to PostgreSQL, do that after you migrated your existing SQLite database. -5. Adjust ``docker-compose.yml`` and - ``docker-compose.env`` to your needs. - See `docker route`_ for details on which edits are advised. +5. Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs. + See :ref:`setup-docker_hub` for details on which edits are advised. 6. Since ``docker-compose`` would just use the the old paperless image, we need to manually build a new image: @@ -452,7 +455,7 @@ Migration to paperless-ng is then performed in a few simple steps: This will run paperless in the background and automatically start it on system boot. 9. Paperless installed a permanent redirect to ``admin/`` in your browser. This - redirect is still in place and prevents access to the new UI. Clear + redirect is still in place and prevents access to the new UI. Clear your browsing cache in order to fix this. 10. Optionally, follow the instructions below to migrate your existing data to PostgreSQL. @@ -604,6 +607,4 @@ For details, refer to :ref:`configuration`. The actual matching of the algorithm is fast and works on Raspberry Pi as well as on any other device. - - .. _redis: https://redis.io/ diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 8786657db..b8343710f 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -75,6 +75,6 @@ You might encounter errors such as: This happens when paperless does not have permission to delete files inside the consumption directory. Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group id you use on the host operating system, if these are -different from ``1000``. See :ref:`setup-docker_route`. +different from ``1000``. See :ref:`setup-docker_hub`. Also ensure that you are able to read and write to the consumption directory on the host. From 9e7257634a06a5409a3ebef0f10de1ded9a9c7bb Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 11:23:17 +0100 Subject: [PATCH 076/148] Testing excluding docker builds if not specific ref is selected --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a3329b7ba..14a4eb042 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -87,6 +87,7 @@ jobs: buildx: runs-on: ubuntu-latest needs: [tests, documentation, frontend] + if: github.ref == 'refs/heads/github-action-test' steps: - uses: actions/checkout@v2 From 729f0178037a23dfb1bead0c4de5f999d88b707c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 11:33:08 +0100 Subject: [PATCH 077/148] Separate github action into separete workflow files --- .../{docker-publish.yml => docker.yml} | 78 ------------------- .github/workflows/documentation.yml | 32 ++++++++ .github/workflows/frontend.yml | 16 ++++ .github/workflows/test.yml | 43 ++++++++++ 4 files changed, 91 insertions(+), 78 deletions(-) rename .github/workflows/{docker-publish.yml => docker.yml} (50%) create mode 100644 .github/workflows/documentation.yml create mode 100644 .github/workflows/frontend.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker.yml similarity index 50% rename from .github/workflows/docker-publish.yml rename to .github/workflows/docker.yml index 14a4eb042..338ae85a9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker.yml @@ -5,84 +5,6 @@ on: branches: travis-multiarch-builds jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - python: - - 3.6.8 - - 3.7.9 - - 3.8.6 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "${{ matrix.python }}" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python }} - - name: Tests - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - cd src/ - pipenv run pytest --cov - pipenv run pycodestyle - pipenv run coveralls - documentation: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "3.6.8" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip3.6.8} - - name: dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - - name: make - run: | - cd docs/ - make html - frontend: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '15' - - name: build - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - ng build --prod # Build and push image to docker hub. buildx: runs-on: ubuntu-latest diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 000000000..2e2e84e84 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,32 @@ +name: backend-tests + +# always run documentation workflow to ensure the documentation can still be built +on: push + +# possibly add a publish job for automatic read-the-docs publishing +jobs: + documentation: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "3.6.8" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip3.6.8} + - name: dependencies + run: | + pip install 'sphinx~=3.3' sphinx_rtd_theme + - name: make + run: | + cd docs/ + make html \ No newline at end of file diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 000000000..64df45127 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,16 @@ + + +jobs: + frontend: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: build + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + ng build --prod \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..930f368e7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: backend-tests + +# always run tests workflow +on: push + +jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + python: + - 3.6.8 + - 3.7.9 + - 3.8.6 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "${{ matrix.python }}" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python }} + - name: Tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + cd src/ + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls From f35c843af7bf18e844c4092a42b9cbf1370f9e56 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 11:34:04 +0100 Subject: [PATCH 078/148] Give the frontend workflow a name --- .github/workflows/frontend.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 64df45127..ad44fb717 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -1,4 +1,7 @@ +name: frontend +# always run frontend workflow +on: push jobs: frontend: From bd8f462fae0a048e70e5cecf79a19815bed5dbd1 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 20:37:25 +0100 Subject: [PATCH 079/148] github action improved tagging --- .github/workflows/docker.yml | 24 +++++++++++++----------- .github/workflows/documentation.yml | 1 + .github/workflows/frontend.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 338ae85a9..715474a85 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,32 +1,34 @@ -name: Docker +name: docker on: push: - branches: travis-multiarch-builds + branches: + - master + - dev + tags: + - 'ng-*' jobs: - # Build and push image to docker hub. + # build and push image to docker hub. buildx: runs-on: ubuntu-latest - needs: [tests, documentation, frontend] - if: github.ref == 'refs/heads/github-action-test' steps: - uses: actions/checkout@v2 - - name: Prepare id: prepare + env: + DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} run: | - DOCKER_IMAGE=moztr/paperless-ng DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/v} + VERSION=${GITHUB_REF#refs/tags/} fi - if [ "${{ github.event_name }}" = "schedule" ]; then - VERSION=nightly + if [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=${GITHUB_REF#refs/heads/} fi TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + if [[ $VERSION == "master" ]]; then TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" fi echo ::set-output name=docker_image::${DOCKER_IMAGE} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 2e2e84e84..12bd3577d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -24,6 +24,7 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip3.6.8} - name: dependencies + # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? run: | pip install 'sphinx~=3.3' sphinx_rtd_theme - name: make diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index ad44fb717..66a9a9b0c 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -1,6 +1,6 @@ name: frontend -# always run frontend workflow +# always run frontend workflow to see if it builds and tests run fine on: push jobs: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 930f368e7..1a235f763 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,8 +8,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] - python: + os: ubuntu-20.04] + python: # pyenv-action requires the exact version - 3.6.8 - 3.7.9 - 3.8.6 From 1ffe4e6f646a3a3a31a833460475fbb8192c65f8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 20:39:20 +0100 Subject: [PATCH 080/148] Fix typo in yaml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a235f763..ec601d57e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: ubuntu-20.04] + os: [ubuntu-20.04] python: # pyenv-action requires the exact version - 3.6.8 - 3.7.9 From af0834236ddd7afec5c337bc526ee36e20af1108 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 20:42:17 +0100 Subject: [PATCH 081/148] Fix name of documentation workflow --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 12bd3577d..67047d6cd 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,4 +1,4 @@ -name: backend-tests +name: documentation # always run documentation workflow to ensure the documentation can still be built on: push From d89c045049b7c78f3c7eff53d177f0498e6005ea Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 9 Jan 2021 08:06:55 +0100 Subject: [PATCH 082/148] Run tests, documentation and frontend build also on pull_request --- .github/workflows/documentation.yml | 2 +- .github/workflows/frontend.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 67047d6cd..00835d240 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,7 +1,7 @@ name: documentation # always run documentation workflow to ensure the documentation can still be built -on: push +on: [push, pull_request] # possibly add a publish job for automatic read-the-docs publishing jobs: diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 66a9a9b0c..4cb36d3a4 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -1,7 +1,7 @@ name: frontend # always run frontend workflow to see if it builds and tests run fine -on: push +on: [push, pull_request] jobs: frontend: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec601d57e..9766c4b5b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: backend-tests # always run tests workflow -on: push +on: [push, pull_request] jobs: tests: From 4eac7398fb3c56fd676bd55c58f25aabc18dd805 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 15:11:58 +0100 Subject: [PATCH 083/148] enable docker builds on this branch --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 715474a85..b6f75df24 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,6 +5,7 @@ on: branches: - master - dev + - travis-multiarch-builds tags: - 'ng-*' @@ -18,8 +19,9 @@ jobs: id: prepare env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} + # ,linux/arm/v7,linux/arm64/v8 run: | - DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 + DOCKER_PLATFORMS=linux/amd64 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then VERSION=${GITHUB_REF#refs/tags/} From 38deea3e27117b3ef55a4e2db164ec54938038d6 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 17:54:56 +0100 Subject: [PATCH 084/148] now with cache? --- .github/workflows/docker.yml | 87 ++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b6f75df24..185856157 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,51 +14,80 @@ jobs: buildx: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Prepare + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: crazy-max/ghaction-docker-buildx@v3.1.0 + - + name: Cache Docker layers + uses: actions/cache@v2 + id: cache + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - + name: Prepare id: prepare env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} # ,linux/arm/v7,linux/arm64/v8 run: | DOCKER_PLATFORMS=linux/amd64 - VERSION=edge + DOCKER_VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} + DOCKER_VERSION=${GITHUB_REF#refs/tags/} fi if [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=${GITHUB_REF#refs/heads/} + DOCKER_VERSION=${GITHUB_REF#refs/heads/} fi - TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION == "master" ]]; then - TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + DOCKER_TAG="${DOCKER_IMAGE}:${DOCKER_VERSION}" + if [[ $DOCKER_VERSION == "master" ]]; then + DOCKER_TAG="${DOCKER_IMAGE}:latest" fi - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=version::${VERSION} - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ - --build-arg VERSION=${VERSION} \ + echo ::set-output name=version::${DOCKER_VERSION} + echo ::set-output name=platforms::${DOCKER_PLATFORMS} + echo ::set-output name=tag::${DOCKER_TAG} + - + name: Docker buildx (build) + run: | + docker buildx build \ + --cache-from "type=local,src=/tmp/.buildx-cache" \ + --cache-to "type=local,dest=/tmp/.buildx-cache" \ + --platform ${{steps.prepare.outputs.platforms}} \ + --output "type=image,push=false" \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ - ${TAGS} --file ./Dockerfile . - - name: install buildx - id: buildx - uses: crazy-max/ghaction-docker-buildx@v3.1.0 - - name: Docker Login - if: success() && github.event_name != 'pull_request' - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + --platform ${{steps.prepare.outputs.platforms}} \ + --tag ${{steps.prepare.outputs.tag}} ./ + + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - + name: Docker Buildx (push) run: | - echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin - - name: Docker Buildx (push) - if: success() && github.event_name != 'pull_request' - run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build \ + --cache-from "type=local,src=/tmp/.buildx-cache" \ + --platform ${{steps.prepare.outputs.platforms}} \ + --output "type=image,push=true" \ + --tag ${{steps.prepare.outputs.tag}} ./ + - name: Docker Check Manifest - if: always() && github.event_name != 'pull_request' run: | - docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} + - + name: Inspect image + run: | + docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} + - name: Clear - if: always() && github.event_name != 'pull_request' run: | rm -f ${HOME}/.docker/config.json From 950fe0f29630a580b23faca7a06315fcaaa2b6e5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:02:36 +0100 Subject: [PATCH 085/148] arm --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 185856157..f858cd57e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,9 +34,8 @@ jobs: id: prepare env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} - # ,linux/arm/v7,linux/arm64/v8 run: | - DOCKER_PLATFORMS=linux/amd64 + DOCKER_PLATFORMS=linux/amd64,linux/arm/v7 DOCKER_VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then DOCKER_VERSION=${GITHUB_REF#refs/tags/} From 7c4ef5d627b744394fc73dc16837d7a63ef399b4 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:25:56 +0100 Subject: [PATCH 086/148] new buildx setup action --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f858cd57e..0af841f24 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Docker Buildx - uses: crazy-max/ghaction-docker-buildx@v3.1.0 + uses: docker/setup-buildx-action@v1 - name: Cache Docker layers uses: actions/cache@v2 @@ -35,7 +35,7 @@ jobs: env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} run: | - DOCKER_PLATFORMS=linux/amd64,linux/arm/v7 + DOCKER_PLATFORMS=linux/amd64 DOCKER_VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then DOCKER_VERSION=${GITHUB_REF#refs/tags/} From 0a8b56757af8d16ca5e7fbdb1e81ac1852b4653a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:33:58 +0100 Subject: [PATCH 087/148] duplicate platform specification --- .github/workflows/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0af841f24..8cf13cc37 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -56,7 +56,6 @@ jobs: docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ --cache-to "type=local,dest=/tmp/.buildx-cache" \ - --platform ${{steps.prepare.outputs.platforms}} \ --output "type=image,push=false" \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ From d2536a5a20a8b73059968cb888c6864f9e1bc341 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:49:21 +0100 Subject: [PATCH 088/148] disable some steps --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8cf13cc37..eafb80592 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -71,6 +71,8 @@ jobs: - name: Docker Buildx (push) + + if: false run: | docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ @@ -79,10 +81,12 @@ jobs: --tag ${{steps.prepare.outputs.tag}} ./ - name: Docker Check Manifest + if: false run: | docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} - name: Inspect image + if: false run: | docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} From 59b1c76d94bb55f4f3e265f5bf132ae48cc16026 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <17569239+jonaswinkler@users.noreply.github.com> Date: Sat, 9 Jan 2021 19:02:07 +0100 Subject: [PATCH 089/148] Update docker.yml --- .github/workflows/docker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eafb80592..9083af82f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -71,8 +71,6 @@ jobs: - name: Docker Buildx (push) - - if: false run: | docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ From d21919d471b1602306b0e76e05a2585950d383bb Mon Sep 17 00:00:00 2001 From: Jonas Winkler <17569239+jonaswinkler@users.noreply.github.com> Date: Sat, 9 Jan 2021 19:06:00 +0100 Subject: [PATCH 090/148] Update docker.yml --- .github/workflows/docker.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9083af82f..d0c5a97b0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -68,7 +68,6 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Docker Buildx (push) run: | @@ -77,17 +76,11 @@ jobs: --platform ${{steps.prepare.outputs.platforms}} \ --output "type=image,push=true" \ --tag ${{steps.prepare.outputs.tag}} ./ - - - name: Docker Check Manifest - if: false + - + name: Docker Check Manifest run: | docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} - name: Inspect image - if: false run: | docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} - - - name: Clear - run: | - rm -f ${HOME}/.docker/config.json From 1b6fd67d1915e5f8c60145fca559a83ab0cb965c Mon Sep 17 00:00:00 2001 From: Jonas Winkler <17569239+jonaswinkler@users.noreply.github.com> Date: Sat, 9 Jan 2021 19:10:09 +0100 Subject: [PATCH 091/148] Update docker.yml --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d0c5a97b0..56a6d1b39 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,9 +43,9 @@ jobs: if [[ $GITHUB_REF == refs/heads/* ]]; then DOCKER_VERSION=${GITHUB_REF#refs/heads/} fi - DOCKER_TAG="${DOCKER_IMAGE}:${DOCKER_VERSION}" + DOCKER_TAG="jonaswinkler/paperless-ng-test:${DOCKER_VERSION}" if [[ $DOCKER_VERSION == "master" ]]; then - DOCKER_TAG="${DOCKER_IMAGE}:latest" + DOCKER_TAG="jonaswinkler/paperless-ng-test:latest" fi echo ::set-output name=version::${DOCKER_VERSION} echo ::set-output name=platforms::${DOCKER_PLATFORMS} From 9daf7278fa9cdfbfdc5eb98ae988ed7b96409de0 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 19:31:41 +0100 Subject: [PATCH 092/148] tags? --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 56a6d1b39..585ec61e4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -60,7 +60,7 @@ jobs: --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ --platform ${{steps.prepare.outputs.platforms}} \ - --tag ${{steps.prepare.outputs.tag}} ./ + --tag jonaswinkler/paperless-ng:edge ./ - name: Login to DockerHub @@ -75,8 +75,8 @@ jobs: --cache-from "type=local,src=/tmp/.buildx-cache" \ --platform ${{steps.prepare.outputs.platforms}} \ --output "type=image,push=true" \ - --tag ${{steps.prepare.outputs.tag}} ./ - - + --tag jonaswinkler/paperless-ng:edge ./ + - name: Docker Check Manifest run: | docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} From bfa04acc87022c986fcedd0293c16c3e1312df73 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 19:50:11 +0100 Subject: [PATCH 093/148] simplify some things --- .github/workflows/docker.yml | 61 +++++++++--------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 585ec61e4..cda034085 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,10 @@ on: tags: - 'ng-*' + +# --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ +# --build-arg VCS_REF=${GITHUB_SHA::8} \ + jobs: # build and push image to docker hub. buildx: @@ -23,45 +27,11 @@ jobs: - name: Cache Docker layers uses: actions/cache@v2 - id: cache with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - - - name: Prepare - id: prepare - env: - DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} - run: | - DOCKER_PLATFORMS=linux/amd64 - DOCKER_VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/tags/} - fi - if [[ $GITHUB_REF == refs/heads/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/heads/} - fi - DOCKER_TAG="jonaswinkler/paperless-ng-test:${DOCKER_VERSION}" - if [[ $DOCKER_VERSION == "master" ]]; then - DOCKER_TAG="jonaswinkler/paperless-ng-test:latest" - fi - echo ::set-output name=version::${DOCKER_VERSION} - echo ::set-output name=platforms::${DOCKER_PLATFORMS} - echo ::set-output name=tag::${DOCKER_TAG} - - - name: Docker buildx (build) - run: | - docker buildx build \ - --cache-from "type=local,src=/tmp/.buildx-cache" \ - --cache-to "type=local,dest=/tmp/.buildx-cache" \ - --output "type=image,push=false" \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg VCS_REF=${GITHUB_SHA::8} \ - --platform ${{steps.prepare.outputs.platforms}} \ - --tag jonaswinkler/paperless-ng:edge ./ - - name: Login to DockerHub uses: docker/login-action@v1 @@ -69,18 +39,17 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Docker Buildx (push) - run: | - docker buildx build \ - --cache-from "type=local,src=/tmp/.buildx-cache" \ - --platform ${{steps.prepare.outputs.platforms}} \ - --output "type=image,push=true" \ - --tag jonaswinkler/paperless-ng:edge ./ - - - name: Docker Check Manifest - run: | - docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64 + push: true + tags: jonaswinkler/paperless-ng:edge + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache - name: Inspect image run: | - docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} + docker buildx imagetools inspect jonaswinkler/paperless-ng:edge From eca44cd227f5667852d795fd07da1c0fae57c1b2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 20:06:53 +0100 Subject: [PATCH 094/148] add arm --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cda034085..1fe772e7b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,6 +24,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 - name: Cache Docker layers uses: actions/cache@v2 @@ -44,7 +47,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64 + platforms: linux/amd64,linux/arm/v7 push: true tags: jonaswinkler/paperless-ng:edge cache-from: type=local,src=/tmp/.buildx-cache From eb4284238668c92823eecf80f4b3fd650aa2093d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 20:29:29 +0100 Subject: [PATCH 095/148] undo removing a required library --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5e6b2690e..0ccfcd84e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,7 +63,7 @@ RUN apt-get update \ && pipenv install --system --deploy --ignore-pipfile \ && pipenv --clear \ && pip3 uninstall -y pipenv \ - && apt-get -y purge build-essential libqpdf-dev libxslt1-dev \ + && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord From ba667eb1dee4a38ed1db89c76ce941c0712625d7 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 20:51:46 +0100 Subject: [PATCH 096/148] always update cache --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1fe772e7b..be271a3b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,7 +29,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Cache Docker layers - uses: actions/cache@v2 + uses: pat-s/always-upload-cache@v2 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} From 5fe44a2c572caf75be0a05ebbcb648accacb53a2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:21:50 +0100 Subject: [PATCH 097/148] use correct version --- .github/workflows/docker.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index be271a3b7..4afc1027f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,6 +18,18 @@ jobs: buildx: runs-on: ubuntu-latest steps: + - + name: Prepare + id: prepare + run: | + VERSION=edge + if [[ $GITHUB_REF == refs/tags/ng-* ]]; then + VERSION=${GITHUB_REF#refs/tags/ng-} + fi + if [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=${GITHUB_REF#refs/heads/} + fi + echo ::set-output name=version::${VERSION} - name: Checkout uses: actions/checkout@v2 @@ -49,10 +61,10 @@ jobs: file: ./Dockerfile platforms: linux/amd64,linux/arm/v7 push: true - tags: jonaswinkler/paperless-ng:edge + tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - name: Inspect image run: | - docker buildx imagetools inspect jonaswinkler/paperless-ng:edge + docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} From d1b0a96d1f7349070a89d1c2c6a0e0cd2e694f56 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:23:04 +0100 Subject: [PATCH 098/148] use correct version of cache action --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4afc1027f..e7b426a07 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,7 +41,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2 + uses: pat-s/always-upload-cache@v2.1.3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} From b0ab5be87bd661301090970de8ac23218a9bbad1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:49:43 +0100 Subject: [PATCH 099/148] move everything into a single file so that the docker image build can depend on the tests being successful --- .github/workflows/ci.yml | 140 ++++++++++++++++++++++++++++ .github/workflows/docker.yml | 70 -------------- .github/workflows/documentation.yml | 33 ------- .github/workflows/frontend.yml | 19 ---- .github/workflows/test.yml | 43 --------- 5 files changed, 140 insertions(+), 165 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/documentation.yml delete mode 100644 .github/workflows/frontend.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..3ccfedd43 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,140 @@ +name: docker + +on: [push, pull_request] + +jobs: + documentation: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "3.6.8" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip3.6.8} + - name: dependencies + # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? + run: | + pip install 'sphinx~=3.3' sphinx_rtd_theme + - name: make + run: | + cd docs/ + make html + + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-20.04 ] + python: # pyenv-action requires the exact version + - 3.6.8 + - 3.7.9 + - 3.8.6 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "${{ matrix.python }}" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python }} + - name: Tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + cd src/ + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls + + frontend: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: build + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + ng build --prod + + # build and push image to docker hub. + build-docker-image: + if: github.event_name == push && (github.ref == refs/heads/master || github.ref == refs/heads/dev || github.ref == refs/tags/ng-*) + runs-on: ubuntu-latest + needs: tests + steps: + - + name: Prepare + id: prepare + run: | + VERSION=edge + if [[ $GITHUB_REF == refs/tags/ng-* ]]; then + VERSION=${GITHUB_REF#refs/tags/ng-} + fi + if [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=${GITHUB_REF#refs/heads/} + fi + echo ::set-output name=version::${VERSION} + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Cache Docker layers + uses: pat-s/always-upload-cache@v2.1.3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm/v7 + push: true + tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + - + name: Inspect image + run: | + docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index e7b426a07..000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: docker - -on: - push: - branches: - - master - - dev - - travis-multiarch-builds - tags: - - 'ng-*' - - -# --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ -# --build-arg VCS_REF=${GITHUB_SHA::8} \ - -jobs: - # build and push image to docker hub. - buildx: - runs-on: ubuntu-latest - steps: - - - name: Prepare - id: prepare - run: | - VERSION=edge - if [[ $GITHUB_REF == refs/tags/ng-* ]]; then - VERSION=${GITHUB_REF#refs/tags/ng-} - fi - if [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=${GITHUB_REF#refs/heads/} - fi - echo ::set-output name=version::${VERSION} - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2.1.3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7 - push: true - tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Inspect image - run: | - docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml deleted file mode 100644 index 00835d240..000000000 --- a/.github/workflows/documentation.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: documentation - -# always run documentation workflow to ensure the documentation can still be built -on: [push, pull_request] - -# possibly add a publish job for automatic read-the-docs publishing -jobs: - documentation: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "3.6.8" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip3.6.8} - - name: dependencies - # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? - run: | - pip install 'sphinx~=3.3' sphinx_rtd_theme - - name: make - run: | - cd docs/ - make html \ No newline at end of file diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml deleted file mode 100644 index 4cb36d3a4..000000000 --- a/.github/workflows/frontend.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: frontend - -# always run frontend workflow to see if it builds and tests run fine -on: [push, pull_request] - -jobs: - frontend: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '15' - - name: build - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - ng build --prod \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 9766c4b5b..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: backend-tests - -# always run tests workflow -on: [push, pull_request] - -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - python: # pyenv-action requires the exact version - - 3.6.8 - - 3.7.9 - - 3.8.6 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "${{ matrix.python }}" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python }} - - name: Tests - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - cd src/ - pipenv run pytest --cov - pipenv run pycodestyle - pipenv run coveralls From 66786d17c257e2f468f7ff15df3c72aa327e2515 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:50:53 +0100 Subject: [PATCH 100/148] fix condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ccfedd43..492c977bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == push && (github.ref == refs/heads/master || github.ref == refs/heads/dev || github.ref == refs/tags/ng-*) + if: github.event_name == "push" && (github.ref == "refs/heads/master" || github.ref == "refs/heads/dev" || github.ref == "refs/tags/ng-*") runs-on: ubuntu-latest needs: tests steps: From 3b37fd0f21d6a9e1e73e998aa23f22ed37b19e23 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:53:13 +0100 Subject: [PATCH 101/148] fix condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 492c977bf..b841dee8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == "push" && (github.ref == "refs/heads/master" || github.ref == "refs/heads/dev" || github.ref == "refs/tags/ng-*") + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest needs: tests steps: From fe1e685ffc2515bb4722762f8797c4f096932e47 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:55:03 +0100 Subject: [PATCH 102/148] rename workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b841dee8d..dcef91911 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: docker +name: ci on: [push, pull_request] From 61232bb3400d2406c4d85a4c012864df551f8b14 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:03:38 +0100 Subject: [PATCH 103/148] test include this branch --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcef91911..3fe9a6ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') + if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest needs: tests steps: From 2a7bcbc94bd3d7b610a04d05fa86c07aa63058f1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:33:07 +0100 Subject: [PATCH 104/148] try separate build / push --- .github/workflows/ci.yml | 45 +++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe9a6ad3..78dd931d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,9 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest - needs: tests steps: - name: Prepare @@ -118,19 +116,10 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push + name: Build image uses: docker/build-push-action@v2 with: - context: . - file: ./Dockerfile platforms: linux/amd64,linux/arm/v7 - push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache @@ -138,3 +127,35 @@ jobs: name: Inspect image run: | docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} + + push-docker-image: + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') + needs: [tests, build-docker-image] + + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Cache Docker layers + uses: pat-s/always-upload-cache@v2.1.3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Push image + uses: docker/build-push-action@v2 + with: + platforms: linux/amd64,linux/arm/v7 + tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} + cache-from: type=local,src=/tmp/.buildx-cache + push: true From 6c604e3034af51d133df90c42312650fcef87861 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:38:05 +0100 Subject: [PATCH 105/148] Revert "try separate build / push" This reverts commit 6fa0d00b --- .github/workflows/ci.yml | 45 +++++++++++----------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78dd931d0..3fe9a6ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,9 @@ jobs: # build and push image to docker hub. build-docker-image: + if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest + needs: tests steps: - name: Prepare @@ -116,10 +118,19 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - - name: Build image + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build and push uses: docker/build-push-action@v2 with: + context: . + file: ./Dockerfile platforms: linux/amd64,linux/arm/v7 + push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache @@ -127,35 +138,3 @@ jobs: name: Inspect image run: | docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} - - push-docker-image: - runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') - needs: [tests, build-docker-image] - - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2.1.3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Push image - uses: docker/build-push-action@v2 - with: - platforms: linux/amd64,linux/arm/v7 - tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} - cache-from: type=local,src=/tmp/.buildx-cache - push: true From 082f021cf2f262ddfe776ec6071014e1f99a05e1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:53:45 +0100 Subject: [PATCH 106/148] add arm64 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe9a6ad3..78e1eec42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7 + platforms: linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache From fbeab59cdabff4f1aa75c79e6a042a38ed97c3a1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:09:10 +0100 Subject: [PATCH 107/148] test updated Pipfile --- Pipfile | 3 --- Pipfile.lock | 15 +++++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index 0c182d5b6..1a9b43950 100644 --- a/Pipfile +++ b/Pipfile @@ -8,9 +8,6 @@ url = "https://www.piwheels.org/simple" verify_ssl = true name = "piwheels" -[requires] -python_version = "3.6" - [packages] dateparser = "~=0.7.6" django = "~=3.1.3" diff --git a/Pipfile.lock b/Pipfile.lock index b36e5e496..58cf35894 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,12 +1,10 @@ { "_meta": { "hash": { - "sha256": "c35d84fd7f4f1c7d599039712362935e7c41a226b0ab3d83d8c1c2fb2ad0962a" + "sha256": "241c8d042e9b607bbeb7d501b3fc1b57376e3cc8c2b2fe117ec60e425dfd79c7" }, "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, + "requires": {}, "sources": [ { "name": "pypi", @@ -1375,6 +1373,15 @@ "index": "pypi", "version": "==3.20.1" }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "markers": "python_version < '3.8'", + "version": "==3.7.4.3" + }, "urllib3": { "hashes": [ "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", From 900b0cc3438211a22683ae4ba1e87b3587c9f4c4 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:28:33 +0100 Subject: [PATCH 108/148] try new python setup action --- .github/workflows/ci.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78e1eec42..7af141cd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,17 +35,13 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python: # pyenv-action requires the exact version - - 3.6.8 - - 3.7.9 - - 3.8.6 + python-version: ['3.6', '3.7', '3.8'] steps: - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 + - name: Set up Python + uses: actions/setup-python@v2 with: - default: "${{ matrix.python }}" - command: "pip install -U pip" + default: "${{ matrix.python-version }}" - name: Get pip cache dir id: pip-cache run: | From b1f0311b998bf8fdb3dfad143804e425b09b0bc5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:32:47 +0100 Subject: [PATCH 109/148] fix python action --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7af141cd2..a9bbda8ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,17 +31,16 @@ jobs: make html tests: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-20.04 strategy: matrix: - os: [ ubuntu-20.04 ] python-version: ['3.6', '3.7', '3.8'] steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: - default: "${{ matrix.python-version }}" + python-version: "${{ matrix.python-version }}" - name: Get pip cache dir id: pip-cache run: | From 3355229fb53bfa41404e3490c95fdda040a08d85 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:41:11 +0100 Subject: [PATCH 110/148] no need to run commands in pipenv when using --system --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9bbda8ad..9d398e6d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,11 +57,11 @@ jobs: sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv - pipenv install --system --dev + pipenv install --system --dev --ignore-pipfile cd src/ - pipenv run pytest --cov - pipenv run pycodestyle - pipenv run coveralls + pytest + pycodestyle + coveralls frontend: runs-on: ubuntu-20.04 From 8df24fed22b34f71e000a8218378508adb80fa8d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:50:06 +0100 Subject: [PATCH 111/148] documentation action --- Dockerfile | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0ccfcd84e..ea3136bb3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS jbig2 +FROM ubuntu:20.04 AS jbig2enc WORKDIR /usr/src/jbig2enc @@ -19,7 +19,7 @@ RUN npm install COPY src-ui . RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false -FROM python:3.7-slim +FROM ubuntu:20.04 WORKDIR /usr/src/paperless/ @@ -27,7 +27,7 @@ COPY Pipfile* ./ #Dependencies RUN apt-get update \ - && apt-get -y --no-install-recommends install \ + && DEBIAN_FRONTEND="noninteractive" apt-get -y --no-install-recommends install \ build-essential \ curl \ file \ @@ -48,6 +48,9 @@ RUN apt-get update \ mime-support \ optipng \ pngquant \ + python3 \ + python3-pip \ + python3-dev \ qpdf \ sudo \ tesseract-ocr \ @@ -59,11 +62,12 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor pipenv \ + + && pip3 install --upgrade supervisor pipenv setuptools \ && pipenv install --system --deploy --ignore-pipfile \ && pipenv --clear \ && pip3 uninstall -y pipenv \ - && apt-get -y purge build-essential libqpdf-dev \ + && apt-get -y purge build-essential libqpdf-dev python3-dev python3-pip \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord @@ -75,10 +79,10 @@ COPY docker/gunicorn.conf.py ./ COPY docker/supervisord.conf /etc/supervisord.conf COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh -# copy jbic -COPY --from=jbig2 /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ -COPY --from=jbig2 /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ -COPY --from=jbig2 /usr/src/jbig2enc/src/*.h /usr/local/include/ +# copy jbig2enc +COPY --from=jbig2enc /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ +COPY --from=jbig2enc /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ +COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/ # copy app From 25e501e5e0ec33806cb6ff4a1321c5e971e4b67e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:53:30 +0100 Subject: [PATCH 112/148] documentation action --- .github/workflows/ci.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d398e6d7..98c61efb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,11 +7,10 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 + - name: Set up Python + uses: actions/setup-python@v2 with: - default: "3.6.8" - command: "pip install -U pip" + python-version: 3.8 - name: Get pip cache dir id: pip-cache run: | @@ -21,11 +20,9 @@ jobs: with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip3.6.8} - - name: dependencies - # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? - run: | - pip install 'sphinx~=3.3' sphinx_rtd_theme - - name: make + - name: Install dependencies + run: pipenv install --system --dev --ignore-pipfile + - name: Make documentation run: | cd docs/ make html From 1330126cdfba1278f6e7839ad9b154e5fe6e4ede Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:54:48 +0100 Subject: [PATCH 113/148] install pipenv --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98c61efb4..54ff75670 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,9 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip3.6.8} - name: Install dependencies - run: pipenv install --system --dev --ignore-pipfile + run: | + pip install --upgrade pipenv + pipenv install --system --dev --ignore-pipfile - name: Make documentation run: | cd docs/ From 2ac131506ce3a8c1a647f951ae40a40cf1e7279a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:58:14 +0100 Subject: [PATCH 114/148] dependencies --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ff75670..98e5ecf8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: key: ${{ runner.os }}-pip3.6.8} - name: Install dependencies run: | + sudo apt-get update -qq + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev pip install --upgrade pipenv pipenv install --system --dev --ignore-pipfile - name: Make documentation @@ -54,7 +56,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv pipenv install --system --dev --ignore-pipfile cd src/ From 51a8593b5a4b1dc1739177e5725a1d085054abcd Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 11:26:14 +0100 Subject: [PATCH 115/148] fix up pipfile for arm/v7 --- .github/workflows/ci.yml | 2 +- Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98e5ecf8a..aa4ee9b74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,7 +107,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2.1.3 + uses: actions/cache@v2 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/Dockerfile b/Dockerfile index ea3136bb3..4bd9b5e06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ RUN apt-get update \ icc-profiles-free \ imagemagick \ libatlas-base-dev \ + libffi-dev \ liblept5 \ libmagic-dev \ libpoppler-cpp-dev \ From b26d22dbbeeab8eb43ba29c6c601e710bc011681 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 11:39:30 +0100 Subject: [PATCH 116/148] fix cryphtography for armv7 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 4bd9b5e06..16c52caa9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,7 @@ RUN apt-get update \ libpoppler-cpp-dev \ libpq-dev \ libqpdf-dev \ + libssl-dev \ libxml2 \ libxslt1-dev \ mime-support \ From 52cba3c0103ce7525acba29a052c4d282e7ac04b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 15:51:40 +0100 Subject: [PATCH 117/148] maybe use requirements instead? --- .github/workflows/ci.yml | 2 +- Dockerfile | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa4ee9b74..523f74e61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/arm/v7 # linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache diff --git a/Dockerfile b/Dockerfile index 16c52caa9..d124ddc09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,8 +65,9 @@ RUN apt-get update \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor pipenv setuptools \ - && pipenv install --system --deploy --ignore-pipfile \ + && pip3 install --upgrade supervisor pipenv setuptools \ + && pipenv lock -r > requirements.txt \ + && pip3 install -r requirements.txt \ && pipenv --clear \ && pip3 uninstall -y pipenv \ && apt-get -y purge build-essential libqpdf-dev python3-dev python3-pip \ From 4192dc915826bdb8dee17b062b1cc49f68e283c0 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 17:18:35 +0100 Subject: [PATCH 118/148] revert to python:3.7-slim base image since ubuntu:20.04 is clearly not working for arm --- Dockerfile | 21 ++++---------- requirements.txt | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index d124ddc09..ee19e0cc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,15 +19,15 @@ RUN npm install COPY src-ui . RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false -FROM ubuntu:20.04 +FROM python:3.7-slim WORKDIR /usr/src/paperless/ -COPY Pipfile* ./ +COPY requirements.txt ./ #Dependencies RUN apt-get update \ - && DEBIAN_FRONTEND="noninteractive" apt-get -y --no-install-recommends install \ + && apt-get -y --no-install-recommends install \ build-essential \ curl \ file \ @@ -38,21 +38,16 @@ RUN apt-get update \ icc-profiles-free \ imagemagick \ libatlas-base-dev \ - libffi-dev \ liblept5 \ libmagic-dev \ libpoppler-cpp-dev \ libpq-dev \ libqpdf-dev \ - libssl-dev \ libxml2 \ libxslt1-dev \ mime-support \ optipng \ pngquant \ - python3 \ - python3-pip \ - python3-dev \ qpdf \ sudo \ tesseract-ocr \ @@ -64,13 +59,9 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - - && pip3 install --upgrade supervisor pipenv setuptools \ - && pipenv lock -r > requirements.txt \ - && pip3 install -r requirements.txt \ - && pipenv --clear \ - && pip3 uninstall -y pipenv \ - && apt-get -y purge build-essential libqpdf-dev python3-dev python3-pip \ + && pip3 install --upgrade supervisor \ + && python3 -m pip install -r requirements.txt \ + && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..9932d146b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,74 @@ +# +# These requirements were autogenerated by pipenv +# To regenerate from the project's Pipfile, run: +# +# pipenv lock --requirements +# + +-i https://pypi.python.org/simple +--extra-index-url https://www.piwheels.org/simple +arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +asgiref==3.3.1; python_version >= '3.5' +blessed==1.17.12 +certifi==2020.12.5 +cffi==1.14.4 +chardet==4.0.0; python_version >= '3.1' +coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +dateparser==0.7.6 +django-cors-headers==3.6.0 +django-extensions==3.1.0 +django-filter==2.4.0 +django-picklefield==3.0.1; python_version >= '3' +django-q==1.3.4 +django==3.1.5 +djangorestframework==3.12.2 +filelock==3.0.12 +fuzzywuzzy==0.18.0 +gunicorn==20.0.4 +humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +imap-tools==0.34.0 +img2pdf==0.4.0 +importlib-metadata==3.3.0; python_version < '3.8' +inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +inotifyrecursive==0.3.5 +joblib==1.0.0; python_version >= '3.6' +langdetect==1.0.8 +lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +numpy==1.19.5; python_version >= '3.6' +ocrmypdf==11.4.5 +pathvalidate==2.3.2 +pdfminer.six==20201018; python_version >= '3.4' +pdftotext==2.1.5 +pikepdf==2.2.5 +pillow==8.1.0 +pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +psycopg2-binary==2.8.6 +pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +python-dateutil==2.8.1 +python-dotenv==0.15.0 +python-gnupg==0.4.6 +python-levenshtein==0.12.0 +python-magic==0.4.18 +pytz==2020.5 +redis==3.5.3 +regex==2020.11.13 +reportlab==3.5.59 +requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +scikit-learn==0.23.2 +scipy==1.5.4; python_version >= '3.6' +six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +sortedcontainers==2.3.0 +sqlparse==0.4.1; python_version >= '3.5' +threadpoolctl==2.1.0; python_version >= '3.5' +tika==1.24 +tqdm==4.55.1 +typing-extensions==3.7.4.3; python_version < '3.8' +tzlocal==2.1 +urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' +watchdog==1.0.2 +wcwidth==0.2.5 +whitenoise==5.2.0 +whoosh==2.7.4 +zipp==3.4.0; python_version >= '3.6' From eb4752e1b68a4ac651f3d5c4952b39566b9d6f7e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 17:19:00 +0100 Subject: [PATCH 119/148] disable tests until this is working --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 523f74e61..cf2c8b137 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest - needs: tests + # needs: tests steps: - name: Prepare From 85df57a848b50c777f5a6da9c164b83688fdfaef Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 17:46:43 +0100 Subject: [PATCH 120/148] enable the other architectures --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf2c8b137..e2918d276 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/arm/v7 # linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache From 8c64dbd216db84acf4c7749f37732ae760b55802 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 19:59:05 +0100 Subject: [PATCH 121/148] don't build front end with docker --- .github/workflows/ci.yml | 18 ++++++++++++------ Dockerfile | 39 ++++++++++++++++----------------------- compile-frontend.sh | 7 +++++++ src-ui/angular.json | 1 + 4 files changed, 36 insertions(+), 29 deletions(-) create mode 100755 compile-frontend.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2918d276..f0df7e608 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,17 +72,18 @@ jobs: with: node-version: '15' - name: build - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - ng build --prod + run: ./build-frontend.sh + - uses: actions/upload-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ # build and push image to docker hub. build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest # needs: tests + needs: frontend steps: - name: Prepare @@ -99,6 +100,11 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - + uses: actions/download-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -125,7 +131,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/amd64 # ,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache diff --git a/Dockerfile b/Dockerfile index ee19e0cc0..4a52568ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,27 +8,15 @@ RUN git clone https://github.com/agl/jbig2enc . RUN ./autogen.sh RUN ./configure && make - -FROM node:15 AS frontend - -WORKDIR /usr/src/paperless/src-ui/ - -COPY src-ui/package* ./ -RUN npm install - -COPY src-ui . -RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false - FROM python:3.7-slim WORKDIR /usr/src/paperless/ COPY requirements.txt ./ -#Dependencies +# Binary dependencies RUN apt-get update \ && apt-get -y --no-install-recommends install \ - build-essential \ curl \ file \ fonts-liberation \ @@ -37,13 +25,6 @@ RUN apt-get update \ gnupg \ icc-profiles-free \ imagemagick \ - libatlas-base-dev \ - liblept5 \ - libmagic-dev \ - libpoppler-cpp-dev \ - libpq-dev \ - libqpdf-dev \ - libxml2 \ libxslt1-dev \ mime-support \ optipng \ @@ -59,8 +40,21 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor \ - && python3 -m pip install -r requirements.txt \ + && rm -rf /var/lib/apt/lists/* + +# Python dependencies +RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + libatlas-base-dev \ + liblept5 \ + libmagic-dev \ + libpoppler-cpp-dev \ + libpq-dev \ + libqpdf-dev \ + libxml2 \ + && python3 -m pip install --upgrade --no-cache-dir supervisor \ + && python3 -m pip install --no-cache-dir -r requirements.txt \ && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ @@ -81,7 +75,6 @@ COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/ # copy app COPY src/ ./src/ -COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/compile-frontend.sh b/compile-frontend.sh new file mode 100755 index 000000000..98b88d033 --- /dev/null +++ b/compile-frontend.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +cd src-ui +npm install +./node_modules/.bin/ng build --prod diff --git a/src-ui/angular.json b/src-ui/angular.json index 414cd4f64..829a482af 100644 --- a/src-ui/angular.json +++ b/src-ui/angular.json @@ -58,6 +58,7 @@ "with": "src/environments/environment.prod.ts" } ], + "outputPath": "../src/documents/static/frontend/", "optimization": true, "outputHashing": "none", "sourceMap": false, From 2ae06525154389b9946d833db078bb262ed5312e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 20:00:11 +0100 Subject: [PATCH 122/148] fix script name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0df7e608..ce5117256 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: with: node-version: '15' - name: build - run: ./build-frontend.sh + run: ./compile-frontend.sh - uses: actions/upload-artifact@v2 with: name: frontend-compiled From 6556f6e5f8a4c746a8a8d78692d794468db5d2b3 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:03:50 +0100 Subject: [PATCH 123/148] add release archive action --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce5117256..eea3b0ba0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,10 @@ jobs: run: | cd docs/ make html + - uses: actions/upload-artifact@v2 + with: + name: documentation + path: docs/_build/html/ tests: runs-on: ubuntu-20.04 @@ -78,6 +82,39 @@ jobs: name: frontend-compiled path: src/documents/static/frontend/ + release: + needs: [frontend, documentation] # tests + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - + uses: actions/download-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ + - + uses: actions/download-artifact@v2 + with: + name: documentation + path: src/documents/static/frontend/ + - + run: | + mkdir dist + mkdir dist/paperless-ng + mkdir dist/paperless-ng/docker + mkdir dist/paperless-ng/scripts + cp .dockerignore .env docker-compose.* Dockerfile paperless.conf.example Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/ + cp docker/* dist/paperless-ng/docker/ + cp scripts/* dist/paperless-ng/scripts/ + cp src/ dist/paperless-ng/src -R + cd dist + tar -cJf paperless-ng.tar.xz paperless-ng/ + - uses: actions/upload-artifact@v2 + with: + name: release + path: dist/paperless-ng/paperless-ng.tar.xz + + # build and push image to docker hub. build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') From 812341501b33c6fb4cbefc98efc0b01106fd3be9 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:17:01 +0100 Subject: [PATCH 124/148] more release archive --- .github/workflows/ci.yml | 42 ++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eea3b0ba0..2d0f98c3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,19 @@ jobs: needs: [frontend, documentation] # tests runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - + uses: actions/checkout@v2 + - + name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - + name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext + pip3 install -r requirements.txt - uses: actions/download-artifact@v2 with: @@ -96,20 +108,34 @@ jobs: uses: actions/download-artifact@v2 with: name: documentation - path: src/documents/static/frontend/ + path: docs/_build/html/ - + name: Move files run: | mkdir dist mkdir dist/paperless-ng mkdir dist/paperless-ng/docker mkdir dist/paperless-ng/scripts cp .dockerignore .env docker-compose.* Dockerfile paperless.conf.example Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/ - cp docker/* dist/paperless-ng/docker/ - cp scripts/* dist/paperless-ng/scripts/ - cp src/ dist/paperless-ng/src -R - cd dist - tar -cJf paperless-ng.tar.xz paperless-ng/ - - uses: actions/upload-artifact@v2 + cp docker/*.* dist/paperless-ng/docker/ + cp scripts/*.service scripts/*.sh dist/paperless-ng/scripts/ + cp src/ dist/paperless-ng/src -r + cp docs/_build/html/ dist/paperless-ng/docs -r + - + name: Compile messages + run: | + cd dist/paperless-ng/src + python3 manage.py compilemessages + - + name: Collect static files + run: | + cd dist/paperless-ng/src + python3 manage.py collectstatic --no-input + - + name: Make release package + run: tar -cJf paperless-ng.tar.xz paperless-ng/ + - + uses: actions/upload-artifact@v2 with: name: release path: dist/paperless-ng/paperless-ng.tar.xz From 371a55e7129bfaacdc6191e3110dd48b67108bd8 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:20:55 +0100 Subject: [PATCH 125/148] fix wrong directory --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d0f98c3b..5acae09f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,9 @@ jobs: python3 manage.py collectstatic --no-input - name: Make release package - run: tar -cJf paperless-ng.tar.xz paperless-ng/ + run: | + cd dist + tar -cJf paperless-ng.tar.xz paperless-ng/ - uses: actions/upload-artifact@v2 with: From 1cf02fb50fed57f92e29cdc184b494400969e994 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:27:41 +0100 Subject: [PATCH 126/148] forgot a dependency --- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5acae09f2..e965d50c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: name: Install dependencies run: | sudo apt-get update -qq - sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5 pip3 install -r requirements.txt - uses: actions/download-artifact@v2 diff --git a/Dockerfile b/Dockerfile index 4a52568ea..7e921fcda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ RUN apt-get update \ gnupg \ icc-profiles-free \ imagemagick \ + liblept5 \ libxslt1-dev \ mime-support \ optipng \ @@ -47,7 +48,6 @@ RUN apt-get update \ && apt-get -y --no-install-recommends install \ build-essential \ libatlas-base-dev \ - liblept5 \ libmagic-dev \ libpoppler-cpp-dev \ libpq-dev \ From b7f0fead7ac1af1d4aaf373bbf2bee167125cc46 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:35:43 +0100 Subject: [PATCH 127/148] wrong path to release --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e965d50c6..84949dec0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: uses: actions/upload-artifact@v2 with: name: release - path: dist/paperless-ng/paperless-ng.tar.xz + path: dist/paperless-ng.tar.xz # build and push image to docker hub. From 32cdfda69deed210b9bc330ae98cf46d0c578657 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:53:18 +0100 Subject: [PATCH 128/148] some small fixes --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84949dec0..4379f8abb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: path: src/documents/static/frontend/ release: - needs: [frontend, documentation] # tests + needs: [frontend, documentation, tests] runs-on: ubuntu-20.04 steps: - @@ -135,6 +135,7 @@ jobs: name: Make release package run: | cd dist + find . -name __pycache__ | xargs rm -r tar -cJf paperless-ng.tar.xz paperless-ng/ - uses: actions/upload-artifact@v2 @@ -147,8 +148,7 @@ jobs: build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest - # needs: tests - needs: frontend + needs: [frontend, tests] steps: - name: Prepare From 27f0c5bf77e5e8dc77e6a9e4f514f5e3bf6f3f48 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:22:23 +0100 Subject: [PATCH 129/148] add a publish release step --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4379f8abb..c26c0672f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: name: frontend-compiled path: src/documents/static/frontend/ - release: + build-release: needs: [frontend, documentation, tests] runs-on: ubuntu-20.04 steps: @@ -143,6 +143,38 @@ jobs: name: release path: dist/paperless-ng.tar.xz + publish-release: + runs-on: ubuntu-latest + needs: build-release + steps: + - + name: Download Release Artifact + uses: actions/download-artifact@v2 + with: + name: release + path: paperless-ng.tar.xz + - + name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Paperless-ng ${{ github.ref }} + draft: true + prerelease: true + - + name: Upload Release Archive + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./paperless-ng.tar.xz + asset_name: paperless-ng-${{ github.ref }}.tar.xz + asset_content_type: application/x-xz # build and push image to docker hub. build-docker-image: From 58ded0a340aa2b2b2075007ea5e8da6754dd22e4 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:28:46 +0100 Subject: [PATCH 130/148] reorganized the Dockerfile --- Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7e921fcda..789a15641 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,28 +17,31 @@ COPY requirements.txt ./ # Binary dependencies RUN apt-get update \ && apt-get -y --no-install-recommends install \ + # Basic dependencies curl \ file \ fonts-liberation \ gettext \ - ghostscript \ gnupg \ - icc-profiles-free \ imagemagick \ - liblept5 \ libxslt1-dev \ mime-support \ optipng \ + sudo \ + tzdata \ + # OCRmyPDF dependencies + ghostscript \ + icc-profiles-free \ + liblept5 \ + libxml2 \ pngquant \ qpdf \ - sudo \ tesseract-ocr \ tesseract-ocr-eng \ tesseract-ocr-deu \ tesseract-ocr-fra \ tesseract-ocr-ita \ tesseract-ocr-spa \ - tzdata \ unpaper \ zlib1g \ && rm -rf /var/lib/apt/lists/* @@ -52,7 +55,6 @@ RUN apt-get update \ libpoppler-cpp-dev \ libpq-dev \ libqpdf-dev \ - libxml2 \ && python3 -m pip install --upgrade --no-cache-dir supervisor \ && python3 -m pip install --no-cache-dir -r requirements.txt \ && apt-get -y purge build-essential libqpdf-dev \ From c4c198487c4b2768b20f77cf9386df6f98a6d74a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:56:10 +0100 Subject: [PATCH 131/148] update release name --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c26c0672f..b5eb9f318 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ github.ref }} - release_name: Paperless-ng ${{ github.ref }} + tag_name: ${{ github.ref#refs/heads/ }} + release_name: Paperless-ng ${{ github.ref#refs/heads/ }} draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${{ github.ref }}.tar.xz + asset_name: paperless-ng-${{ github.ref#refs/heads/ }}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From 169aba9aca701c67caa8966a9135471e99289658 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:57:56 +0100 Subject: [PATCH 132/148] maybe this is better? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5eb9f318..8bcd5de92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ github.ref#refs/heads/ }} - release_name: Paperless-ng ${{ github.ref#refs/heads/ }} + tag_name: ${GITHUB_REF#refs/heads/} + release_name: Paperless-ng ${GITHUB_REF#refs/heads/} draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${{ github.ref#refs/heads/ }}.tar.xz + asset_name: paperless-ng-${GITHUB_REF#refs/heads/}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From 0294274ca53145a9fa32eef551b9eb283d6f8ffe Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:09:09 +0100 Subject: [PATCH 133/148] maybe this is better? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bcd5de92..41bf07776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${GITHUB_REF#refs/heads/} - release_name: Paperless-ng ${GITHUB_REF#refs/heads/} + tag_name: ${{ GITHUB_REF#refs/heads/ }} + release_name: Paperless-ng ${{ GITHUB_REF#refs/heads/ }} draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${GITHUB_REF#refs/heads/}.tar.xz + asset_name: paperless-ng-${{ GITHUB_REF#refs/heads/ }}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From 1d23a33dc247f219c60e003a936c29031baba66c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:17:53 +0100 Subject: [PATCH 134/148] I have no idea what I am doing --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41bf07776..8f33d5fb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ GITHUB_REF#refs/heads/ }} - release_name: Paperless-ng ${{ GITHUB_REF#refs/heads/ }} + tag_name: $GITHUB_REF#refs/heads/ + release_name: Paperless-ng $GITHUB_REF#refs/heads/ draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${{ GITHUB_REF#refs/heads/ }}.tar.xz + asset_name: paperless-ng-$GITHUB_REF#refs/heads/.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From 4380c3ca2c1df193b00b8a592a46f9cdfd16f982 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:43:59 +0100 Subject: [PATCH 135/148] maybe this --- .github/workflows/ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f33d5fb4..a4523b6f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,11 @@ jobs: with: name: release path: paperless-ng.tar.xz + - + id: get_branch_name + run: | + branch_name=$(echo ${{ GITHUB_REF }} | cut -d/ -f3) + echo "::set-output name=branch_name::$branch_name" - name: Create Release id: create_release @@ -160,8 +165,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: $GITHUB_REF#refs/heads/ - release_name: Paperless-ng $GITHUB_REF#refs/heads/ + tag_name: ${{ steps.get_branch_name.outputs.branch_name }} + release_name: Paperless-ng ${{ steps.get_branch_name.outputs.branch_name }} draft: true prerelease: true - @@ -173,7 +178,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-$GITHUB_REF#refs/heads/.tar.xz + asset_name: paperless-ng-${{ steps.get_branch_name.outputs.branch_name }}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From 5cdf0d563246afcd1f9922acbd36499d99fde3b0 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:46:19 +0100 Subject: [PATCH 136/148] github.ref instead of GITHUB_REF --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4523b6f9..d66078023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,7 +156,7 @@ jobs: - id: get_branch_name run: | - branch_name=$(echo ${{ GITHUB_REF }} | cut -d/ -f3) + branch_name=$(echo ${{ github.ref }} | cut -d/ -f3) echo "::set-output name=branch_name::$branch_name" - name: Create Release From 525ab36f4d4ef502e8f7e0ff4f6ee78bb152dcda Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:57:20 +0100 Subject: [PATCH 137/148] fix upload-release-asset path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d66078023..7a1dcbe7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,7 @@ jobs: uses: actions/download-artifact@v2 with: name: release - path: paperless-ng.tar.xz + path: ./ - id: get_branch_name run: | From 463e0147e6551af647e0416505584bbdb0d9bfb6 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 00:08:56 +0100 Subject: [PATCH 138/148] enable multiarch builds, release only with ng- tags --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a1dcbe7e..2ac2fc38f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,7 @@ jobs: publish-release: runs-on: ubuntu-latest needs: build-release + if: contains(github.ref, 'refs/tags/ng-') steps: - name: Download Release Artifact @@ -156,7 +157,7 @@ jobs: - id: get_branch_name run: | - branch_name=$(echo ${{ github.ref }} | cut -d/ -f3) + branch_name=$(echo ${{ github.ref }} | cut -d- -f2) echo "::set-output name=branch_name::$branch_name" - name: Create Release @@ -167,8 +168,10 @@ jobs: with: tag_name: ${{ steps.get_branch_name.outputs.branch_name }} release_name: Paperless-ng ${{ steps.get_branch_name.outputs.branch_name }} - draft: true + draft: false prerelease: true + body: | + TODO: Insert changelog here (it's also in the documentation) - name: Upload Release Archive id: upload-release-asset @@ -233,7 +236,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64 # ,linux/arm/v7,linux/arm64 + platforms: linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache From db1e6ea6ddf37b14486398bc140a87fc37f8ad30 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 13:19:29 +0100 Subject: [PATCH 139/148] a couple fixes to the ci script --- .github/workflows/ci.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ac2fc38f..004a66fca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip3.6.8} + key: ${{ runner.os }}-pip3.8} - name: Install dependencies run: | sudo apt-get update -qq @@ -75,9 +75,10 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '15' - - name: build + - name: Build frontend run: ./compile-frontend.sh - - uses: actions/upload-artifact@v2 + - name: Upload artifact + uses: actions/upload-artifact@v2 with: name: frontend-compiled path: src/documents/static/frontend/ @@ -100,11 +101,13 @@ jobs: sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5 pip3 install -r requirements.txt - + name: Download frontend artifact uses: actions/download-artifact@v2 with: name: frontend-compiled path: src/documents/static/frontend/ - + name: Download documentation artifact uses: actions/download-artifact@v2 with: name: documentation @@ -138,6 +141,7 @@ jobs: find . -name __pycache__ | xargs rm -r tar -cJf paperless-ng.tar.xz paperless-ng/ - + name: Upload release artifact uses: actions/upload-artifact@v2 with: name: release @@ -149,18 +153,18 @@ jobs: if: contains(github.ref, 'refs/tags/ng-') steps: - - name: Download Release Artifact + name: Download release artifact uses: actions/download-artifact@v2 with: name: release path: ./ - + name: Get branch name id: get_branch_name run: | - branch_name=$(echo ${{ github.ref }} | cut -d- -f2) - echo "::set-output name=branch_name::$branch_name" + echo ::set-output name=branch_name::${GITHUB_REF#refs/tags/ng-} - - name: Create Release + name: Create release id: create_release uses: actions/create-release@v1 env: @@ -173,7 +177,7 @@ jobs: body: | TODO: Insert changelog here (it's also in the documentation) - - name: Upload Release Archive + name: Upload release archive id: upload-release-asset uses: actions/upload-release-asset@v1 env: @@ -206,6 +210,7 @@ jobs: name: Checkout uses: actions/checkout@v2 - + name: Download frontend artifact uses: actions/download-artifact@v2 with: name: frontend-compiled From 262b952ae77b3745b17760239179cca5a6d65c4e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 17:13:56 +0100 Subject: [PATCH 140/148] try building this with updated dependencies --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 789a15641..b5d2e5b13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,8 @@ WORKDIR /usr/src/paperless/ COPY requirements.txt ./ # Binary dependencies -RUN apt-get update \ +RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \ + && apt-get update \ && apt-get -y --no-install-recommends install \ # Basic dependencies curl \ From e94a3569fc5b0ae6f503ed3db372c3692ded11c3 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 21:55:06 +0100 Subject: [PATCH 141/148] don't use dependencies from testing --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5d2e5b13..789a15641 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,7 @@ WORKDIR /usr/src/paperless/ COPY requirements.txt ./ # Binary dependencies -RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \ - && apt-get update \ +RUN apt-get update \ && apt-get -y --no-install-recommends install \ # Basic dependencies curl \ From 3bbb2eeeed92b6db8329b1c9df0384c840145080 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 21:56:50 +0100 Subject: [PATCH 142/148] pin pillow dependency --- Pipfile | 3 ++- Pipfile.lock | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 1a9b43950..5c1c136b5 100644 --- a/Pipfile +++ b/Pipfile @@ -23,7 +23,8 @@ imap-tools = "*" langdetect = "*" pdftotext = "*" pathvalidate = "*" -pillow = "*" +# pinned to 8.1.0, since aarch64 wheels might not be available beyond that +pillow = "==8.1.0" pikepdf = "*" python-gnupg = "*" python-dotenv = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 58cf35894..c3aa4a4f5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "241c8d042e9b607bbeb7d501b3fc1b57376e3cc8c2b2fe117ec60e425dfd79c7" + "sha256": "27c8973d55150371bd18406cc90e61ad4a7ec4b9353007045d611a87120a2703" }, "pipfile-spec": 6, "requires": {}, @@ -437,6 +437,7 @@ "pillow": { "hashes": [ "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6", + "sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded", "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865", "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174", "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032", @@ -462,9 +463,12 @@ "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8", "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59", "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d", + "sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7", "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a", + "sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0", "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b", - "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d" + "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d", + "sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae" ], "index": "pypi", "version": "==8.1.0" From 47ebe5c263c9259d86cea71c185ff2448b5da86a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 22:11:01 +0100 Subject: [PATCH 143/148] fix test case --- src/paperless_mail/tests/test_mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 77db096c4..0104f5023 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -41,7 +41,7 @@ class BogusMailBox(ContextManager): folder = BogusFolderManager() - def fetch(self, criteria, mark_seen): + def fetch(self, criteria, mark_seen, charset=""): msg = self.messages criteria = str(criteria).strip('()').split(" ") From 980ceb18d51e75869e4a2430a3b42ec606030abe Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 23:07:47 +0100 Subject: [PATCH 144/148] remove obsolete steps from the documentation --- docs/administration.rst | 18 ++++-------------- docs/setup.rst | 7 ------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index bc728943e..c3d1d07a6 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -110,27 +110,17 @@ After grabbing the new release and unpacking the contents, do the following: This creates a new virtual environment (or uses your existing environment) and installs all dependencies into it. - -3. Collect static files. - - .. code:: shell-session - - $ cd src - $ pipenv run python3 manage.py collectstatic --clear -4. Migrate the database. +3. Migrate the database. .. code:: shell-session $ cd src $ pipenv run python3 manage.py migrate -5. Update translation files. - - .. code:: shell-session - - $ cd src - $ pipenv run python3 manage.py compilemessages + This might not actually do anything. Not every new paperless version comes with new + database migrations. + Management utilities #################### diff --git a/docs/setup.rst b/docs/setup.rst index ed7f84885..48569a04c 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -225,7 +225,6 @@ writing. Windows is not and will never be supported. * ``python3-pip``, optionally ``pipenv`` for package installation * ``python3-dev`` - * ``gettext`` for compiling interface translations * ``fonts-liberation`` for generating thumbnails for plain text files * ``imagemagick`` >= 6 for PDF conversion * ``optipng`` for optimizing thumbnails @@ -293,15 +292,9 @@ writing. Windows is not and will never be supported. .. code:: bash - # This collects static files from paperless and django. - python3 manage.py collectstatic --clear --no-input - # This creates the database schema. python3 manage.py migrate - # This creates the translation files for paperless. - python3 manage.py compilemessages - # This creates your first paperless user python3 manage.py createsuperuser From 07f1ecab3216f19e129cce6f0b3f2a73e58c1f81 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:09:10 +0100 Subject: [PATCH 145/148] copy files into place --- .gitignore | 4 ++-- docker/hub/.env | 1 + docker/hub/docker-compose.env | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 docker/hub/.env create mode 100644 docker/hub/docker-compose.env diff --git a/.gitignore b/.gitignore index e58fb12e7..b427d5c5c 100644 --- a/.gitignore +++ b/.gitignore @@ -65,8 +65,8 @@ target/ .virtualenv virtualenv /venv -docker-compose.env -docker-compose.yml +/docker-compose.env +/docker-compose.yml # Used for development scripts/import-for-development diff --git a/docker/hub/.env b/docker/hub/.env new file mode 100644 index 000000000..511a1386d --- /dev/null +++ b/docker/hub/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=paperless \ No newline at end of file diff --git a/docker/hub/docker-compose.env b/docker/hub/docker-compose.env new file mode 100644 index 000000000..4271bce6e --- /dev/null +++ b/docker/hub/docker-compose.env @@ -0,0 +1,34 @@ +# The UID and GID of the user used to run paperless in the container. Set this +# to your UID and GID on the host so that you have write access to the +# consumption directory. +#USERMAP_UID=1000 +#USERMAP_GID=1000 + +# Additional languages to install for text recognition, separated by a +# whitespace. Note that this is +# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the +# default language used when guessing the language from the OCR output. +# The container installs English, German, Italian, Spanish and French by +# default. +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# for available languages. +#PAPERLESS_OCR_LANGUAGES=tur ces + +############################################################################### +# Paperless-specific settings # +############################################################################### + +# All settings defined in the paperless.conf.example can be used here. The +# Docker setup does not use the configuration file. +# A few commonly adjusted settings are provided below. + +# Adjust this key if you plan to make paperless available publicly. It should +# be a very long sequence of random characters. You don't need to remember it. +#PAPERLESS_SECRET_KEY=change-me + +# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. +#PAPERLESS_TIME_ZONE=America/Los_Angeles + +# The default language to use for OCR. Set this to the language most of your +# documents are written in. +#PAPERLESS_OCR_LANGUAGE=eng From 6f8aa3500aa51393642eec2e12534f32c90a4d97 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:21:05 +0100 Subject: [PATCH 146/148] only restart docker unless stopped --- docker/hub/docker-compose.postgres.yml | 6 +++--- docker/hub/docker-compose.sqlite.yml | 4 ++-- docker/hub/docker-compose.tika.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml index f821a6781..863c99c32 100644 --- a/docker/hub/docker-compose.postgres.yml +++ b/docker/hub/docker-compose.postgres.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped db: image: postgres:13 - restart: always + restart: unless-stopped volumes: - pgdata:/var/lib/postgresql/data environment: @@ -16,7 +16,7 @@ services: webserver: image: jonaswinkler/paperless-ng:latest - restart: always + restart: unless-stopped depends_on: - db - broker diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml index cc26dc46c..2ab0cbb9c 100644 --- a/docker/hub/docker-compose.sqlite.yml +++ b/docker/hub/docker-compose.sqlite.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: image: jonaswinkler/paperless-ng:latest - restart: always + restart: unless-stopped depends_on: - broker ports: diff --git a/docker/hub/docker-compose.tika.yml b/docker/hub/docker-compose.tika.yml index 7ad7685d3..f9e44472a 100644 --- a/docker/hub/docker-compose.tika.yml +++ b/docker/hub/docker-compose.tika.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: image: jonaswinkler/paperless-ng:latest - restart: always + restart: unless-stopped depends_on: - broker ports: From f5382860e05d2fcce74e18666523e4d5d7003908 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:29:05 +0100 Subject: [PATCH 147/148] documentation --- docs/administration.rst | 5 ++--- docs/setup.rst | 17 +++++++---------- docs/usage_overview.rst | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index c3d1d07a6..80f57d849 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -48,9 +48,6 @@ Options available to bare-metal and non-docker installations: Restoring ========= - - - .. _administration-updating: Updating Paperless @@ -84,9 +81,11 @@ B. If you built the image yourself, do the following: .. code:: shell-session $ git pull + $ ./compile-frontend.sh $ docker-compose build $ docker-compose up +Running `docker-compose up` will also apply any new database migrations. If you see everything working, you can start paperless-ng with "-d" to have it run in the background. diff --git a/docs/setup.rst b/docs/setup.rst index 48569a04c..1e46608ae 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -97,9 +97,10 @@ run the above mentioned components yourself. Install Paperless from Docker Hub ================================= -1. Go to the `project page `_ +1. Go to the `/docker/hub directory on the project page `_ and download one of the ``docker-compose.*.yml`` files, depending on which database backend you - want to use. Download the ``docker-compose.env`` file and the ``.env`` file as well and store them + want to use. Rename this file to `docker-compose.yml`. + Download the ``docker-compose.env`` file and the ``.env`` file as well and store them in the same directory. .. hint:: @@ -124,7 +125,6 @@ Install Paperless from Docker Hub .. _Docker installation guide: https://docs.docker.com/engine/installation/ .. _docker-compose installation guide: https://docs.docker.com/compose/install/ - 3. Modify ``docker-compose.yml`` to your preferences. You may want to change the path to the consumption directory in this file. Find the line that specifies where to mount the consumption directory: @@ -201,7 +201,9 @@ Build the docker image yourself depending on which database backend you want to use. Copy ``docker-compose.env.example`` to ``docker-compose.env``. -3. Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run +3. Run the `compile-frontend.sh` script. This requires `node` and `npm`. + +4. Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run ``docker-compose up -d`` to start the containers, do .. code:: shell-session @@ -422,12 +424,7 @@ Migration to paperless-ng is then performed in a few simple steps: 5. Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs. See :ref:`setup-docker_hub` for details on which edits are advised. -6. Since ``docker-compose`` would just use the the old paperless image, we need to - manually build a new image: - - .. code:: shell-session - - $ docker-compose build +6. :ref:`Update paperless. ` 7. In order to find your existing documents with the new search feature, you need to invoke a one-time operation that will create the search index: diff --git a/docs/usage_overview.rst b/docs/usage_overview.rst index 7a4fd7740..6c34b2823 100644 --- a/docs/usage_overview.rst +++ b/docs/usage_overview.rst @@ -118,7 +118,7 @@ to share any documents with paperless. This can be combined with any of the mobi scanning apps out there, such as Office Lens. Furthermore, there is the `Paperless App `_ as well, -which no only has document upload, but also document editing and browsing. +which not only has document upload, but also document browsing and download features. .. _usage-email: From 1b7e507002c2fb6a3cccc2565c23f8e322e4f864 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:40:49 +0100 Subject: [PATCH 148/148] unless-stopped in local dockerfiles --- docker-compose.postgres.yml | 6 +++--- docker-compose.sqlite.yml | 4 ++-- docker-compose.tika.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml index 59ae28939..53fea9e28 100644 --- a/docker-compose.postgres.yml +++ b/docker-compose.postgres.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped db: image: postgres:13 - restart: always + restart: unless-stopped volumes: - pgdata:/var/lib/postgresql/data environment: @@ -16,7 +16,7 @@ services: webserver: build: . - restart: always + restart: unless-stopped depends_on: - db - broker diff --git a/docker-compose.sqlite.yml b/docker-compose.sqlite.yml index 2d21fa3f1..87aa100a6 100644 --- a/docker-compose.sqlite.yml +++ b/docker-compose.sqlite.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: build: . - restart: always + restart: unless-stopped depends_on: - broker ports: diff --git a/docker-compose.tika.yml b/docker-compose.tika.yml index 889713908..d7b7e0da9 100644 --- a/docker-compose.tika.yml +++ b/docker-compose.tika.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: build: . - restart: always + restart: unless-stopped depends_on: - broker ports: