From 5b344963b921afdaf06c4c58df9d59b23f4e53d2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 15:58:29 +0100 Subject: [PATCH 001/148] reorganized docker build. --- docker/local/Dockerfile => Dockerfile | 17 ++- ...ostgres.yml => docker-compose.postgres.yml | 0 ...se.sqlite.yml => docker-compose.sqlite.yml | 0 docker/docker-compose.env | 34 ----- scripts/make-release.sh | 125 ------------------ scripts/push-release.sh | 23 ---- 6 files changed, 14 insertions(+), 185 deletions(-) rename docker/local/Dockerfile => Dockerfile (78%) rename docker/local/docker-compose.postgres.yml => docker-compose.postgres.yml (100%) rename docker/local/docker-compose.sqlite.yml => docker-compose.sqlite.yml (100%) delete mode 100644 docker/docker-compose.env delete mode 100755 scripts/make-release.sh delete mode 100755 scripts/push-release.sh diff --git a/docker/local/Dockerfile b/Dockerfile similarity index 78% rename from docker/local/Dockerfile rename to Dockerfile index 461b9e4fc..732b4183c 100644 --- a/docker/local/Dockerfile +++ b/Dockerfile @@ -1,8 +1,18 @@ +FROM node:15 AS frontend + +WORKDIR /usr/src/paperless/src-ui/ + +COPY src-ui/package* ./ +RUN npm install + +COPY src-ui . +RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false + FROM python:3.7-slim WORKDIR /usr/src/paperless/ -COPY requirements.txt ./ +COPY Pipfile* ./ #Dependencies RUN apt-get update \ @@ -33,8 +43,8 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools \ - && pip install --no-cache-dir -r requirements.txt \ + && pip3 install --upgrade supervisor setuptools pipenv \ + && pipenv install --system --deploy --ignore-pipfile --clear \ && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ @@ -49,6 +59,7 @@ COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh # copy app COPY src/ ./src/ +COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/docker/local/docker-compose.postgres.yml b/docker-compose.postgres.yml similarity index 100% rename from docker/local/docker-compose.postgres.yml rename to docker-compose.postgres.yml diff --git a/docker/local/docker-compose.sqlite.yml b/docker-compose.sqlite.yml similarity index 100% rename from docker/local/docker-compose.sqlite.yml rename to docker-compose.sqlite.yml diff --git a/docker/docker-compose.env b/docker/docker-compose.env deleted file mode 100644 index 4271bce6e..000000000 --- a/docker/docker-compose.env +++ /dev/null @@ -1,34 +0,0 @@ -# The UID and GID of the user used to run paperless in the container. Set this -# to your UID and GID on the host so that you have write access to the -# consumption directory. -#USERMAP_UID=1000 -#USERMAP_GID=1000 - -# Additional languages to install for text recognition, separated by a -# whitespace. Note that this is -# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the -# default language used when guessing the language from the OCR output. -# The container installs English, German, Italian, Spanish and French by -# default. -# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster -# for available languages. -#PAPERLESS_OCR_LANGUAGES=tur ces - -############################################################################### -# Paperless-specific settings # -############################################################################### - -# All settings defined in the paperless.conf.example can be used here. The -# Docker setup does not use the configuration file. -# A few commonly adjusted settings are provided below. - -# Adjust this key if you plan to make paperless available publicly. It should -# be a very long sequence of random characters. You don't need to remember it. -#PAPERLESS_SECRET_KEY=change-me - -# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. -#PAPERLESS_TIME_ZONE=America/Los_Angeles - -# The default language to use for OCR. Set this to the language most of your -# documents are written in. -#PAPERLESS_OCR_LANGUAGE=eng diff --git a/scripts/make-release.sh b/scripts/make-release.sh deleted file mode 100755 index f5c9028fa..000000000 --- a/scripts/make-release.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -# Release checklist -# - wait for travis build. -# adjust src/paperless/version.py -# changelog in the documentation -# adjust versions in docker/hub/* -# adjust version in src-ui/src/environments/prod -# If docker-compose was modified: all compose files are the same. - -# Steps: -# run release script "dev", push -# if it works: new tag, merge into master -# on master: make release "lastest", push -# on master: make release "version-tag", push -# publish release files - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" -PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles" - -if [ -d "$PAPERLESS_DIST" ] -then - echo "Removing $PAPERLESS_DIST" - rm "$PAPERLESS_DIST" -r -fi - -mkdir "$PAPERLESS_DIST" -mkdir "$PAPERLESS_DIST_APP" -mkdir "$PAPERLESS_DIST_APP/docker" -mkdir "$PAPERLESS_DIST_APP/scripts" -mkdir "$PAPERLESS_DIST_DOCKERFILES" - -# setup dependencies. - -cd "$PAPERLESS_ROOT" - -pipenv clean -pipenv install --dev -pipenv lock --keep-outdated -r > "$PAPERLESS_DIST_APP/requirements.txt" - -# test if the application works. - -cd "$PAPERLESS_ROOT/src" -pipenv run pytest --cov -pipenv run pycodestyle - -# make the documentation. - -cd "$PAPERLESS_ROOT/docs" -make clean html - -# copy stuff into place - -# the application itself - -cp "$PAPERLESS_ROOT/.env" \ - "$PAPERLESS_ROOT/.dockerignore" \ - "$PAPERLESS_ROOT/CONTRIBUTING.md" \ - "$PAPERLESS_ROOT/LICENSE" \ - "$PAPERLESS_ROOT/Pipfile" \ - "$PAPERLESS_ROOT/Pipfile.lock" \ - "$PAPERLESS_ROOT/README.md" "$PAPERLESS_DIST_APP" - -cp "$PAPERLESS_ROOT/paperless.conf.example" "$PAPERLESS_DIST_APP/paperless.conf" - -# copy python source, templates and static files. -cd "$PAPERLESS_ROOT" -find src -wholename '*/templates/*' -o -wholename '*/static/*' -o -name '*.py' | cpio -pdm "$PAPERLESS_DIST_APP" - -# build the front end. - -cd "$PAPERLESS_ROOT/src-ui" -ng build --prod --output-hashing none --sourceMap=false --output-path "$PAPERLESS_DIST_APP/src/documents/static/frontend" - -# documentation -cp "$PAPERLESS_ROOT/docs/_build/html/" "$PAPERLESS_DIST_APP/docs" -r - -# docker files for building the image yourself -cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP" - -# docker files for pulling from docker hub -cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES" - -# auxiliary files required for the docker image -cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/gunicorn.conf.py" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/imagemagick-policy.xml" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/" - -# auxiliary files for bare metal installs -cp "$PAPERLESS_ROOT/scripts/paperless-webserver.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-consumer.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-scheduler.service" "$PAPERLESS_DIST_APP/scripts/" - -# try to make the docker build. - -cd "$PAPERLESS_DIST_APP" - -docker build . -t "jonaswinkler/paperless-ng:$VERSION" - -# works. package the app! - -cd "$PAPERLESS_DIST" - -tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/ -tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/ diff --git a/scripts/push-release.sh b/scripts/push-release.sh deleted file mode 100755 index cfa63f5cf..000000000 --- a/scripts/push-release.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" - -cd "$PAPERLESS_DIST_APP" - -docker push "jonaswinkler/paperless-ng:$VERSION" From dd6aa2f77539fc66835484f7a95e80cb08a7e026 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 16:01:16 +0100 Subject: [PATCH 002/148] added missing file --- docker-compose.example.env | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docker-compose.example.env diff --git a/docker-compose.example.env b/docker-compose.example.env new file mode 100644 index 000000000..4271bce6e --- /dev/null +++ b/docker-compose.example.env @@ -0,0 +1,34 @@ +# The UID and GID of the user used to run paperless in the container. Set this +# to your UID and GID on the host so that you have write access to the +# consumption directory. +#USERMAP_UID=1000 +#USERMAP_GID=1000 + +# Additional languages to install for text recognition, separated by a +# whitespace. Note that this is +# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the +# default language used when guessing the language from the OCR output. +# The container installs English, German, Italian, Spanish and French by +# default. +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# for available languages. +#PAPERLESS_OCR_LANGUAGES=tur ces + +############################################################################### +# Paperless-specific settings # +############################################################################### + +# All settings defined in the paperless.conf.example can be used here. The +# Docker setup does not use the configuration file. +# A few commonly adjusted settings are provided below. + +# Adjust this key if you plan to make paperless available publicly. It should +# be a very long sequence of random characters. You don't need to remember it. +#PAPERLESS_SECRET_KEY=change-me + +# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. +#PAPERLESS_TIME_ZONE=America/Los_Angeles + +# The default language to use for OCR. Set this to the language most of your +# documents are written in. +#PAPERLESS_OCR_LANGUAGE=eng From 23381f7d170d8dcf1aabcbb26f5dbe04816ca0a8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 13:13:52 +0100 Subject: [PATCH 003/148] Added multi arch docker builds to travis ci --- .travis.yml | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.travis.yml b/.travis.yml index b745d6bd7..307a6073a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,102 @@ jobs: - ng build --prod after_success: true + - stage: build_docker + name: amd64 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 + on: + condition: '"${BUILD_DOCKER}" = 1' + - stage: build_docker + name: arm64v8 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 + arch: arm64 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: build_docker + name: arm32v7 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: publish_manifest + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} + - | + if [ "${TRAVIS_BRANCH}" = "master" ]; then + echo "Master branch detected" + DOCKER_TAG="latest" + else + DOCKER_TAG=${TRAVIS_TAG} + fi + - | + if [ "${DOCKER_TAG}" != "" ]; then + echo "Create Tag ${DOCKER_TAG}" + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} + else + echo "Not a tag and not on master, so not pushing tag/master specific manifest" + fi + on: + condition: '"${BUILD_DOCKER}" = 1' + before_install: - sudo apt-get update -qq From ba18258750e06d4a9f6529baf3722ec467474a88 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 16:19:24 +0100 Subject: [PATCH 004/148] Add libxslt-dev to dependencies. At least one arm based image does not pull in dependencies the same way the amd64 one does, resulting in a missing libxslt shared lib. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..016bf7da9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ + libxslt-dev \ optipng \ pngquant \ qpdf \ From 7817315f8b4b6fb9c95c08f4f788a8c068c4b401 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 17:08:01 +0100 Subject: [PATCH 005/148] Adding some echos to bette runderstand where the build hangs --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 307a6073a..203861391 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,9 +60,12 @@ jobs: after_success: - true script: + - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64 on: From 48b59c8e24081d06d9c38457517cf55cdcc5f79b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 21:53:39 +0100 Subject: [PATCH 006/148] Use qemu also for arm64v8 build - something might be causing trouble on the native hosts --- .travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 203861391..d2534c152 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,14 +60,17 @@ jobs: after_success: - true script: - - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64 + arch: amd64 on: condition: '"${BUILD_DOCKER}" = 1' From ade951a600ee4e00fb1aaeeb8a13fec569640d2e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 23:02:26 +0100 Subject: [PATCH 007/148] Adds jbig2 to the build, fixes #93 --- Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..6c04a1d07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,14 @@ +FROM ubuntu:20.04 AS jbig2 + +WORKDIR /usr/src/jbig2enc + +RUN apt-get update && apt-get install -y --no-install-recommends build-essential automake libtool libleptonica-dev zlib1g-dev git ca-certificates + +RUN git clone https://github.com/agl/jbig2enc . +RUN ./autogen.sh +RUN ./configure && make + + FROM node:15 AS frontend WORKDIR /usr/src/paperless/src-ui/ @@ -57,6 +68,12 @@ COPY docker/gunicorn.conf.py ./ COPY docker/supervisord.conf /etc/supervisord.conf COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh +# copy jbic +COPY --from=jbig2 /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ +COPY --from=jbig2 /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ +COPY --from=jbig2 /usr/src/jbig2enc/src/*.h /usr/local/include/ + + # copy app COPY src/ ./src/ COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ From 7d1f931234147bb4588778fda9d3b1fccf4a7ecd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:49:36 +0100 Subject: [PATCH 008/148] Use arm64-gravis instead of regular arm64. See if that is fast enough. --- .travis.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2534c152..b91e43a28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,17 +60,12 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: amd64 + arch: arm64-graviton2 + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From bf3b110804597af01ade2047527582d9f24ae9b2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:58:35 +0100 Subject: [PATCH 009/148] Remove accidentially added quote --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b91e43a28..5f838755d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 From 162626209c82ce8f8f286790302a279afdac6f86 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:27:19 +0100 Subject: [PATCH 010/148] Tell me, what kind of arm64 is it that produces amd64 docker images? It's weird --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f838755d..f99a52eba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,12 +60,13 @@ jobs: after_success: - true script: + - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: vm + virt: lxc on: condition: '"${BUILD_DOCKER}" = 1' @@ -85,8 +86,8 @@ jobs: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile + - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 From e8ea373eb2c4e3a686e07d2fc8919b0b15bb3281 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:42:21 +0100 Subject: [PATCH 011/148] Enable manifests support --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f99a52eba..6d26d6ccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,8 @@ jobs: # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: condition: '"${BUILD_DOCKER}" = 1' From 46ecdefde84eb581b038da1cdc2728cc3f11ea8d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:43:51 +0100 Subject: [PATCH 012/148] Remove steps not necessary to build docker images --- .travis.yml | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d26d6ccb..c04f765d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: - - name: "Paperless on Python 3.6" - python: "3.6" - - - name: "Paperless on Python 3.7" - python: "3.7" - - - name: "Paperless on Python 3.8" - python: "3.8" - - - name: "Documentation" - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true +# - name: "Paperless on Python 3.6" +# python: "3.6" +# +# - name: "Paperless on Python 3.7" +# python: "3.7" +# +# - name: "Paperless on Python 3.8" +# python: "3.8" +# +# - name: "Documentation" +# script: +# - cd docs/ +# - make html +# after_success: true +# +# - name: "Front end" +# language: node_js +# node_js: +# - 15 +# before_install: true +# install: +# - cd src-ui/ +# - npm install -g @angular/cli +# - npm install +# script: +# - ng build --prod +# after_success: true - stage: build_docker name: amd64 docker build From c751fe752079bf1367944e81c26a4d718a4531be Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:15:06 +0100 Subject: [PATCH 013/148] Switching back to virt: vm as there seems to be a prob with the lxc ones --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c04f765d0..9803a4d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ jobs: - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: lxc + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From 1f62b6d0d4c44516818a3e8e5d6e3efaa50f97f2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:18:11 +0100 Subject: [PATCH 014/148] arm64-graviton2 build requires group:edge --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9803a4d3b..8422229c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 virt: vm + group: edge on: condition: '"${BUILD_DOCKER}" = 1' From ccfa14b9b7c48ed60fc49ce64d673dde02edb308 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 09:14:40 +0100 Subject: [PATCH 015/148] Remove arm32v6 from docker manifest , it is not getting created --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8422229c7..281635ace 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ jobs: - true script: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} @@ -124,7 +124,7 @@ jobs: - | if [ "${DOCKER_TAG}" != "" ]; then echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} From 8135de49f00e01170c1450f380fb1789626d68af Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 12:13:12 +0100 Subject: [PATCH 016/148] Re-enable test steps --- .travis.yml | 55 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 281635ace..e5f3cd91e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: -# - name: "Paperless on Python 3.6" -# python: "3.6" -# -# - name: "Paperless on Python 3.7" -# python: "3.7" -# -# - name: "Paperless on Python 3.8" -# python: "3.8" -# -# - name: "Documentation" -# script: -# - cd docs/ -# - make html -# after_success: true -# -# - name: "Front end" -# language: node_js -# node_js: -# - 15 -# before_install: true -# install: -# - cd src-ui/ -# - npm install -g @angular/cli -# - npm install -# script: -# - ng build --prod -# after_success: true + - name: "Paperless on Python 3.6" + python: "3.6" + + - name: "Paperless on Python 3.7" + python: "3.7" + + - name: "Paperless on Python 3.8" + python: "3.8" + + - name: "Documentation" + script: + - cd docs/ + - make html + after_success: true + + - name: "Front end" + language: node_js + node_js: + - 15 + before_install: true + install: + - cd src-ui/ + - npm install -g @angular/cli + - npm install + script: + - ng build --prod + after_success: true - stage: build_docker name: amd64 docker build @@ -60,7 +60,6 @@ jobs: after_success: - true script: - - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . From 02e81f7ab5942afac95d1346ab743f17be4a04f2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 23 Dec 2020 17:03:00 +0100 Subject: [PATCH 017/148] update Dockerfile --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f0112e529..686496475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ - libxslt-dev \ + libxslt1-dev \ optipng \ pngquant \ qpdf \ @@ -55,9 +55,11 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools pipenv \ - && pipenv install --system --deploy --ignore-pipfile --clear \ - && apt-get -y purge build-essential libqpdf-dev \ + && pip3 install --upgrade supervisor pipenv \ + && pipenv install --system --deploy --ignore-pipfile \ + && pipenv --clear \ + && pip3 uninstall -y pipenv \ + && apt-get -y purge build-essential libqpdf-dev libxslt1-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord From 0560ac2a957957030b695b7645d2d35f7809b683 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sun, 27 Dec 2020 20:01:21 +0100 Subject: [PATCH 018/148] Use conditions to prevent builds on branches / tags other than master, dev and ng-* --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5f3cd91e..cf18825e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ jobs: - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -68,8 +68,7 @@ jobs: virt: vm group: edge on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -94,8 +93,7 @@ jobs: env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -131,7 +129,7 @@ jobs: echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From 61834581d5b840eedf35177cee29efc33100330b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:22:35 +0100 Subject: [PATCH 019/148] Move build condition into separate stages list --- .travis.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf18825e8..26153588e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,12 @@ language: python dist: focal os: linux +stages: + - name: build_docker + if: branch =~ /^(master|dev|ng-.*)$/ + - name: publish_manifest + if: branch =~ /^(master|dev|ng-.*)$/ + jobs: include: - name: "Paperless on Python 3.6" @@ -47,8 +53,6 @@ jobs: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -67,8 +71,6 @@ jobs: arch: arm64-graviton2 virt: vm group: edge - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -92,8 +94,6 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -128,8 +128,6 @@ jobs: else echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From ba9b5c50d2a6cfac6c8615f5a9a603f4e0d33b5d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:24:39 +0100 Subject: [PATCH 020/148] Add stage definitions of test jobs and add test to list of stages Defining the stages instead of having travis guess them and implicitly build an order ensures that the test stage is executed before the docker images are built. --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 26153588e..f4873cf3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ dist: focal os: linux stages: + - name: test - name: build_docker if: branch =~ /^(master|dev|ng-.*)$/ - name: publish_manifest @@ -12,21 +13,26 @@ stages: jobs: include: - name: "Paperless on Python 3.6" + stage: test python: "3.6" - name: "Paperless on Python 3.7" + stage: test python: "3.7" - name: "Paperless on Python 3.8" + stage: test python: "3.8" - name: "Documentation" + stage: test script: - cd docs/ - make html after_success: true - name: "Front end" + stage: test language: node_js node_js: - 15 From b855222ee150e419930054ad3ea750a584c2b58b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:24:57 +0100 Subject: [PATCH 021/148] Removing travis in favor of github actions --- .travis.yml | 153 ---------------------------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f4873cf3c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,153 +0,0 @@ -language: python - -dist: focal -os: linux - -stages: - - name: test - - name: build_docker - if: branch =~ /^(master|dev|ng-.*)$/ - - name: publish_manifest - if: branch =~ /^(master|dev|ng-.*)$/ - -jobs: - include: - - name: "Paperless on Python 3.6" - stage: test - python: "3.6" - - - name: "Paperless on Python 3.7" - stage: test - python: "3.7" - - - name: "Paperless on Python 3.8" - stage: test - python: "3.8" - - - name: "Documentation" - stage: test - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - stage: test - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true - - - stage: build_docker - name: amd64 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - - stage: build_docker - name: arm64v8 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64-graviton2 - virt: vm - group: edge - - stage: build_docker - name: arm32v7 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - - stage: publish_manifest - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} - - | - if [ "${TRAVIS_BRANCH}" = "master" ]; then - echo "Master branch detected" - DOCKER_TAG="latest" - else - DOCKER_TAG=${TRAVIS_TAG} - fi - - | - if [ "${DOCKER_TAG}" != "" ]; then - echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} - else - echo "Not a tag and not on master, so not pushing tag/master specific manifest" - fi - - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - -install: - - pip install --upgrade pipenv - - pipenv install --system --dev - -script: - - cd src/ - - pipenv run pytest --cov - - pipenv run pycodestyle - -after_success: - - pipenv run coveralls From 916ea79e5749f295ad4652fe46862015cbd259c7 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:27:22 +0100 Subject: [PATCH 022/148] Experimenting with github actions --- .github/workflows/docker-publish.yml | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..f2da6b3b8 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,63 @@ +name: Docker + +on: + push: + branches: travis-multiarch-builds + +jobs: + # Build and push image to docker hub. + buildx: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + fi + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./Dockerfile . + - name: install buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v3.1.0 + + - name: Docker Buildx (build) + run: | + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Login + if: success() && github.event_name != 'pull_request' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + - name: Docker Buildx (push) + if: success() && github.event_name != 'pull_request' + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Check Manifest + if: always() && github.event_name != 'pull_request' + run: | + docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + - name: Clear + if: always() && github.event_name != 'pull_request' + run: | + rm -f ${HOME}/.docker/config.json \ No newline at end of file From 3f01a9e1e274105296fc52add82b6fd486abb988 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 19:32:04 +0100 Subject: [PATCH 023/148] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f2da6b3b8..5ee210ef2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,7 +15,7 @@ jobs: - name: Prepare id: prepare run: | - DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_IMAGE=moztr/paperless-ng DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then @@ -60,4 +60,4 @@ jobs: - name: Clear if: always() && github.event_name != 'pull_request' run: | - rm -f ${HOME}/.docker/config.json \ No newline at end of file + rm -f ${HOME}/.docker/config.json From 97d068c3bbd79002eca7147190ec0d61234f1278 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 1 Jan 2021 12:53:16 +0100 Subject: [PATCH 024/148] Removed separate build step It looks like part of the build step is repeated in the push step due to bad caching possibly. --- .github/workflows/docker-publish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5ee210ef2..53d53e25c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,10 +38,6 @@ jobs: - name: install buildx id: buildx uses: crazy-max/ghaction-docker-buildx@v3.1.0 - - - name: Docker Buildx (build) - run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} - name: Docker Login if: success() && github.event_name != 'pull_request' env: From 8ab251c5f7d86fce37bd92f6cb18e070c0f67221 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:04:46 +0100 Subject: [PATCH 025/148] Execute python tests --- .github/workflows/docker-publish.yml | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 53d53e25c..a2897c05b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -5,6 +5,47 @@ on: branches: travis-multiarch-builds jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python-version }} + - name: Update system, install system requirements + run: | + apt-get update -qq + apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + - name: Upgrade pip, Install dependencies + run: | + pip install --upgrade pip pipenv sphinx + pipenv lock -r > requirements.txt + pip install -r requirements.txt + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W + - name: Execute Tests + run: | + cd src/ + pytest --cov + pycodestyle + - name: Execute coveralls + run: | + coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 41c1b711c2d29692c4d366dd2ba789b91bb2c15c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:08:19 +0100 Subject: [PATCH 026/148] Use sudo for apt --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a2897c05b..f6ac55c1e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,8 +28,8 @@ jobs: key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | - apt-get update -qq - apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip, Install dependencies run: | pip install --upgrade pip pipenv sphinx From b8b0b61e67725a215e341f2ed967343b7e5cd81f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:15:10 +0100 Subject: [PATCH 027/148] Execute pip commands one after the other to find issue --- .github/workflows/docker-publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f6ac55c1e..a810f134f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -30,10 +30,14 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip, Install dependencies + - name: Upgrade pip run: | pip install --upgrade pip pipenv sphinx + - name: Gather pip requirements + run: | pipenv lock -r > requirements.txt + - name: Insall pip requirements + run: | pip install -r requirements.txt - name: Run sphinx run: | From ef28ae811767a51a94d1cf0b94584c852f9ab383 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:19:58 +0100 Subject: [PATCH 028/148] Move sphinx step below tests step as it used to be --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a810f134f..e020fb66b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,17 +36,17 @@ jobs: - name: Gather pip requirements run: | pipenv lock -r > requirements.txt - - name: Insall pip requirements + - name: Install pip requirements run: | pip install -r requirements.txt - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - name: Execute Tests run: | cd src/ pytest --cov pycodestyle + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W - name: Execute coveralls run: | coveralls From 4a45a77753addc28f9b845a51978d8729b1c2db8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:23:49 +0100 Subject: [PATCH 029/148] Add pytest to install step --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e020fb66b..c074b1d2f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From d292f710077ad6fca94f64902b3ed9a2dc70888a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:31:20 +0100 Subject: [PATCH 030/148] Add pytest-cov to install --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c074b1d2f..9bba2e031 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest + pip install --upgrade pip pipenv sphinx pytest pytest-cov - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 90f4ccfb941c4d973094d84b1dc8ce7f0605ad48 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:41:21 +0100 Subject: [PATCH 031/148] Remove cache to see if that helps with pytest-cov issues --- .github/workflows/docker-publish.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9bba2e031..090cfa937 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,15 +17,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | sudo apt-get update -qq From 4bea504690611de0340150a75b205f31d8638665 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:44:08 +0100 Subject: [PATCH 032/148] Try o make pipenv also install dev dependencies --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 090cfa937..24c3fe062 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov + pip install --upgrade pip pipenv sphinx - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 4684d199538c2b49ba5372f3db7bd99e99c8524b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:23:30 +0100 Subject: [PATCH 033/148] github-actions: readding explicit install explicitly install pytest, pytest-cov and coveralls. I think travis already did that and we need to explicitly do so now. removed installing dev-dependencies, do it explicitly. (just trying to reproduce the travis-ci build) --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 24c3fe062..afbb2571f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | - pipenv lock --dev -r > requirements.txt + pipenv lock -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 376efcde27f6f3c0b9f55695ea8013a2af63e650 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:27:38 +0100 Subject: [PATCH 034/148] github-actions: removing unsupported parameter I don't understand what the "-n auto" parameter does, pytest doesn't either (in the version installed) - so removing it to see if that produces something usable. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index 2a1a348bd..f43c9adf6 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html -n auto +addopts = --pythonwarnings=all --cov --cov-report=html env = PAPERLESS_DISABLE_DBHANDLER=true From 28209bb4746f048c42d2285aa36328e91bf57558 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:30:12 +0100 Subject: [PATCH 035/148] github-actions: also install wheel There's an error about pip wheel not being available and falling back to some legacy version. Let's see if this helps. --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index afbb2571f..a7a399c3f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,7 +23,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From a21fb36cd27e381d5297113ddb9ca442e138ca7c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:33:56 +0100 Subject: [PATCH 036/148] github-actions: readding previously removed "-n auto" possibly this is fixed by the wheel dependency that was missing before. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index f43c9adf6..2a1a348bd 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html +addopts = --pythonwarnings=all --cov --cov-report=html -n auto env = PAPERLESS_DISABLE_DBHANDLER=true From 2e641b0e3037fe76044a51129c0dc4b08624f6c2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:18:56 +0100 Subject: [PATCH 037/148] Explicitly set ubuntu-20.04 as OS and install dev dependencies --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a7a399c3f..d29fd7740 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,8 +9,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] - python-version: [3.7, 3.8, 3.9] + os: [ubuntu-20.04] + python-version: 3.7 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From e91befabc06f0e611c5d6a63d9df35d8b346bd36 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:22:35 +0100 Subject: [PATCH 038/148] Revert job strategy matrix change --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d29fd7740..cfda3d10d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: 3.7 + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 05f57249d6fdfa89fba0e3966528a62116d0dbde Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:24:48 +0100 Subject: [PATCH 039/148] Only build python 3.7 and say so in the Pipfile too --- .github/workflows/docker-publish.yml | 2 +- Pipfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cfda3d10d..0d2c8bd68 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7, 3.8, 3.9] + python-version: [3.7] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/Pipfile b/Pipfile index 48759307c..e0f1d83d8 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.6" +python_version = "3.7" [packages] dateparser = "~=0.7.6" From 69ac7b9bcf0efbfeadbfe4210c991e601674ce7e Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:00:35 +0100 Subject: [PATCH 040/148] Revert python version in Pipfile to 3.6 --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index e0f1d83d8..48759307c 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.7" +python_version = "3.6" [packages] dateparser = "~=0.7.6" From c9de5928320a74ad016128bfae5531c0220d208d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:10:16 +0100 Subject: [PATCH 041/148] Try to reproduce exactly what's done in the travis build to test --- .github/workflows/docker-publish.yml | 30 ++++++++-------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0d2c8bd68..66bc73ea8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,37 +10,23 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Update system, install system requirements + - name: Tests run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip - run: | - pip install --upgrade pip pipenv - - name: Gather pip requirements - run: | - pipenv lock --dev -r > requirements.txt - - name: Install pip requirements - run: | - pip install -r requirements.txt - - name: Execute Tests - run: | + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev cd src/ - pytest --cov - pycodestyle - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - - name: Execute coveralls - run: | - coveralls + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 7e99d7515f82dfbd3de680a3250796bcd907c7cb Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:17:13 +0100 Subject: [PATCH 042/148] Try manually replacing the python-version --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 66bc73ea8..264552168 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,6 +21,8 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + echo "Replacing python version with ${matrix.python-version}" + sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From b41dd2780e4e6cb50a9eb77f5d1034f365a13b08 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:19:00 +0100 Subject: [PATCH 043/148] Print the env --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 264552168..9904d8623 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -19,6 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | + env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng echo "Replacing python version with ${matrix.python-version}" From 336b908e8f57b41be05cf69cf024c6bc62bb627f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:28:47 +0100 Subject: [PATCH 044/148] Only build for 3.6 --- .github/workflows/docker-publish.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9904d8623..23c8aef20 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6, 3.7, 3.8] + python-version: [3.6] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -19,11 +19,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | - env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - echo "Replacing python version with ${matrix.python-version}" - sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From 574ab7d696bdcda4967cbf22d41b8e0ad679c8e1 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:44:02 +0100 Subject: [PATCH 045/148] Add GITHUB_TOKEN to python tests --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 23c8aef20..01bf72595 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From b0588321e0ff3d38b4272f0bb8e65124e3e04807 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:45:12 +0100 Subject: [PATCH 046/148] Fix yaml --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 01bf72595..c0a1565c6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,7 +18,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From b31bfeebd7936a0dcad0a5e1bbfd2ebbc11f2ea6 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 09:05:14 +0100 Subject: [PATCH 047/148] Try to make pip and pipenv use the installed python version --- .github/workflows/docker-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c0a1565c6..a61c570ce 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6] + python-version: [3.6,3.7,3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -20,10 +20,13 @@ jobs: - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MATRIX_PYTHON_VERSION: ${{strategy.matrix.python-version}} run: | + echo "Selected python version from matrix is: ${MATRIX_PYTHON_VERSION}" sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv + pyenv global system ${MATRIX_PYTHON_VERSION} pipenv install --system --dev cd src/ pipenv run pytest --cov From a6c6f6445350e7f4a365541325cf5f53c4880306 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 09:10:45 +0100 Subject: [PATCH 048/148] Use another python action instead --- .github/workflows/docker-publish.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a61c570ce..ec60880c9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -12,21 +12,20 @@ jobs: os: [ubuntu-20.04] python-version: [3.6,3.7,3.8] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v5 + with: + default: "${{ matrix.python-version }}" + command: pip install -U pip # upgrade pip after installing python - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MATRIX_PYTHON_VERSION: ${{strategy.matrix.python-version}} run: | echo "Selected python version from matrix is: ${MATRIX_PYTHON_VERSION}" sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv - pyenv global system ${MATRIX_PYTHON_VERSION} pipenv install --system --dev cd src/ pipenv run pytest --cov From 2587d7fec853907eedc938d5a42057fe9d2befec Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 10:20:31 +0100 Subject: [PATCH 049/148] Remove comment from yaml --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ec60880c9..81eba7e4f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,7 +17,7 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python-version }}" - command: pip install -U pip # upgrade pip after installing python + command: pip install -U pip - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 729d2da764e7abce7b52236e672b24307a4fd0fa Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 10:22:24 +0100 Subject: [PATCH 050/148] Remove command - should be unnecessary --- .github/workflows/docker-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 81eba7e4f..f13b7399e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,7 +17,6 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python-version }}" - command: pip install -U pip - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 96da61f4e4421c9a5eb96f5670f030fc55c0d7fd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:48:03 +0100 Subject: [PATCH 051/148] Trying to use pyenv action step --- .github/workflows/docker-publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f13b7399e..490a5edec 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,13 +10,14 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6,3.7,3.8] + python: [3.6,3.7,3.8] steps: - uses: actions/checkout@v2 - name: Install python version uses: gabrielfalcao/pyenv-action@v5 with: - default: "${{ matrix.python-version }}" + default: "${{ matrix.python }}" + command: pip install -U pip # upgrade pip after installing python - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e761cc76faad59f5a3f8cb1134cc5d018f35d426 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:50:11 +0100 Subject: [PATCH 052/148] List python versions differently so the pyenv action might light it better --- .github/workflows/docker-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 490a5edec..6507a6203 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,10 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python: [3.6,3.7,3.8] + python: + - 3.6 + - 3.7 + - 3.8 steps: - uses: actions/checkout@v2 - name: Install python version From b7ff94900fcf9563a57259cccbd595492ac09493 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:52:40 +0100 Subject: [PATCH 053/148] Specify exact version --- .github/workflows/docker-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6507a6203..203133335 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,9 +11,9 @@ jobs: matrix: os: [ubuntu-20.04] python: - - 3.6 - - 3.7 - - 3.8 + - 3.6.8 + - 3.7.5 + - 3.8.2 steps: - uses: actions/checkout@v2 - name: Install python version From 5b4a3e4288792a250ab8b809352a901d4d2eb34e Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:53:32 +0100 Subject: [PATCH 054/148] Remove comment --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 203133335..b0dc46ba5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -20,7 +20,7 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python }}" - command: pip install -U pip # upgrade pip after installing python + command: pip install -U pip - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6f3cdbc165fcbf00fdca7fbfde4fd3a37ac94e69 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:54:11 +0100 Subject: [PATCH 055/148] Quote command --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b0dc46ba5..558ebea69 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -20,7 +20,7 @@ jobs: uses: gabrielfalcao/pyenv-action@v5 with: default: "${{ matrix.python }}" - command: pip install -U pip + command: "pip install -U pip" - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c7c167a058f29210b2a155daa7dcd47532c9dd2e Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:55:41 +0100 Subject: [PATCH 056/148] Update to a possibly working version of pyenv-action According to https://github.com/gabrielfalcao/pyenv-action/issues/136 this version should work --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 558ebea69..6b43b8710 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Install python version - uses: gabrielfalcao/pyenv-action@v5 + uses: gabrielfalcao/pyenv-action@v7 with: default: "${{ matrix.python }}" command: "pip install -U pip" From 44da3fd19553af4c68a4ec910bb2d5ee73e681f5 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 22:57:34 +0100 Subject: [PATCH 057/148] Fix yaml: indentation was wrong --- .github/workflows/docker-publish.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6b43b8710..1ec70f146 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,12 +15,12 @@ jobs: - 3.7.5 - 3.8.2 steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "${{ matrix.python }}" - command: "pip install -U pip" + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "${{ matrix.python }}" + command: "pip install -U pip" - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 9257cdac8f330197a46f6a31644f4ea630bf85dd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 23:06:40 +0100 Subject: [PATCH 058/148] Remove debugging line and specify release versions --- .github/workflows/docker-publish.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1ec70f146..5e9ed8632 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,9 +11,9 @@ jobs: matrix: os: [ubuntu-20.04] python: - - 3.6.8 - - 3.7.5 - - 3.8.2 + - 3.6 + - 3.7 + - 3.8 steps: - uses: actions/checkout@v2 - name: Install python version @@ -25,7 +25,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo "Selected python version from matrix is: ${MATRIX_PYTHON_VERSION}" sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv From 257ebb88f058486af13eb70fc637f878c6f87dda Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 23:10:06 +0100 Subject: [PATCH 059/148] Looks like we need to specify the exact version --- .github/workflows/docker-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5e9ed8632..042d64f24 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,9 +11,9 @@ jobs: matrix: os: [ubuntu-20.04] python: - - 3.6 - - 3.7 - - 3.8 + - 3.6.8 + - 3.7.9 + - 3.8.7 steps: - uses: actions/checkout@v2 - name: Install python version From d7b3e7bd69f6744cd23e43f194b56301b2a68820 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 6 Jan 2021 23:11:54 +0100 Subject: [PATCH 060/148] Use 3.8.6 --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 042d64f24..8a199abcd 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,7 +13,7 @@ jobs: python: - 3.6.8 - 3.7.9 - - 3.8.7 + - 3.8.6 steps: - uses: actions/checkout@v2 - name: Install python version From cc9f9b6e8b00d84b77c23d7584061d3f5b1a0e25 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:32:43 +0100 Subject: [PATCH 061/148] Docker build step depends on tests --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8a199abcd..649fba14d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,7 +36,7 @@ jobs: # Build and push image to docker hub. buildx: runs-on: ubuntu-latest - + needs: tests steps: - uses: actions/checkout@v2 From 98bc51a3ec914060c57b990cb15149957eb63d7f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:42:25 +0100 Subject: [PATCH 062/148] Adding documentation and frontend steps from travis build --- .github/workflows/docker-publish.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 649fba14d..50fdbfd31 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -33,6 +33,33 @@ jobs: pipenv run pytest --cov pipenv run pycodestyle pipenv run coveralls + documentation: + steps: + - uses: actions/checkout@v2 + - name: dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + - name: make + run: | + cd docs/ + make html + frontend: + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: dependencies + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + - name: build + run: | + ng build --prod # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 10995788d1e545e04deb31ffb9140cba36f6d9c9 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:43:45 +0100 Subject: [PATCH 063/148] Fix identation --- .github/workflows/docker-publish.yml | 54 ++++++++++++++-------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 50fdbfd31..74e5c0ac7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -33,33 +33,33 @@ jobs: pipenv run pytest --cov pipenv run pycodestyle pipenv run coveralls - documentation: - steps: - - uses: actions/checkout@v2 - - name: dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - - name: make - run: | - cd docs/ - make html - frontend: - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '15' - - name: dependencies - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - - name: build - run: | - ng build --prod + documentation: + steps: + - uses: actions/checkout@v2 + - name: dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + - name: make + run: | + cd docs/ + make html + frontend: + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: dependencies + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + - name: build + run: | + ng build --prod # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 9ffd9027faa2be8f08485f12ed8598380cd2152c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:44:36 +0100 Subject: [PATCH 064/148] Add missing runs-on --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 74e5c0ac7..76a7f18e2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -34,6 +34,7 @@ jobs: pipenv run pycodestyle pipenv run coveralls documentation: + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: dependencies @@ -47,6 +48,7 @@ jobs: cd docs/ make html frontend: + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 From 6101f42ae38758357e1291ecc752a475b2d76c8c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:45:17 +0100 Subject: [PATCH 065/148] ADd documentation and frontend to dependency for docker build --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 76a7f18e2..f2467cbf9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -65,7 +65,7 @@ jobs: # Build and push image to docker hub. buildx: runs-on: ubuntu-latest - needs: tests + needs: [tests, documentation, frontend] steps: - uses: actions/checkout@v2 From c06f6145a180ace15a649e13b99a5e1e837a0334 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:47:42 +0100 Subject: [PATCH 066/148] ALso install python before using pip in the documentation job --- .github/workflows/docker-publish.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f2467cbf9..6c3bf87f9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -37,6 +37,11 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "3.6.8" + command: "pip install -U pip" - name: dependencies run: | sudo apt-get update -qq From 08299fb808df27e1b2e39586e2142a2157fe8cb0 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:48:32 +0100 Subject: [PATCH 067/148] Simplify node build --- .github/workflows/docker-publish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6c3bf87f9..ab93fd8e7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -59,13 +59,11 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '15' - - name: dependencies + - name: build run: | cd src-ui/ npm install -g @angular/cli npm install - - name: build - run: | ng build --prod # Build and push image to docker hub. buildx: From 248e4e5eb90ed465a0ca3febaed6bbdd3d2115dd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 07:57:12 +0100 Subject: [PATCH 068/148] Try installing sphinx simply with apt --- .github/workflows/docker-publish.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ab93fd8e7..3d62a9f18 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -45,9 +45,7 @@ jobs: - name: dependencies run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev + sudo apt-get install -qq python3-sphinx - name: make run: | cd docs/ From 460d202eb0b33450f67b460ccdd6cebf8969cb72 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:37:25 +0100 Subject: [PATCH 069/148] Use pip to install sphinx Not sure why pipenv install doesn't do the trick .. let's try this way first --- .github/workflows/docker-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3d62a9f18..54281492e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -44,8 +44,7 @@ jobs: command: "pip install -U pip" - name: dependencies run: | - sudo apt-get update -qq - sudo apt-get install -qq python3-sphinx + pip install 'sphinx~=3.3' sphinx_rtd_theme - name: make run: | cd docs/ From 1238d8b994887b2abfea7016ca82bf43071c4937 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:44:05 +0100 Subject: [PATCH 070/148] Add pip cache Try to be a good citizen and use caches instead of hitting the repos all the time --- .github/workflows/docker-publish.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 54281492e..bce52b2e0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,6 +21,15 @@ jobs: with: default: "${{ matrix.python }}" command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python }} - name: Tests env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 40da245d8d5a64e5bf55be2822167f7b03432cec Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:50:04 +0100 Subject: [PATCH 071/148] Try to insall sphinx from pipfile --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index bce52b2e0..479f52a2c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -53,7 +53,8 @@ jobs: command: "pip install -U pip" - name: dependencies run: | - pip install 'sphinx~=3.3' sphinx_rtd_theme + pip install --upgrade pipenv + pipenv install --system --dev --ignore-pipfile - name: make run: | cd docs/ From 1b61765df84719817ec6b4c77f35c46eced942e9 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 08:56:48 +0100 Subject: [PATCH 072/148] Install pip --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 479f52a2c..fcdf93c8f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -54,7 +54,7 @@ jobs: - name: dependencies run: | pip install --upgrade pipenv - pipenv install --system --dev --ignore-pipfile + pipenv install --system --dev - name: make run: | cd docs/ From 89476e44794f0bbacbe60247237a62afc6409ba9 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 7 Jan 2021 09:02:29 +0100 Subject: [PATCH 073/148] Try to use pip cache and install dependnecies like in tests --- .github/workflows/docker-publish.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index fcdf93c8f..a3329b7ba 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,9 +50,20 @@ jobs: uses: gabrielfalcao/pyenv-action@v7 with: default: "3.6.8" - command: "pip install -U pip" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip3.6.8} - name: dependencies run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv pipenv install --system --dev - name: make From 1170827f3dfbb33a85ed34d581d5defe1a75eeea Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 7 Jan 2021 18:42:46 +0100 Subject: [PATCH 074/148] update docker ignore --- .dockerignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.dockerignore b/.dockerignore index 065d2a7e6..a332c2ee7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +**/__pycache__ /src-ui/.vscode /src-ui/node_modules /src-ui/dist @@ -10,3 +11,9 @@ .pytest_cache /dist /scripts +/resources +**/tests +**/*.spec.ts +**/htmlcov +/src/.pytest_cache +.idea From 38f279fd2a6175dc1ac29c694e7c675ecbfadf70 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 7 Jan 2021 21:16:16 +0100 Subject: [PATCH 075/148] update installation instructions --- docker/hub/docker-compose.postgres.yml | 2 +- docker/hub/docker-compose.sqlite.yml | 2 +- docker/hub/docker-compose.tika.yml | 2 +- docs/administration.rst | 44 ++------ docs/setup.rst | 133 +++++++++++++------------ docs/troubleshooting.rst | 2 +- 6 files changed, 81 insertions(+), 104 deletions(-) diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml index d7673d891..f821a6781 100644 --- a/docker/hub/docker-compose.postgres.yml +++ b/docker/hub/docker-compose.postgres.yml @@ -15,7 +15,7 @@ services: POSTGRES_PASSWORD: paperless webserver: - image: jonaswinkler/paperless-ng:0.9.12 + image: jonaswinkler/paperless-ng:latest restart: always depends_on: - db diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml index 054eb52d3..cc26dc46c 100644 --- a/docker/hub/docker-compose.sqlite.yml +++ b/docker/hub/docker-compose.sqlite.yml @@ -5,7 +5,7 @@ services: restart: always webserver: - image: jonaswinkler/paperless-ng:0.9.12 + image: jonaswinkler/paperless-ng:latest restart: always depends_on: - broker diff --git a/docker/hub/docker-compose.tika.yml b/docker/hub/docker-compose.tika.yml index 47c477bef..7ad7685d3 100644 --- a/docker/hub/docker-compose.tika.yml +++ b/docker/hub/docker-compose.tika.yml @@ -5,7 +5,7 @@ services: restart: always webserver: - image: jonaswinkler/paperless-ng:0.9.12 + image: jonaswinkler/paperless-ng:latest restart: always depends_on: - broker diff --git a/docs/administration.rst b/docs/administration.rst index 727d9a9d5..bc728943e 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -53,7 +53,7 @@ Restoring .. _administration-updating: -Updating paperless +Updating Paperless ################## If a new release of paperless-ng is available, upgrading depends on how you @@ -69,50 +69,26 @@ First of all, ensure that paperless is stopped. After that, :ref:`make a backup `. -A. If you used the dockerfiles archive, simply download the files of the new release, - adjust the settings in the files (i.e., the path to your consumption directory), - and replace your existing docker-compose files. Then start paperless as usual, - which will pull the new image, and update your database, if necessary: +A. If you pull the image from the docker hub, all you need to do is: .. code:: shell-session - $ cd /path/to/paperless + $ docker-compose pull $ docker-compose up - - If you see everything working, you can start paperless-ng with "-d" to have it - run in the background. - - .. hint:: - - The released docker-compose files specify exact versions to be pulled from the hub. - This is to ensure that if the docker-compose files should change at some point - (i.e., services updates/configured differently), you wont run into trouble due to - docker pulling the ``latest`` image and running it in an older environment. + + The docker-compose files refer to the ``latest`` version, which is always the latest + stable release. -B. If you built the image yourself, grab the new archive and replace your current - paperless folder with the new contents. - - After that, make the necessary adjustments to the docker-compose.yml (i.e., - adjust your consumption directory). - - Build and start the new image with: +B. If you built the image yourself, do the following: .. code:: shell-session - $ cd /path/to/paperless + $ git pull $ docker-compose build $ docker-compose up - If you see everything working, you can start paperless-ng with "-d" to have it - run in the background. - -.. hint:: - - You can usually keep your ``docker-compose.env`` file, since this file will - never include mandatory configuration options. However, it is worth checking - out the new version of this file, since it might have new recommendations - on what to configure. - +If you see everything working, you can start paperless-ng with "-d" to have it +run in the background. Updating paperless without docker ================================= diff --git a/docs/setup.rst b/docs/setup.rst index ddd246b16..5a38c3241 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -3,35 +3,6 @@ Setup ***** -Download -######## - -Go to the project page on GitHub and download the -`latest release `_. -There are multiple options available. - -* Download the dockerfiles archive if you want to pull paperless from - Docker Hub. - -* Download the dist archive and extract it if you want to build the docker image - yourself or want to install paperless without docker. - -.. hint:: - - In contrast to paperless, the recommended way to get and update paperless-ng - is not to pull the entire git repository. Paperless-ng includes artifacts - that need to be compiled, and that's already done for you in the release. - -.. admonition:: Want to try out paperless-ng before migrating? - - The release contains a file ``.env`` which sets the docker-compose project - name to "paperless", which is the same as before and instructs docker-compose - to reuse and upgrade your paperless volumes. - - Just rename the project name in that file to anything else and docker-compose - will create fresh volumes for you! - - Overview of Paperless-ng ######################## @@ -110,22 +81,33 @@ Installation You can go multiple routes with setting up and running Paperless: -* The `docker route`_ -* The `bare metal route`_ +* :ref:`Pull the image from Docker Hub ` +* :ref:`Build the Docker image yourself ` +* :ref:`Install Paperless directly on your system (bare metal) ` -The `docker route`_ is quick & easy. This is the recommended route. This configures all the stuff +The Docker routes are quick & easy. These are the recommended routes. This configures all the stuff from above automatically so that it just works and uses sensible defaults for all configuration options. -The `bare metal route`_ is more complicated to setup but makes it easier +The bare metal route is more complicated to setup but makes it easier should you want to contribute some code back. You need to configure and run the above mentioned components yourself. -.. _setup-docker_route: +.. _setup-docker_hub: -Docker Route -============ +Install Paperless from Docker Hub +================================= -1. Install `Docker`_ and `docker-compose`_. [#compose]_ +1. Go to the `project page `_ + and download one of the ``docker-compose.*.yml`` files, depending on which database backend you + want to use. Download the ``docker-compose.env`` file and the ``.env`` file as well and store them + in the same directory. + + .. hint:: + + For new installations, it is recommended to use PostgreSQL as the database + backend. + +2. Install `Docker`_ and `docker-compose`_. .. caution:: @@ -142,15 +124,8 @@ Docker Route .. _Docker installation guide: https://docs.docker.com/engine/installation/ .. _docker-compose installation guide: https://docs.docker.com/compose/install/ -2. Copy either ``docker-compose.sqlite.yml`` or ``docker-compose.postgres.yml`` to - ``docker-compose.yml``, depending on which database backend you want to use. - .. hint:: - - For new installations, it is recommended to use PostgreSQL as the database - backend. - -2. Modify ``docker-compose.yml`` to your preferences. You may want to change the path +3. Modify ``docker-compose.yml`` to your preferences. You may want to change the path to the consumption directory in this file. Find the line that specifies where to mount the consumption directory: @@ -167,7 +142,7 @@ Docker Route Don't change the part after the colon or paperless wont find your documents. -3. Modify ``docker-compose.env``, following the comments in the file. The +4. Modify ``docker-compose.env``, following the comments in the file. The most important change is to set ``USERMAP_UID`` and ``USERMAP_GID`` to the uid and gid of your user on the host system. This ensures that both the docker container and you on the host machine have write access @@ -177,7 +152,7 @@ Docker Route .. note:: - You can use any settings from the file ``paperless.conf`` in this file. + You can use any settings from the file ``paperless.conf.example`` in this file. Have a look at :ref:`configuration` to see whats available. .. caution:: @@ -188,11 +163,10 @@ Docker Route with the default configuration. You will need to use ``PAPERLESS_CONSUMER_POLLING``, which will disable inotify. See :ref:`here `. -4. Run ``docker-compose up -d``. This will create and start the necessary - containers. This will also build the image of paperless if you grabbed the - source archive. +5. Run ``docker-compose up -d``. This will create and start the necessary + containers. -5. To be able to login, you will need a super user. To create it, execute the +6. To be able to login, you will need a super user. To create it, execute the following command: .. code-block:: shell-session @@ -202,7 +176,7 @@ Docker Route This will prompt you to set a username, an optional e-mail address and finally a password. -6. The default ``docker-compose.yml`` exports the webserver on your local port +7. The default ``docker-compose.yml`` exports the webserver on your local port 8000. If you haven't adapted this, you should now be able to visit your Paperless instance at ``http://127.0.0.1:8000``. You can login with the user and password you just created. @@ -210,11 +184,33 @@ Docker Route .. _Docker: https://www.docker.com/ .. _docker-compose: https://docs.docker.com/compose/install/ -.. [#compose] You of course don't have to use docker-compose, but it - simplifies deployment immensely. If you know your way around Docker, feel - free to tinker around without using compose! +.. _setup-docker_build: -.. _`setup-bare_metal`: +Build the docker image yourself +=============================== + +1. Clone the entire repository of paperless: + + .. code:: shell-session + + git clone https://github.com/jonaswinkler/paperless-ng + + The master branch always reflects the latest stable version. + +2. Copy one of the ``docker-compose.*.yml`` to ``docker-compose.yml``, + depending on which database backend you want to use. Copy + ``docker-compose.env.example`` to ``docker-compose.env``. + +3. Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run + ``docker-compose up -d`` to start the containers, do + + .. code:: shell-session + + $ docker-compose build + + before that to build the image. + +.. _setup-bare_metal: Bare Metal Route ================ @@ -229,6 +225,7 @@ writing. Windows is not and will never be supported. * ``python3-pip``, optionally ``pipenv`` for package installation * ``python3-dev`` + * ``gettext`` for localization * ``fonts-liberation`` for generating thumbnails for plain text files * ``imagemagick`` >= 6 for PDF conversion * ``optipng`` for optimizing thumbnails @@ -258,8 +255,9 @@ writing. Windows is not and will never be supported. 3. Optional. Install ``postgresql`` and configure a database, user and password for paperless. If you do not wish to use PostgreSQL, SQLite is avialable as well. -4. Get the release archive. If you pull the git repo as it is, you also have to compile the front end by yourself. - Extract the frontend to a place from where you wish to execute it, such as ``/opt/paperless``. +4. Get the release archive from ``_. + If you clone the git repo as it is, you also have to compile the front end by yourself. + Extract the archive to a place from where you wish to execute it, such as ``/opt/paperless``. 5. Configure paperless. See :ref:`configuration` for details. Edit the included ``paperless.conf`` and adjust the settings to your needs. Required settings for getting paperless running are: @@ -372,6 +370,11 @@ writing. Windows is not and will never be supported. .. code:: + +13. Optional: Install the `jbig2enc `_ + encoder. This will reduce the size of generated PDF documents. You'll most likely need + to compile this by yourself, because this software has been patented until around 2017 and + binary packages are not available for most distributions. Migration to paperless-ng ######################### @@ -406,13 +409,14 @@ Migration to paperless-ng is then performed in a few simple steps: paperless. 3. Download the latest release of paperless-ng. You can either go with the - docker-compose files or use the archive to build the image yourself. + docker-compose files from `here `_ + or clone the repository to build the image yourself. You can either replace your current paperless folder or put paperless-ng in a different location. .. caution:: - The release include a ``.env`` file. This will set the + Paperless includes a ``.env`` file. This will set the project name for docker compose to ``paperless`` so that paperless-ng will automatically reuse your existing paperless volumes. When you start it, it will migrate your existing data. After that, your old paperless installation @@ -422,9 +426,8 @@ Migration to paperless-ng is then performed in a few simple steps: If you want to switch to PostgreSQL, do that after you migrated your existing SQLite database. -5. Adjust ``docker-compose.yml`` and - ``docker-compose.env`` to your needs. - See `docker route`_ for details on which edits are advised. +5. Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs. + See :ref:`setup-docker_hub` for details on which edits are advised. 6. Since ``docker-compose`` would just use the the old paperless image, we need to manually build a new image: @@ -452,7 +455,7 @@ Migration to paperless-ng is then performed in a few simple steps: This will run paperless in the background and automatically start it on system boot. 9. Paperless installed a permanent redirect to ``admin/`` in your browser. This - redirect is still in place and prevents access to the new UI. Clear + redirect is still in place and prevents access to the new UI. Clear your browsing cache in order to fix this. 10. Optionally, follow the instructions below to migrate your existing data to PostgreSQL. @@ -604,6 +607,4 @@ For details, refer to :ref:`configuration`. The actual matching of the algorithm is fast and works on Raspberry Pi as well as on any other device. - - .. _redis: https://redis.io/ diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 8786657db..b8343710f 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -75,6 +75,6 @@ You might encounter errors such as: This happens when paperless does not have permission to delete files inside the consumption directory. Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group id you use on the host operating system, if these are -different from ``1000``. See :ref:`setup-docker_route`. +different from ``1000``. See :ref:`setup-docker_hub`. Also ensure that you are able to read and write to the consumption directory on the host. From 42064df3e091a559738f2915a1ae5344fc0f860b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 11:23:17 +0100 Subject: [PATCH 076/148] Testing excluding docker builds if not specific ref is selected --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a3329b7ba..14a4eb042 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -87,6 +87,7 @@ jobs: buildx: runs-on: ubuntu-latest needs: [tests, documentation, frontend] + if: github.ref == 'refs/heads/github-action-test' steps: - uses: actions/checkout@v2 From 968725f09feb9bc1482c7f7115355e524bba1577 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 11:33:08 +0100 Subject: [PATCH 077/148] Separate github action into separete workflow files --- .../{docker-publish.yml => docker.yml} | 78 ------------------- .github/workflows/documentation.yml | 32 ++++++++ .github/workflows/frontend.yml | 16 ++++ .github/workflows/test.yml | 43 ++++++++++ 4 files changed, 91 insertions(+), 78 deletions(-) rename .github/workflows/{docker-publish.yml => docker.yml} (50%) create mode 100644 .github/workflows/documentation.yml create mode 100644 .github/workflows/frontend.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker.yml similarity index 50% rename from .github/workflows/docker-publish.yml rename to .github/workflows/docker.yml index 14a4eb042..338ae85a9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker.yml @@ -5,84 +5,6 @@ on: branches: travis-multiarch-builds jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - python: - - 3.6.8 - - 3.7.9 - - 3.8.6 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "${{ matrix.python }}" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python }} - - name: Tests - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - cd src/ - pipenv run pytest --cov - pipenv run pycodestyle - pipenv run coveralls - documentation: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "3.6.8" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip3.6.8} - - name: dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - - name: make - run: | - cd docs/ - make html - frontend: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '15' - - name: build - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - ng build --prod # Build and push image to docker hub. buildx: runs-on: ubuntu-latest diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 000000000..2e2e84e84 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,32 @@ +name: backend-tests + +# always run documentation workflow to ensure the documentation can still be built +on: push + +# possibly add a publish job for automatic read-the-docs publishing +jobs: + documentation: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "3.6.8" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip3.6.8} + - name: dependencies + run: | + pip install 'sphinx~=3.3' sphinx_rtd_theme + - name: make + run: | + cd docs/ + make html \ No newline at end of file diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 000000000..64df45127 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,16 @@ + + +jobs: + frontend: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: build + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + ng build --prod \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..930f368e7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: backend-tests + +# always run tests workflow +on: push + +jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + python: + - 3.6.8 + - 3.7.9 + - 3.8.6 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "${{ matrix.python }}" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python }} + - name: Tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + cd src/ + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls From 29f73b2dbbe317bfaee34ff8b2e738ee50bf9805 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 11:34:04 +0100 Subject: [PATCH 078/148] Give the frontend workflow a name --- .github/workflows/frontend.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 64df45127..ad44fb717 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -1,4 +1,7 @@ +name: frontend +# always run frontend workflow +on: push jobs: frontend: From 52640f08d2c9026cb55f05e869cd5ca0839de053 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 20:37:25 +0100 Subject: [PATCH 079/148] github action improved tagging --- .github/workflows/docker.yml | 24 +++++++++++++----------- .github/workflows/documentation.yml | 1 + .github/workflows/frontend.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 338ae85a9..715474a85 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,32 +1,34 @@ -name: Docker +name: docker on: push: - branches: travis-multiarch-builds + branches: + - master + - dev + tags: + - 'ng-*' jobs: - # Build and push image to docker hub. + # build and push image to docker hub. buildx: runs-on: ubuntu-latest - needs: [tests, documentation, frontend] - if: github.ref == 'refs/heads/github-action-test' steps: - uses: actions/checkout@v2 - - name: Prepare id: prepare + env: + DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} run: | - DOCKER_IMAGE=moztr/paperless-ng DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/v} + VERSION=${GITHUB_REF#refs/tags/} fi - if [ "${{ github.event_name }}" = "schedule" ]; then - VERSION=nightly + if [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=${GITHUB_REF#refs/heads/} fi TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + if [[ $VERSION == "master" ]]; then TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" fi echo ::set-output name=docker_image::${DOCKER_IMAGE} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 2e2e84e84..12bd3577d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -24,6 +24,7 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip3.6.8} - name: dependencies + # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? run: | pip install 'sphinx~=3.3' sphinx_rtd_theme - name: make diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index ad44fb717..66a9a9b0c 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -1,6 +1,6 @@ name: frontend -# always run frontend workflow +# always run frontend workflow to see if it builds and tests run fine on: push jobs: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 930f368e7..1a235f763 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,8 +8,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] - python: + os: ubuntu-20.04] + python: # pyenv-action requires the exact version - 3.6.8 - 3.7.9 - 3.8.6 From f4f9200fba9d30e2c3039f1c7fb81d4f903abe95 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 20:39:20 +0100 Subject: [PATCH 080/148] Fix typo in yaml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a235f763..ec601d57e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: ubuntu-20.04] + os: [ubuntu-20.04] python: # pyenv-action requires the exact version - 3.6.8 - 3.7.9 From a2b5fb8374ed2c3df1b9f2c11ba6455dc6ec6f30 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 8 Jan 2021 20:42:17 +0100 Subject: [PATCH 081/148] Fix name of documentation workflow --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 12bd3577d..67047d6cd 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,4 +1,4 @@ -name: backend-tests +name: documentation # always run documentation workflow to ensure the documentation can still be built on: push From ad241f12a3d3f681c8488bcdebe9d66062c6bf3a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 9 Jan 2021 08:06:55 +0100 Subject: [PATCH 082/148] Run tests, documentation and frontend build also on pull_request --- .github/workflows/documentation.yml | 2 +- .github/workflows/frontend.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 67047d6cd..00835d240 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,7 +1,7 @@ name: documentation # always run documentation workflow to ensure the documentation can still be built -on: push +on: [push, pull_request] # possibly add a publish job for automatic read-the-docs publishing jobs: diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 66a9a9b0c..4cb36d3a4 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -1,7 +1,7 @@ name: frontend # always run frontend workflow to see if it builds and tests run fine -on: push +on: [push, pull_request] jobs: frontend: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec601d57e..9766c4b5b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: backend-tests # always run tests workflow -on: push +on: [push, pull_request] jobs: tests: From b1ae8d1332eb9fd0bc321ae7ae0096a3db7076eb Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 15:11:58 +0100 Subject: [PATCH 083/148] enable docker builds on this branch --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 715474a85..b6f75df24 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,6 +5,7 @@ on: branches: - master - dev + - travis-multiarch-builds tags: - 'ng-*' @@ -18,8 +19,9 @@ jobs: id: prepare env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} + # ,linux/arm/v7,linux/arm64/v8 run: | - DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 + DOCKER_PLATFORMS=linux/amd64 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then VERSION=${GITHUB_REF#refs/tags/} From 327f00e3129402daca505005998981fbf0dc28d1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 17:54:56 +0100 Subject: [PATCH 084/148] now with cache? --- .github/workflows/docker.yml | 87 ++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b6f75df24..185856157 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,51 +14,80 @@ jobs: buildx: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Prepare + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: crazy-max/ghaction-docker-buildx@v3.1.0 + - + name: Cache Docker layers + uses: actions/cache@v2 + id: cache + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - + name: Prepare id: prepare env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} # ,linux/arm/v7,linux/arm64/v8 run: | DOCKER_PLATFORMS=linux/amd64 - VERSION=edge + DOCKER_VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} + DOCKER_VERSION=${GITHUB_REF#refs/tags/} fi if [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=${GITHUB_REF#refs/heads/} + DOCKER_VERSION=${GITHUB_REF#refs/heads/} fi - TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION == "master" ]]; then - TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + DOCKER_TAG="${DOCKER_IMAGE}:${DOCKER_VERSION}" + if [[ $DOCKER_VERSION == "master" ]]; then + DOCKER_TAG="${DOCKER_IMAGE}:latest" fi - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=version::${VERSION} - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ - --build-arg VERSION=${VERSION} \ + echo ::set-output name=version::${DOCKER_VERSION} + echo ::set-output name=platforms::${DOCKER_PLATFORMS} + echo ::set-output name=tag::${DOCKER_TAG} + - + name: Docker buildx (build) + run: | + docker buildx build \ + --cache-from "type=local,src=/tmp/.buildx-cache" \ + --cache-to "type=local,dest=/tmp/.buildx-cache" \ + --platform ${{steps.prepare.outputs.platforms}} \ + --output "type=image,push=false" \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ - ${TAGS} --file ./Dockerfile . - - name: install buildx - id: buildx - uses: crazy-max/ghaction-docker-buildx@v3.1.0 - - name: Docker Login - if: success() && github.event_name != 'pull_request' - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + --platform ${{steps.prepare.outputs.platforms}} \ + --tag ${{steps.prepare.outputs.tag}} ./ + + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - + name: Docker Buildx (push) run: | - echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin - - name: Docker Buildx (push) - if: success() && github.event_name != 'pull_request' - run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build \ + --cache-from "type=local,src=/tmp/.buildx-cache" \ + --platform ${{steps.prepare.outputs.platforms}} \ + --output "type=image,push=true" \ + --tag ${{steps.prepare.outputs.tag}} ./ + - name: Docker Check Manifest - if: always() && github.event_name != 'pull_request' run: | - docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} + - + name: Inspect image + run: | + docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} + - name: Clear - if: always() && github.event_name != 'pull_request' run: | rm -f ${HOME}/.docker/config.json From b48c290e9c52b633c2de6c489a73b2780dce02b6 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:02:36 +0100 Subject: [PATCH 085/148] arm --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 185856157..f858cd57e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -34,9 +34,8 @@ jobs: id: prepare env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} - # ,linux/arm/v7,linux/arm64/v8 run: | - DOCKER_PLATFORMS=linux/amd64 + DOCKER_PLATFORMS=linux/amd64,linux/arm/v7 DOCKER_VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then DOCKER_VERSION=${GITHUB_REF#refs/tags/} From 001cd83ef801f6f4cee40d78375ab9d7ae22167c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:25:56 +0100 Subject: [PATCH 086/148] new buildx setup action --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f858cd57e..0af841f24 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Docker Buildx - uses: crazy-max/ghaction-docker-buildx@v3.1.0 + uses: docker/setup-buildx-action@v1 - name: Cache Docker layers uses: actions/cache@v2 @@ -35,7 +35,7 @@ jobs: env: DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} run: | - DOCKER_PLATFORMS=linux/amd64,linux/arm/v7 + DOCKER_PLATFORMS=linux/amd64 DOCKER_VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then DOCKER_VERSION=${GITHUB_REF#refs/tags/} From 5822273c8d4defb8a54f8fe2fb5be4ec22b9253b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:33:58 +0100 Subject: [PATCH 087/148] duplicate platform specification --- .github/workflows/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0af841f24..8cf13cc37 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -56,7 +56,6 @@ jobs: docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ --cache-to "type=local,dest=/tmp/.buildx-cache" \ - --platform ${{steps.prepare.outputs.platforms}} \ --output "type=image,push=false" \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ From 7c7065921216e31d5d41e36a35545b3668edae89 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 18:49:21 +0100 Subject: [PATCH 088/148] disable some steps --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8cf13cc37..eafb80592 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -71,6 +71,8 @@ jobs: - name: Docker Buildx (push) + + if: false run: | docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ @@ -79,10 +81,12 @@ jobs: --tag ${{steps.prepare.outputs.tag}} ./ - name: Docker Check Manifest + if: false run: | docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} - name: Inspect image + if: false run: | docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} From 9c2e55407f9762192c75fcd29ea254d83040a766 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <17569239+jonaswinkler@users.noreply.github.com> Date: Sat, 9 Jan 2021 19:02:07 +0100 Subject: [PATCH 089/148] Update docker.yml --- .github/workflows/docker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eafb80592..9083af82f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -71,8 +71,6 @@ jobs: - name: Docker Buildx (push) - - if: false run: | docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ From d6ad41ab3a3506e107118c28061e13b0e1db57a7 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <17569239+jonaswinkler@users.noreply.github.com> Date: Sat, 9 Jan 2021 19:06:00 +0100 Subject: [PATCH 090/148] Update docker.yml --- .github/workflows/docker.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9083af82f..d0c5a97b0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -68,7 +68,6 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Docker Buildx (push) run: | @@ -77,17 +76,11 @@ jobs: --platform ${{steps.prepare.outputs.platforms}} \ --output "type=image,push=true" \ --tag ${{steps.prepare.outputs.tag}} ./ - - - name: Docker Check Manifest - if: false + - + name: Docker Check Manifest run: | docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} - name: Inspect image - if: false run: | docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} - - - name: Clear - run: | - rm -f ${HOME}/.docker/config.json From ebc5f1a01d243cb0d185e9d8fe363ba930f2c756 Mon Sep 17 00:00:00 2001 From: Jonas Winkler <17569239+jonaswinkler@users.noreply.github.com> Date: Sat, 9 Jan 2021 19:10:09 +0100 Subject: [PATCH 091/148] Update docker.yml --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d0c5a97b0..56a6d1b39 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,9 +43,9 @@ jobs: if [[ $GITHUB_REF == refs/heads/* ]]; then DOCKER_VERSION=${GITHUB_REF#refs/heads/} fi - DOCKER_TAG="${DOCKER_IMAGE}:${DOCKER_VERSION}" + DOCKER_TAG="jonaswinkler/paperless-ng-test:${DOCKER_VERSION}" if [[ $DOCKER_VERSION == "master" ]]; then - DOCKER_TAG="${DOCKER_IMAGE}:latest" + DOCKER_TAG="jonaswinkler/paperless-ng-test:latest" fi echo ::set-output name=version::${DOCKER_VERSION} echo ::set-output name=platforms::${DOCKER_PLATFORMS} From d7095f7ee1c2d40e0a8bb2c03236c878de02f4f7 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 19:31:41 +0100 Subject: [PATCH 092/148] tags? --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 56a6d1b39..585ec61e4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -60,7 +60,7 @@ jobs: --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ --platform ${{steps.prepare.outputs.platforms}} \ - --tag ${{steps.prepare.outputs.tag}} ./ + --tag jonaswinkler/paperless-ng:edge ./ - name: Login to DockerHub @@ -75,8 +75,8 @@ jobs: --cache-from "type=local,src=/tmp/.buildx-cache" \ --platform ${{steps.prepare.outputs.platforms}} \ --output "type=image,push=true" \ - --tag ${{steps.prepare.outputs.tag}} ./ - - + --tag jonaswinkler/paperless-ng:edge ./ + - name: Docker Check Manifest run: | docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} From 4363d4c9ed5d3451f41510b9cf5b77ee3d8f5c8d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 19:50:11 +0100 Subject: [PATCH 093/148] simplify some things --- .github/workflows/docker.yml | 61 +++++++++--------------------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 585ec61e4..cda034085 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,10 @@ on: tags: - 'ng-*' + +# --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ +# --build-arg VCS_REF=${GITHUB_SHA::8} \ + jobs: # build and push image to docker hub. buildx: @@ -23,45 +27,11 @@ jobs: - name: Cache Docker layers uses: actions/cache@v2 - id: cache with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - - - name: Prepare - id: prepare - env: - DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} - run: | - DOCKER_PLATFORMS=linux/amd64 - DOCKER_VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/tags/} - fi - if [[ $GITHUB_REF == refs/heads/* ]]; then - DOCKER_VERSION=${GITHUB_REF#refs/heads/} - fi - DOCKER_TAG="jonaswinkler/paperless-ng-test:${DOCKER_VERSION}" - if [[ $DOCKER_VERSION == "master" ]]; then - DOCKER_TAG="jonaswinkler/paperless-ng-test:latest" - fi - echo ::set-output name=version::${DOCKER_VERSION} - echo ::set-output name=platforms::${DOCKER_PLATFORMS} - echo ::set-output name=tag::${DOCKER_TAG} - - - name: Docker buildx (build) - run: | - docker buildx build \ - --cache-from "type=local,src=/tmp/.buildx-cache" \ - --cache-to "type=local,dest=/tmp/.buildx-cache" \ - --output "type=image,push=false" \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg VCS_REF=${GITHUB_SHA::8} \ - --platform ${{steps.prepare.outputs.platforms}} \ - --tag jonaswinkler/paperless-ng:edge ./ - - name: Login to DockerHub uses: docker/login-action@v1 @@ -69,18 +39,17 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Docker Buildx (push) - run: | - docker buildx build \ - --cache-from "type=local,src=/tmp/.buildx-cache" \ - --platform ${{steps.prepare.outputs.platforms}} \ - --output "type=image,push=true" \ - --tag jonaswinkler/paperless-ng:edge ./ - - - name: Docker Check Manifest - run: | - docker run --rm mplatform/mquery ${{steps.prepare.outputs.tag}} + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64 + push: true + tags: jonaswinkler/paperless-ng:edge + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache - name: Inspect image run: | - docker buildx imagetools inspect ${{steps.prepare.outputs.tag}} + docker buildx imagetools inspect jonaswinkler/paperless-ng:edge From c0915a3deeb84aa5505dcdf18c157485ed605179 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 20:06:53 +0100 Subject: [PATCH 094/148] add arm --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cda034085..1fe772e7b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,6 +24,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 - name: Cache Docker layers uses: actions/cache@v2 @@ -44,7 +47,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64 + platforms: linux/amd64,linux/arm/v7 push: true tags: jonaswinkler/paperless-ng:edge cache-from: type=local,src=/tmp/.buildx-cache From 0051d9f76eb472c79ea914a5d2f59330721b21da Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 20:29:29 +0100 Subject: [PATCH 095/148] undo removing a required library --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5e6b2690e..0ccfcd84e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,7 +63,7 @@ RUN apt-get update \ && pipenv install --system --deploy --ignore-pipfile \ && pipenv --clear \ && pip3 uninstall -y pipenv \ - && apt-get -y purge build-essential libqpdf-dev libxslt1-dev \ + && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord From 517e03772401d769c5a978aa45f0b126c103b625 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 20:51:46 +0100 Subject: [PATCH 096/148] always update cache --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1fe772e7b..be271a3b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,7 +29,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Cache Docker layers - uses: actions/cache@v2 + uses: pat-s/always-upload-cache@v2 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} From d2ec38d5fd1875bf03bd7d8615a219df48f049b2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:21:50 +0100 Subject: [PATCH 097/148] use correct version --- .github/workflows/docker.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index be271a3b7..4afc1027f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,6 +18,18 @@ jobs: buildx: runs-on: ubuntu-latest steps: + - + name: Prepare + id: prepare + run: | + VERSION=edge + if [[ $GITHUB_REF == refs/tags/ng-* ]]; then + VERSION=${GITHUB_REF#refs/tags/ng-} + fi + if [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=${GITHUB_REF#refs/heads/} + fi + echo ::set-output name=version::${VERSION} - name: Checkout uses: actions/checkout@v2 @@ -49,10 +61,10 @@ jobs: file: ./Dockerfile platforms: linux/amd64,linux/arm/v7 push: true - tags: jonaswinkler/paperless-ng:edge + tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - name: Inspect image run: | - docker buildx imagetools inspect jonaswinkler/paperless-ng:edge + docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} From 2c4bbf8a005521e13e85d118141dea21a86bbd00 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:23:04 +0100 Subject: [PATCH 098/148] use correct version of cache action --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4afc1027f..e7b426a07 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,7 +41,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2 + uses: pat-s/always-upload-cache@v2.1.3 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} From a4bff61e5b714dfe2be93bea3ef2f4f04f161901 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:49:43 +0100 Subject: [PATCH 099/148] move everything into a single file so that the docker image build can depend on the tests being successful --- .github/workflows/ci.yml | 140 ++++++++++++++++++++++++++++ .github/workflows/docker.yml | 70 -------------- .github/workflows/documentation.yml | 33 ------- .github/workflows/frontend.yml | 19 ---- .github/workflows/test.yml | 43 --------- 5 files changed, 140 insertions(+), 165 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/documentation.yml delete mode 100644 .github/workflows/frontend.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..3ccfedd43 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,140 @@ +name: docker + +on: [push, pull_request] + +jobs: + documentation: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "3.6.8" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip3.6.8} + - name: dependencies + # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? + run: | + pip install 'sphinx~=3.3' sphinx_rtd_theme + - name: make + run: | + cd docs/ + make html + + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-20.04 ] + python: # pyenv-action requires the exact version + - 3.6.8 + - 3.7.9 + - 3.8.6 + steps: + - uses: actions/checkout@v2 + - name: Install python version + uses: gabrielfalcao/pyenv-action@v7 + with: + default: "${{ matrix.python }}" + command: "pip install -U pip" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python }} + - name: Tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev + cd src/ + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls + + frontend: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '15' + - name: build + run: | + cd src-ui/ + npm install -g @angular/cli + npm install + ng build --prod + + # build and push image to docker hub. + build-docker-image: + if: github.event_name == push && (github.ref == refs/heads/master || github.ref == refs/heads/dev || github.ref == refs/tags/ng-*) + runs-on: ubuntu-latest + needs: tests + steps: + - + name: Prepare + id: prepare + run: | + VERSION=edge + if [[ $GITHUB_REF == refs/tags/ng-* ]]; then + VERSION=${GITHUB_REF#refs/tags/ng-} + fi + if [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=${GITHUB_REF#refs/heads/} + fi + echo ::set-output name=version::${VERSION} + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Cache Docker layers + uses: pat-s/always-upload-cache@v2.1.3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm/v7 + push: true + tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + - + name: Inspect image + run: | + docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index e7b426a07..000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: docker - -on: - push: - branches: - - master - - dev - - travis-multiarch-builds - tags: - - 'ng-*' - - -# --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ -# --build-arg VCS_REF=${GITHUB_SHA::8} \ - -jobs: - # build and push image to docker hub. - buildx: - runs-on: ubuntu-latest - steps: - - - name: Prepare - id: prepare - run: | - VERSION=edge - if [[ $GITHUB_REF == refs/tags/ng-* ]]; then - VERSION=${GITHUB_REF#refs/tags/ng-} - fi - if [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=${GITHUB_REF#refs/heads/} - fi - echo ::set-output name=version::${VERSION} - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2.1.3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7 - push: true - tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Inspect image - run: | - docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml deleted file mode 100644 index 00835d240..000000000 --- a/.github/workflows/documentation.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: documentation - -# always run documentation workflow to ensure the documentation can still be built -on: [push, pull_request] - -# possibly add a publish job for automatic read-the-docs publishing -jobs: - documentation: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "3.6.8" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip3.6.8} - - name: dependencies - # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? - run: | - pip install 'sphinx~=3.3' sphinx_rtd_theme - - name: make - run: | - cd docs/ - make html \ No newline at end of file diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml deleted file mode 100644 index 4cb36d3a4..000000000 --- a/.github/workflows/frontend.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: frontend - -# always run frontend workflow to see if it builds and tests run fine -on: [push, pull_request] - -jobs: - frontend: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: '15' - - name: build - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - ng build --prod \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 9766c4b5b..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: backend-tests - -# always run tests workflow -on: [push, pull_request] - -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - python: # pyenv-action requires the exact version - - 3.6.8 - - 3.7.9 - - 3.8.6 - steps: - - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 - with: - default: "${{ matrix.python }}" - command: "pip install -U pip" - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python }} - - name: Tests - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - pip install --upgrade pipenv - pipenv install --system --dev - cd src/ - pipenv run pytest --cov - pipenv run pycodestyle - pipenv run coveralls From fb5d35e43eb261a32f14ce76198fd672b849201a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:50:53 +0100 Subject: [PATCH 100/148] fix condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ccfedd43..492c977bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == push && (github.ref == refs/heads/master || github.ref == refs/heads/dev || github.ref == refs/tags/ng-*) + if: github.event_name == "push" && (github.ref == "refs/heads/master" || github.ref == "refs/heads/dev" || github.ref == "refs/tags/ng-*") runs-on: ubuntu-latest needs: tests steps: From 28c3f8d61482678b56bfd512ea264d237ac2ef5b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:53:13 +0100 Subject: [PATCH 101/148] fix condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 492c977bf..b841dee8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == "push" && (github.ref == "refs/heads/master" || github.ref == "refs/heads/dev" || github.ref == "refs/tags/ng-*") + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest needs: tests steps: From 692a5081c7b572164554d54e2566886d1bf2f215 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 21:55:03 +0100 Subject: [PATCH 102/148] rename workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b841dee8d..dcef91911 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: docker +name: ci on: [push, pull_request] From 47339c84b353f6e9be98a9b0715e8d9798f1fcc8 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:03:38 +0100 Subject: [PATCH 103/148] test include this branch --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcef91911..3fe9a6ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') + if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest needs: tests steps: From 6fa0d00b8ff5a0c029c025123c6ed832cfeebfec Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:33:07 +0100 Subject: [PATCH 104/148] try separate build / push --- .github/workflows/ci.yml | 45 +++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe9a6ad3..78dd931d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,9 +84,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest - needs: tests steps: - name: Prepare @@ -118,19 +116,10 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push + name: Build image uses: docker/build-push-action@v2 with: - context: . - file: ./Dockerfile platforms: linux/amd64,linux/arm/v7 - push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache @@ -138,3 +127,35 @@ jobs: name: Inspect image run: | docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} + + push-docker-image: + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') + needs: [tests, build-docker-image] + + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Cache Docker layers + uses: pat-s/always-upload-cache@v2.1.3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Push image + uses: docker/build-push-action@v2 + with: + platforms: linux/amd64,linux/arm/v7 + tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} + cache-from: type=local,src=/tmp/.buildx-cache + push: true From 141315cf427ae21fb6bc9e1479bda1c74bd8ba1c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:38:05 +0100 Subject: [PATCH 105/148] Revert "try separate build / push" This reverts commit 6fa0d00b --- .github/workflows/ci.yml | 45 +++++++++++----------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78dd931d0..3fe9a6ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,9 @@ jobs: # build and push image to docker hub. build-docker-image: + if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest + needs: tests steps: - name: Prepare @@ -116,10 +118,19 @@ jobs: restore-keys: | ${{ runner.os }}-buildx- - - name: Build image + name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Build and push uses: docker/build-push-action@v2 with: + context: . + file: ./Dockerfile platforms: linux/amd64,linux/arm/v7 + push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache @@ -127,35 +138,3 @@ jobs: name: Inspect image run: | docker buildx imagetools inspect jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} - - push-docker-image: - runs-on: ubuntu-latest - if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') - needs: [tests, build-docker-image] - - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2.1.3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Push image - uses: docker/build-push-action@v2 - with: - platforms: linux/amd64,linux/arm/v7 - tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} - cache-from: type=local,src=/tmp/.buildx-cache - push: true From 262e8ee9996674e32bb3c8b8dfafaaac8adfe29e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 9 Jan 2021 22:53:45 +0100 Subject: [PATCH 106/148] add arm64 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe9a6ad3..78e1eec42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7 + platforms: linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache From ecc40d022dce904e5f58e04c76cd499751aa980a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:09:10 +0100 Subject: [PATCH 107/148] test updated Pipfile --- Pipfile | 3 --- Pipfile.lock | 15 +++++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Pipfile b/Pipfile index 0c182d5b6..1a9b43950 100644 --- a/Pipfile +++ b/Pipfile @@ -8,9 +8,6 @@ url = "https://www.piwheels.org/simple" verify_ssl = true name = "piwheels" -[requires] -python_version = "3.6" - [packages] dateparser = "~=0.7.6" django = "~=3.1.3" diff --git a/Pipfile.lock b/Pipfile.lock index b36e5e496..58cf35894 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,12 +1,10 @@ { "_meta": { "hash": { - "sha256": "c35d84fd7f4f1c7d599039712362935e7c41a226b0ab3d83d8c1c2fb2ad0962a" + "sha256": "241c8d042e9b607bbeb7d501b3fc1b57376e3cc8c2b2fe117ec60e425dfd79c7" }, "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, + "requires": {}, "sources": [ { "name": "pypi", @@ -1375,6 +1373,15 @@ "index": "pypi", "version": "==3.20.1" }, + "typing-extensions": { + "hashes": [ + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" + ], + "markers": "python_version < '3.8'", + "version": "==3.7.4.3" + }, "urllib3": { "hashes": [ "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", From e606ae5097b2a5ad4bd81f0758f7821804d6cce3 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:28:33 +0100 Subject: [PATCH 108/148] try new python setup action --- .github/workflows/ci.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78e1eec42..7af141cd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,17 +35,13 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python: # pyenv-action requires the exact version - - 3.6.8 - - 3.7.9 - - 3.8.6 + python-version: ['3.6', '3.7', '3.8'] steps: - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 + - name: Set up Python + uses: actions/setup-python@v2 with: - default: "${{ matrix.python }}" - command: "pip install -U pip" + default: "${{ matrix.python-version }}" - name: Get pip cache dir id: pip-cache run: | From bf634eee6646103ebe174510b59e8d6746532711 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:32:47 +0100 Subject: [PATCH 109/148] fix python action --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7af141cd2..a9bbda8ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,17 +31,16 @@ jobs: make html tests: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-20.04 strategy: matrix: - os: [ ubuntu-20.04 ] python-version: ['3.6', '3.7', '3.8'] steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: - default: "${{ matrix.python-version }}" + python-version: "${{ matrix.python-version }}" - name: Get pip cache dir id: pip-cache run: | From 06ee41226dbf3176d33555381dc131a99d593233 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:41:11 +0100 Subject: [PATCH 110/148] no need to run commands in pipenv when using --system --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9bbda8ad..9d398e6d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,11 +57,11 @@ jobs: sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv - pipenv install --system --dev + pipenv install --system --dev --ignore-pipfile cd src/ - pipenv run pytest --cov - pipenv run pycodestyle - pipenv run coveralls + pytest + pycodestyle + coveralls frontend: runs-on: ubuntu-20.04 From c9ce69c1c1ea67575b6e3c8b26a8738a8e4f0142 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:50:06 +0100 Subject: [PATCH 111/148] documentation action --- Dockerfile | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0ccfcd84e..ea3136bb3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS jbig2 +FROM ubuntu:20.04 AS jbig2enc WORKDIR /usr/src/jbig2enc @@ -19,7 +19,7 @@ RUN npm install COPY src-ui . RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false -FROM python:3.7-slim +FROM ubuntu:20.04 WORKDIR /usr/src/paperless/ @@ -27,7 +27,7 @@ COPY Pipfile* ./ #Dependencies RUN apt-get update \ - && apt-get -y --no-install-recommends install \ + && DEBIAN_FRONTEND="noninteractive" apt-get -y --no-install-recommends install \ build-essential \ curl \ file \ @@ -48,6 +48,9 @@ RUN apt-get update \ mime-support \ optipng \ pngquant \ + python3 \ + python3-pip \ + python3-dev \ qpdf \ sudo \ tesseract-ocr \ @@ -59,11 +62,12 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor pipenv \ + + && pip3 install --upgrade supervisor pipenv setuptools \ && pipenv install --system --deploy --ignore-pipfile \ && pipenv --clear \ && pip3 uninstall -y pipenv \ - && apt-get -y purge build-essential libqpdf-dev \ + && apt-get -y purge build-essential libqpdf-dev python3-dev python3-pip \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord @@ -75,10 +79,10 @@ COPY docker/gunicorn.conf.py ./ COPY docker/supervisord.conf /etc/supervisord.conf COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh -# copy jbic -COPY --from=jbig2 /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ -COPY --from=jbig2 /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ -COPY --from=jbig2 /usr/src/jbig2enc/src/*.h /usr/local/include/ +# copy jbig2enc +COPY --from=jbig2enc /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ +COPY --from=jbig2enc /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ +COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/ # copy app From aa1fb63714c9cd472a1b6f7dd57634e6e38f7f24 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:53:30 +0100 Subject: [PATCH 112/148] documentation action --- .github/workflows/ci.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d398e6d7..98c61efb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,11 +7,10 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - name: Install python version - uses: gabrielfalcao/pyenv-action@v7 + - name: Set up Python + uses: actions/setup-python@v2 with: - default: "3.6.8" - command: "pip install -U pip" + python-version: 3.8 - name: Get pip cache dir id: pip-cache run: | @@ -21,11 +20,9 @@ jobs: with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip3.6.8} - - name: dependencies - # instead of directly installing the dependency in the workflow file, can this be done using the pipfile instead? - run: | - pip install 'sphinx~=3.3' sphinx_rtd_theme - - name: make + - name: Install dependencies + run: pipenv install --system --dev --ignore-pipfile + - name: Make documentation run: | cd docs/ make html From 6a355b8dff6a6fbf358e4ab54fd7fe6d2db42269 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:54:48 +0100 Subject: [PATCH 113/148] install pipenv --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98c61efb4..54ff75670 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,9 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip3.6.8} - name: Install dependencies - run: pipenv install --system --dev --ignore-pipfile + run: | + pip install --upgrade pipenv + pipenv install --system --dev --ignore-pipfile - name: Make documentation run: | cd docs/ From 4bb0da397a52eb89e5dda8438909accbc6ede0ba Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 00:58:14 +0100 Subject: [PATCH 114/148] dependencies --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ff75670..98e5ecf8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: key: ${{ runner.os }}-pip3.6.8} - name: Install dependencies run: | + sudo apt-get update -qq + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev pip install --upgrade pipenv pipenv install --system --dev --ignore-pipfile - name: Make documentation @@ -54,7 +56,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng pip install --upgrade pipenv pipenv install --system --dev --ignore-pipfile cd src/ From a1a415a5d16b532ccb67486fe3a8ca704d70be22 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 11:26:14 +0100 Subject: [PATCH 115/148] fix up pipfile for arm/v7 --- .github/workflows/ci.yml | 2 +- Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98e5ecf8a..aa4ee9b74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,7 +107,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Cache Docker layers - uses: pat-s/always-upload-cache@v2.1.3 + uses: actions/cache@v2 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} diff --git a/Dockerfile b/Dockerfile index ea3136bb3..4bd9b5e06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ RUN apt-get update \ icc-profiles-free \ imagemagick \ libatlas-base-dev \ + libffi-dev \ liblept5 \ libmagic-dev \ libpoppler-cpp-dev \ From 74db555f78b31b34f8ae482781f8e786f2d6e934 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 11:39:30 +0100 Subject: [PATCH 116/148] fix cryphtography for armv7 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 4bd9b5e06..16c52caa9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,7 @@ RUN apt-get update \ libpoppler-cpp-dev \ libpq-dev \ libqpdf-dev \ + libssl-dev \ libxml2 \ libxslt1-dev \ mime-support \ From def85a3d37cafc1df75d5d4e90597eace8f6b1cb Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 15:51:40 +0100 Subject: [PATCH 117/148] maybe use requirements instead? --- .github/workflows/ci.yml | 2 +- Dockerfile | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa4ee9b74..523f74e61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/arm/v7 # linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache diff --git a/Dockerfile b/Dockerfile index 16c52caa9..d124ddc09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,8 +65,9 @@ RUN apt-get update \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor pipenv setuptools \ - && pipenv install --system --deploy --ignore-pipfile \ + && pip3 install --upgrade supervisor pipenv setuptools \ + && pipenv lock -r > requirements.txt \ + && pip3 install -r requirements.txt \ && pipenv --clear \ && pip3 uninstall -y pipenv \ && apt-get -y purge build-essential libqpdf-dev python3-dev python3-pip \ From 22fc16f4a3fc3a5811e201b1fad16dc22fad20c5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 17:18:35 +0100 Subject: [PATCH 118/148] revert to python:3.7-slim base image since ubuntu:20.04 is clearly not working for arm --- Dockerfile | 21 ++++---------- requirements.txt | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index d124ddc09..ee19e0cc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,15 +19,15 @@ RUN npm install COPY src-ui . RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false -FROM ubuntu:20.04 +FROM python:3.7-slim WORKDIR /usr/src/paperless/ -COPY Pipfile* ./ +COPY requirements.txt ./ #Dependencies RUN apt-get update \ - && DEBIAN_FRONTEND="noninteractive" apt-get -y --no-install-recommends install \ + && apt-get -y --no-install-recommends install \ build-essential \ curl \ file \ @@ -38,21 +38,16 @@ RUN apt-get update \ icc-profiles-free \ imagemagick \ libatlas-base-dev \ - libffi-dev \ liblept5 \ libmagic-dev \ libpoppler-cpp-dev \ libpq-dev \ libqpdf-dev \ - libssl-dev \ libxml2 \ libxslt1-dev \ mime-support \ optipng \ pngquant \ - python3 \ - python3-pip \ - python3-dev \ qpdf \ sudo \ tesseract-ocr \ @@ -64,13 +59,9 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - - && pip3 install --upgrade supervisor pipenv setuptools \ - && pipenv lock -r > requirements.txt \ - && pip3 install -r requirements.txt \ - && pipenv --clear \ - && pip3 uninstall -y pipenv \ - && apt-get -y purge build-essential libqpdf-dev python3-dev python3-pip \ + && pip3 install --upgrade supervisor \ + && python3 -m pip install -r requirements.txt \ + && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..9932d146b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,74 @@ +# +# These requirements were autogenerated by pipenv +# To regenerate from the project's Pipfile, run: +# +# pipenv lock --requirements +# + +-i https://pypi.python.org/simple +--extra-index-url https://www.piwheels.org/simple +arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +asgiref==3.3.1; python_version >= '3.5' +blessed==1.17.12 +certifi==2020.12.5 +cffi==1.14.4 +chardet==4.0.0; python_version >= '3.1' +coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +dateparser==0.7.6 +django-cors-headers==3.6.0 +django-extensions==3.1.0 +django-filter==2.4.0 +django-picklefield==3.0.1; python_version >= '3' +django-q==1.3.4 +django==3.1.5 +djangorestframework==3.12.2 +filelock==3.0.12 +fuzzywuzzy==0.18.0 +gunicorn==20.0.4 +humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +imap-tools==0.34.0 +img2pdf==0.4.0 +importlib-metadata==3.3.0; python_version < '3.8' +inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +inotifyrecursive==0.3.5 +joblib==1.0.0; python_version >= '3.6' +langdetect==1.0.8 +lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +numpy==1.19.5; python_version >= '3.6' +ocrmypdf==11.4.5 +pathvalidate==2.3.2 +pdfminer.six==20201018; python_version >= '3.4' +pdftotext==2.1.5 +pikepdf==2.2.5 +pillow==8.1.0 +pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +psycopg2-binary==2.8.6 +pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +python-dateutil==2.8.1 +python-dotenv==0.15.0 +python-gnupg==0.4.6 +python-levenshtein==0.12.0 +python-magic==0.4.18 +pytz==2020.5 +redis==3.5.3 +regex==2020.11.13 +reportlab==3.5.59 +requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +scikit-learn==0.23.2 +scipy==1.5.4; python_version >= '3.6' +six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +sortedcontainers==2.3.0 +sqlparse==0.4.1; python_version >= '3.5' +threadpoolctl==2.1.0; python_version >= '3.5' +tika==1.24 +tqdm==4.55.1 +typing-extensions==3.7.4.3; python_version < '3.8' +tzlocal==2.1 +urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' +watchdog==1.0.2 +wcwidth==0.2.5 +whitenoise==5.2.0 +whoosh==2.7.4 +zipp==3.4.0; python_version >= '3.6' From f6da60815407e4865a22eedcfdab67a595345c85 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 17:19:00 +0100 Subject: [PATCH 119/148] disable tests until this is working --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 523f74e61..cf2c8b137 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest - needs: tests + # needs: tests steps: - name: Prepare From 8430aa75da3f2db91ee8aebd52533447b806d537 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 17:46:43 +0100 Subject: [PATCH 120/148] enable the other architectures --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf2c8b137..e2918d276 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/arm/v7 # linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache From 1e185d15026f285e403525822120170170a922a4 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 19:59:05 +0100 Subject: [PATCH 121/148] don't build front end with docker --- .github/workflows/ci.yml | 18 ++++++++++++------ Dockerfile | 39 ++++++++++++++++----------------------- compile-frontend.sh | 7 +++++++ src-ui/angular.json | 1 + 4 files changed, 36 insertions(+), 29 deletions(-) create mode 100755 compile-frontend.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2918d276..f0df7e608 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,17 +72,18 @@ jobs: with: node-version: '15' - name: build - run: | - cd src-ui/ - npm install -g @angular/cli - npm install - ng build --prod + run: ./build-frontend.sh + - uses: actions/upload-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ # build and push image to docker hub. build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest # needs: tests + needs: frontend steps: - name: Prepare @@ -99,6 +100,11 @@ jobs: - name: Checkout uses: actions/checkout@v2 + - + uses: actions/download-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -125,7 +131,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64,linux/arm/v7,linux/arm64 + platforms: linux/amd64 # ,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache diff --git a/Dockerfile b/Dockerfile index ee19e0cc0..4a52568ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,27 +8,15 @@ RUN git clone https://github.com/agl/jbig2enc . RUN ./autogen.sh RUN ./configure && make - -FROM node:15 AS frontend - -WORKDIR /usr/src/paperless/src-ui/ - -COPY src-ui/package* ./ -RUN npm install - -COPY src-ui . -RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false - FROM python:3.7-slim WORKDIR /usr/src/paperless/ COPY requirements.txt ./ -#Dependencies +# Binary dependencies RUN apt-get update \ && apt-get -y --no-install-recommends install \ - build-essential \ curl \ file \ fonts-liberation \ @@ -37,13 +25,6 @@ RUN apt-get update \ gnupg \ icc-profiles-free \ imagemagick \ - libatlas-base-dev \ - liblept5 \ - libmagic-dev \ - libpoppler-cpp-dev \ - libpq-dev \ - libqpdf-dev \ - libxml2 \ libxslt1-dev \ mime-support \ optipng \ @@ -59,8 +40,21 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor \ - && python3 -m pip install -r requirements.txt \ + && rm -rf /var/lib/apt/lists/* + +# Python dependencies +RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + libatlas-base-dev \ + liblept5 \ + libmagic-dev \ + libpoppler-cpp-dev \ + libpq-dev \ + libqpdf-dev \ + libxml2 \ + && python3 -m pip install --upgrade --no-cache-dir supervisor \ + && python3 -m pip install --no-cache-dir -r requirements.txt \ && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ @@ -81,7 +75,6 @@ COPY --from=jbig2enc /usr/src/jbig2enc/src/*.h /usr/local/include/ # copy app COPY src/ ./src/ -COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/compile-frontend.sh b/compile-frontend.sh new file mode 100755 index 000000000..98b88d033 --- /dev/null +++ b/compile-frontend.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +cd src-ui +npm install +./node_modules/.bin/ng build --prod diff --git a/src-ui/angular.json b/src-ui/angular.json index 414cd4f64..829a482af 100644 --- a/src-ui/angular.json +++ b/src-ui/angular.json @@ -58,6 +58,7 @@ "with": "src/environments/environment.prod.ts" } ], + "outputPath": "../src/documents/static/frontend/", "optimization": true, "outputHashing": "none", "sourceMap": false, From ff3bdd7f5abad3f998e0deab4b5b8558bb083a66 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 20:00:11 +0100 Subject: [PATCH 122/148] fix script name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0df7e608..ce5117256 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: with: node-version: '15' - name: build - run: ./build-frontend.sh + run: ./compile-frontend.sh - uses: actions/upload-artifact@v2 with: name: frontend-compiled From 6a3e4f0857177e8ba9f4af24fbf9fafbcf67473d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:03:50 +0100 Subject: [PATCH 123/148] add release archive action --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce5117256..eea3b0ba0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,10 @@ jobs: run: | cd docs/ make html + - uses: actions/upload-artifact@v2 + with: + name: documentation + path: docs/_build/html/ tests: runs-on: ubuntu-20.04 @@ -78,6 +82,39 @@ jobs: name: frontend-compiled path: src/documents/static/frontend/ + release: + needs: [frontend, documentation] # tests + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - + uses: actions/download-artifact@v2 + with: + name: frontend-compiled + path: src/documents/static/frontend/ + - + uses: actions/download-artifact@v2 + with: + name: documentation + path: src/documents/static/frontend/ + - + run: | + mkdir dist + mkdir dist/paperless-ng + mkdir dist/paperless-ng/docker + mkdir dist/paperless-ng/scripts + cp .dockerignore .env docker-compose.* Dockerfile paperless.conf.example Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/ + cp docker/* dist/paperless-ng/docker/ + cp scripts/* dist/paperless-ng/scripts/ + cp src/ dist/paperless-ng/src -R + cd dist + tar -cJf paperless-ng.tar.xz paperless-ng/ + - uses: actions/upload-artifact@v2 + with: + name: release + path: dist/paperless-ng/paperless-ng.tar.xz + + # build and push image to docker hub. build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') From 2e2e69f07c7bc95e7904f61dfc6719062933e6f6 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:17:01 +0100 Subject: [PATCH 124/148] more release archive --- .github/workflows/ci.yml | 42 ++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eea3b0ba0..2d0f98c3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,19 @@ jobs: needs: [frontend, documentation] # tests runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - + uses: actions/checkout@v2 + - + name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - + name: Install dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext + pip3 install -r requirements.txt - uses: actions/download-artifact@v2 with: @@ -96,20 +108,34 @@ jobs: uses: actions/download-artifact@v2 with: name: documentation - path: src/documents/static/frontend/ + path: docs/_build/html/ - + name: Move files run: | mkdir dist mkdir dist/paperless-ng mkdir dist/paperless-ng/docker mkdir dist/paperless-ng/scripts cp .dockerignore .env docker-compose.* Dockerfile paperless.conf.example Pipfile Pipfile.lock LICENSE README.md requirements.txt dist/paperless-ng/ - cp docker/* dist/paperless-ng/docker/ - cp scripts/* dist/paperless-ng/scripts/ - cp src/ dist/paperless-ng/src -R - cd dist - tar -cJf paperless-ng.tar.xz paperless-ng/ - - uses: actions/upload-artifact@v2 + cp docker/*.* dist/paperless-ng/docker/ + cp scripts/*.service scripts/*.sh dist/paperless-ng/scripts/ + cp src/ dist/paperless-ng/src -r + cp docs/_build/html/ dist/paperless-ng/docs -r + - + name: Compile messages + run: | + cd dist/paperless-ng/src + python3 manage.py compilemessages + - + name: Collect static files + run: | + cd dist/paperless-ng/src + python3 manage.py collectstatic --no-input + - + name: Make release package + run: tar -cJf paperless-ng.tar.xz paperless-ng/ + - + uses: actions/upload-artifact@v2 with: name: release path: dist/paperless-ng/paperless-ng.tar.xz From e1f52d706f43a06c66da5c2fbd807d3fa1de39e8 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:20:55 +0100 Subject: [PATCH 125/148] fix wrong directory --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d0f98c3b..5acae09f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,9 @@ jobs: python3 manage.py collectstatic --no-input - name: Make release package - run: tar -cJf paperless-ng.tar.xz paperless-ng/ + run: | + cd dist + tar -cJf paperless-ng.tar.xz paperless-ng/ - uses: actions/upload-artifact@v2 with: From 527b7bee3fffdd026b9f679e2fd72f9be31f28b9 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:27:41 +0100 Subject: [PATCH 126/148] forgot a dependency --- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5acae09f2..e965d50c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: name: Install dependencies run: | sudo apt-get update -qq - sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext + sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5 pip3 install -r requirements.txt - uses: actions/download-artifact@v2 diff --git a/Dockerfile b/Dockerfile index 4a52568ea..7e921fcda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ RUN apt-get update \ gnupg \ icc-profiles-free \ imagemagick \ + liblept5 \ libxslt1-dev \ mime-support \ optipng \ @@ -47,7 +48,6 @@ RUN apt-get update \ && apt-get -y --no-install-recommends install \ build-essential \ libatlas-base-dev \ - liblept5 \ libmagic-dev \ libpoppler-cpp-dev \ libpq-dev \ From 64d6e6eded98f33c01cdcf6d2e15c7c83f49ccd5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:35:43 +0100 Subject: [PATCH 127/148] wrong path to release --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e965d50c6..84949dec0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,7 @@ jobs: uses: actions/upload-artifact@v2 with: name: release - path: dist/paperless-ng/paperless-ng.tar.xz + path: dist/paperless-ng.tar.xz # build and push image to docker hub. From f24640dec2cd133c5fce504e52ef6f772d6f25e1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 21:53:18 +0100 Subject: [PATCH 128/148] some small fixes --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84949dec0..4379f8abb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: path: src/documents/static/frontend/ release: - needs: [frontend, documentation] # tests + needs: [frontend, documentation, tests] runs-on: ubuntu-20.04 steps: - @@ -135,6 +135,7 @@ jobs: name: Make release package run: | cd dist + find . -name __pycache__ | xargs rm -r tar -cJf paperless-ng.tar.xz paperless-ng/ - uses: actions/upload-artifact@v2 @@ -147,8 +148,7 @@ jobs: build-docker-image: if: github.event_name == 'push' && (github.ref == 'refs/heads/travis-multiarch-builds' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev' || github.ref == 'refs/tags/ng-*') runs-on: ubuntu-latest - # needs: tests - needs: frontend + needs: [frontend, tests] steps: - name: Prepare From a9d6e7f402cb71e94514e69d9645893db076b63c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:22:23 +0100 Subject: [PATCH 129/148] add a publish release step --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4379f8abb..c26c0672f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: name: frontend-compiled path: src/documents/static/frontend/ - release: + build-release: needs: [frontend, documentation, tests] runs-on: ubuntu-20.04 steps: @@ -143,6 +143,38 @@ jobs: name: release path: dist/paperless-ng.tar.xz + publish-release: + runs-on: ubuntu-latest + needs: build-release + steps: + - + name: Download Release Artifact + uses: actions/download-artifact@v2 + with: + name: release + path: paperless-ng.tar.xz + - + name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Paperless-ng ${{ github.ref }} + draft: true + prerelease: true + - + name: Upload Release Archive + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./paperless-ng.tar.xz + asset_name: paperless-ng-${{ github.ref }}.tar.xz + asset_content_type: application/x-xz # build and push image to docker hub. build-docker-image: From ee31249d651a30118822eb356e757a5549872984 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:28:46 +0100 Subject: [PATCH 130/148] reorganized the Dockerfile --- Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7e921fcda..789a15641 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,28 +17,31 @@ COPY requirements.txt ./ # Binary dependencies RUN apt-get update \ && apt-get -y --no-install-recommends install \ + # Basic dependencies curl \ file \ fonts-liberation \ gettext \ - ghostscript \ gnupg \ - icc-profiles-free \ imagemagick \ - liblept5 \ libxslt1-dev \ mime-support \ optipng \ + sudo \ + tzdata \ + # OCRmyPDF dependencies + ghostscript \ + icc-profiles-free \ + liblept5 \ + libxml2 \ pngquant \ qpdf \ - sudo \ tesseract-ocr \ tesseract-ocr-eng \ tesseract-ocr-deu \ tesseract-ocr-fra \ tesseract-ocr-ita \ tesseract-ocr-spa \ - tzdata \ unpaper \ zlib1g \ && rm -rf /var/lib/apt/lists/* @@ -52,7 +55,6 @@ RUN apt-get update \ libpoppler-cpp-dev \ libpq-dev \ libqpdf-dev \ - libxml2 \ && python3 -m pip install --upgrade --no-cache-dir supervisor \ && python3 -m pip install --no-cache-dir -r requirements.txt \ && apt-get -y purge build-essential libqpdf-dev \ From df95fad9be73b1449a731db223fe4f8f79d40158 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:56:10 +0100 Subject: [PATCH 131/148] update release name --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c26c0672f..b5eb9f318 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ github.ref }} - release_name: Paperless-ng ${{ github.ref }} + tag_name: ${{ github.ref#refs/heads/ }} + release_name: Paperless-ng ${{ github.ref#refs/heads/ }} draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${{ github.ref }}.tar.xz + asset_name: paperless-ng-${{ github.ref#refs/heads/ }}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From 8927e0185bedb2bab9126dc1f323ac96eeb7fea9 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 22:57:56 +0100 Subject: [PATCH 132/148] maybe this is better? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5eb9f318..8bcd5de92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ github.ref#refs/heads/ }} - release_name: Paperless-ng ${{ github.ref#refs/heads/ }} + tag_name: ${GITHUB_REF#refs/heads/} + release_name: Paperless-ng ${GITHUB_REF#refs/heads/} draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${{ github.ref#refs/heads/ }}.tar.xz + asset_name: paperless-ng-${GITHUB_REF#refs/heads/}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From bce79189e556fa7fc4b80adb92ed66c7152aa31d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:09:09 +0100 Subject: [PATCH 133/148] maybe this is better? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bcd5de92..41bf07776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${GITHUB_REF#refs/heads/} - release_name: Paperless-ng ${GITHUB_REF#refs/heads/} + tag_name: ${{ GITHUB_REF#refs/heads/ }} + release_name: Paperless-ng ${{ GITHUB_REF#refs/heads/ }} draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${GITHUB_REF#refs/heads/}.tar.xz + asset_name: paperless-ng-${{ GITHUB_REF#refs/heads/ }}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From 0adb8cea91fef7e368a1cffefad59c0ea2ca6100 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:17:53 +0100 Subject: [PATCH 134/148] I have no idea what I am doing --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41bf07776..8f33d5fb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,8 +160,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: ${{ GITHUB_REF#refs/heads/ }} - release_name: Paperless-ng ${{ GITHUB_REF#refs/heads/ }} + tag_name: $GITHUB_REF#refs/heads/ + release_name: Paperless-ng $GITHUB_REF#refs/heads/ draft: true prerelease: true - @@ -173,7 +173,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-${{ GITHUB_REF#refs/heads/ }}.tar.xz + asset_name: paperless-ng-$GITHUB_REF#refs/heads/.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From b5cd713e1675601432dbab4ae835e35c3600e94c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:43:59 +0100 Subject: [PATCH 135/148] maybe this --- .github/workflows/ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f33d5fb4..a4523b6f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,11 @@ jobs: with: name: release path: paperless-ng.tar.xz + - + id: get_branch_name + run: | + branch_name=$(echo ${{ GITHUB_REF }} | cut -d/ -f3) + echo "::set-output name=branch_name::$branch_name" - name: Create Release id: create_release @@ -160,8 +165,8 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: $GITHUB_REF#refs/heads/ - release_name: Paperless-ng $GITHUB_REF#refs/heads/ + tag_name: ${{ steps.get_branch_name.outputs.branch_name }} + release_name: Paperless-ng ${{ steps.get_branch_name.outputs.branch_name }} draft: true prerelease: true - @@ -173,7 +178,7 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./paperless-ng.tar.xz - asset_name: paperless-ng-$GITHUB_REF#refs/heads/.tar.xz + asset_name: paperless-ng-${{ steps.get_branch_name.outputs.branch_name }}.tar.xz asset_content_type: application/x-xz # build and push image to docker hub. From ebcdcc6a1ef79761ec2faded14e31931ea13fe8d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:46:19 +0100 Subject: [PATCH 136/148] github.ref instead of GITHUB_REF --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4523b6f9..d66078023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,7 +156,7 @@ jobs: - id: get_branch_name run: | - branch_name=$(echo ${{ GITHUB_REF }} | cut -d/ -f3) + branch_name=$(echo ${{ github.ref }} | cut -d/ -f3) echo "::set-output name=branch_name::$branch_name" - name: Create Release From f701479d77b17fef3f7ada6d2334ad9566131c13 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 10 Jan 2021 23:57:20 +0100 Subject: [PATCH 137/148] fix upload-release-asset path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d66078023..7a1dcbe7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,7 +152,7 @@ jobs: uses: actions/download-artifact@v2 with: name: release - path: paperless-ng.tar.xz + path: ./ - id: get_branch_name run: | From 6ce0945aebd22ca6eb6a8d9663a934b46b434fc5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 00:08:56 +0100 Subject: [PATCH 138/148] enable multiarch builds, release only with ng- tags --- .github/workflows/ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a1dcbe7e..2ac2fc38f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,7 @@ jobs: publish-release: runs-on: ubuntu-latest needs: build-release + if: contains(github.ref, 'refs/tags/ng-') steps: - name: Download Release Artifact @@ -156,7 +157,7 @@ jobs: - id: get_branch_name run: | - branch_name=$(echo ${{ github.ref }} | cut -d/ -f3) + branch_name=$(echo ${{ github.ref }} | cut -d- -f2) echo "::set-output name=branch_name::$branch_name" - name: Create Release @@ -167,8 +168,10 @@ jobs: with: tag_name: ${{ steps.get_branch_name.outputs.branch_name }} release_name: Paperless-ng ${{ steps.get_branch_name.outputs.branch_name }} - draft: true + draft: false prerelease: true + body: | + TODO: Insert changelog here (it's also in the documentation) - name: Upload Release Archive id: upload-release-asset @@ -233,7 +236,7 @@ jobs: with: context: . file: ./Dockerfile - platforms: linux/amd64 # ,linux/arm/v7,linux/arm64 + platforms: linux/amd64,linux/arm/v7,linux/arm64 push: true tags: jonaswinkler/paperless-ng:${{ steps.prepare.outputs.version }} cache-from: type=local,src=/tmp/.buildx-cache From 8f946074095817022df0d12e1673faed12c241a5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 13:19:29 +0100 Subject: [PATCH 139/148] a couple fixes to the ci script --- .github/workflows/ci.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ac2fc38f..004a66fca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip3.6.8} + key: ${{ runner.os }}-pip3.8} - name: Install dependencies run: | sudo apt-get update -qq @@ -75,9 +75,10 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '15' - - name: build + - name: Build frontend run: ./compile-frontend.sh - - uses: actions/upload-artifact@v2 + - name: Upload artifact + uses: actions/upload-artifact@v2 with: name: frontend-compiled path: src/documents/static/frontend/ @@ -100,11 +101,13 @@ jobs: sudo apt-get install -qq --no-install-recommends libpoppler-cpp-dev gettext liblept5 pip3 install -r requirements.txt - + name: Download frontend artifact uses: actions/download-artifact@v2 with: name: frontend-compiled path: src/documents/static/frontend/ - + name: Download documentation artifact uses: actions/download-artifact@v2 with: name: documentation @@ -138,6 +141,7 @@ jobs: find . -name __pycache__ | xargs rm -r tar -cJf paperless-ng.tar.xz paperless-ng/ - + name: Upload release artifact uses: actions/upload-artifact@v2 with: name: release @@ -149,18 +153,18 @@ jobs: if: contains(github.ref, 'refs/tags/ng-') steps: - - name: Download Release Artifact + name: Download release artifact uses: actions/download-artifact@v2 with: name: release path: ./ - + name: Get branch name id: get_branch_name run: | - branch_name=$(echo ${{ github.ref }} | cut -d- -f2) - echo "::set-output name=branch_name::$branch_name" + echo ::set-output name=branch_name::${GITHUB_REF#refs/tags/ng-} - - name: Create Release + name: Create release id: create_release uses: actions/create-release@v1 env: @@ -173,7 +177,7 @@ jobs: body: | TODO: Insert changelog here (it's also in the documentation) - - name: Upload Release Archive + name: Upload release archive id: upload-release-asset uses: actions/upload-release-asset@v1 env: @@ -206,6 +210,7 @@ jobs: name: Checkout uses: actions/checkout@v2 - + name: Download frontend artifact uses: actions/download-artifact@v2 with: name: frontend-compiled From 4503be110aaa08e1d7972e1cf0ed70f93152359d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 17:13:56 +0100 Subject: [PATCH 140/148] try building this with updated dependencies --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 789a15641..b5d2e5b13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,8 @@ WORKDIR /usr/src/paperless/ COPY requirements.txt ./ # Binary dependencies -RUN apt-get update \ +RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \ + && apt-get update \ && apt-get -y --no-install-recommends install \ # Basic dependencies curl \ From 3fbf24a1c26584877487e630a5369cf025c4e279 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 21:55:06 +0100 Subject: [PATCH 141/148] don't use dependencies from testing --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5d2e5b13..789a15641 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,7 @@ WORKDIR /usr/src/paperless/ COPY requirements.txt ./ # Binary dependencies -RUN echo "deb http://deb.debian.org/debian bullseye main" > /etc/apt/sources.list.d/bullseye.list \ - && apt-get update \ +RUN apt-get update \ && apt-get -y --no-install-recommends install \ # Basic dependencies curl \ From 0e03c48bebcde79a54aad89d9b3e55412cf5a40a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 21:56:50 +0100 Subject: [PATCH 142/148] pin pillow dependency --- Pipfile | 3 ++- Pipfile.lock | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Pipfile b/Pipfile index 1a9b43950..5c1c136b5 100644 --- a/Pipfile +++ b/Pipfile @@ -23,7 +23,8 @@ imap-tools = "*" langdetect = "*" pdftotext = "*" pathvalidate = "*" -pillow = "*" +# pinned to 8.1.0, since aarch64 wheels might not be available beyond that +pillow = "==8.1.0" pikepdf = "*" python-gnupg = "*" python-dotenv = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 58cf35894..c3aa4a4f5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "241c8d042e9b607bbeb7d501b3fc1b57376e3cc8c2b2fe117ec60e425dfd79c7" + "sha256": "27c8973d55150371bd18406cc90e61ad4a7ec4b9353007045d611a87120a2703" }, "pipfile-spec": 6, "requires": {}, @@ -437,6 +437,7 @@ "pillow": { "hashes": [ "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6", + "sha256:1d208e670abfeb41b6143537a681299ef86e92d2a3dac299d3cd6830d5c7bded", "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865", "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174", "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032", @@ -462,9 +463,12 @@ "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8", "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59", "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d", + "sha256:cf6e33d92b1526190a1de904df21663c46a456758c0424e4f947ae9aa6088bf7", "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a", + "sha256:d673c4990acd016229a5c1c4ee8a9e6d8f481b27ade5fc3d95938697fa443ce0", "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b", - "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d" + "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d", + "sha256:f50e7a98b0453f39000619d845be8b06e611e56ee6e8186f7f60c3b1e2f0feae" ], "index": "pypi", "version": "==8.1.0" From 9dd8b7cd88ce36ae4da94a14049f8247bca0bc8f Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 22:11:01 +0100 Subject: [PATCH 143/148] fix test case --- src/paperless_mail/tests/test_mail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 77db096c4..0104f5023 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -41,7 +41,7 @@ class BogusMailBox(ContextManager): folder = BogusFolderManager() - def fetch(self, criteria, mark_seen): + def fetch(self, criteria, mark_seen, charset=""): msg = self.messages criteria = str(criteria).strip('()').split(" ") From 040e652e6a050c3fa2cee16b1b83a478599e442b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 11 Jan 2021 23:07:47 +0100 Subject: [PATCH 144/148] remove obsolete steps from the documentation --- docs/administration.rst | 18 ++++-------------- docs/setup.rst | 7 ------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index bc728943e..c3d1d07a6 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -110,27 +110,17 @@ After grabbing the new release and unpacking the contents, do the following: This creates a new virtual environment (or uses your existing environment) and installs all dependencies into it. - -3. Collect static files. - - .. code:: shell-session - - $ cd src - $ pipenv run python3 manage.py collectstatic --clear -4. Migrate the database. +3. Migrate the database. .. code:: shell-session $ cd src $ pipenv run python3 manage.py migrate -5. Update translation files. - - .. code:: shell-session - - $ cd src - $ pipenv run python3 manage.py compilemessages + This might not actually do anything. Not every new paperless version comes with new + database migrations. + Management utilities #################### diff --git a/docs/setup.rst b/docs/setup.rst index ed7f84885..48569a04c 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -225,7 +225,6 @@ writing. Windows is not and will never be supported. * ``python3-pip``, optionally ``pipenv`` for package installation * ``python3-dev`` - * ``gettext`` for compiling interface translations * ``fonts-liberation`` for generating thumbnails for plain text files * ``imagemagick`` >= 6 for PDF conversion * ``optipng`` for optimizing thumbnails @@ -293,15 +292,9 @@ writing. Windows is not and will never be supported. .. code:: bash - # This collects static files from paperless and django. - python3 manage.py collectstatic --clear --no-input - # This creates the database schema. python3 manage.py migrate - # This creates the translation files for paperless. - python3 manage.py compilemessages - # This creates your first paperless user python3 manage.py createsuperuser From 681b769a71f577333540fc2485274dc21d749ae3 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:09:10 +0100 Subject: [PATCH 145/148] copy files into place --- .gitignore | 4 ++-- docker/hub/.env | 1 + docker/hub/docker-compose.env | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 docker/hub/.env create mode 100644 docker/hub/docker-compose.env diff --git a/.gitignore b/.gitignore index e58fb12e7..b427d5c5c 100644 --- a/.gitignore +++ b/.gitignore @@ -65,8 +65,8 @@ target/ .virtualenv virtualenv /venv -docker-compose.env -docker-compose.yml +/docker-compose.env +/docker-compose.yml # Used for development scripts/import-for-development diff --git a/docker/hub/.env b/docker/hub/.env new file mode 100644 index 000000000..511a1386d --- /dev/null +++ b/docker/hub/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=paperless \ No newline at end of file diff --git a/docker/hub/docker-compose.env b/docker/hub/docker-compose.env new file mode 100644 index 000000000..4271bce6e --- /dev/null +++ b/docker/hub/docker-compose.env @@ -0,0 +1,34 @@ +# The UID and GID of the user used to run paperless in the container. Set this +# to your UID and GID on the host so that you have write access to the +# consumption directory. +#USERMAP_UID=1000 +#USERMAP_GID=1000 + +# Additional languages to install for text recognition, separated by a +# whitespace. Note that this is +# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the +# default language used when guessing the language from the OCR output. +# The container installs English, German, Italian, Spanish and French by +# default. +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# for available languages. +#PAPERLESS_OCR_LANGUAGES=tur ces + +############################################################################### +# Paperless-specific settings # +############################################################################### + +# All settings defined in the paperless.conf.example can be used here. The +# Docker setup does not use the configuration file. +# A few commonly adjusted settings are provided below. + +# Adjust this key if you plan to make paperless available publicly. It should +# be a very long sequence of random characters. You don't need to remember it. +#PAPERLESS_SECRET_KEY=change-me + +# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. +#PAPERLESS_TIME_ZONE=America/Los_Angeles + +# The default language to use for OCR. Set this to the language most of your +# documents are written in. +#PAPERLESS_OCR_LANGUAGE=eng From 221cf1218481d51eed427e6f2792b5f65b2c96a0 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:21:05 +0100 Subject: [PATCH 146/148] only restart docker unless stopped --- docker/hub/docker-compose.postgres.yml | 6 +++--- docker/hub/docker-compose.sqlite.yml | 4 ++-- docker/hub/docker-compose.tika.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml index f821a6781..863c99c32 100644 --- a/docker/hub/docker-compose.postgres.yml +++ b/docker/hub/docker-compose.postgres.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped db: image: postgres:13 - restart: always + restart: unless-stopped volumes: - pgdata:/var/lib/postgresql/data environment: @@ -16,7 +16,7 @@ services: webserver: image: jonaswinkler/paperless-ng:latest - restart: always + restart: unless-stopped depends_on: - db - broker diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml index cc26dc46c..2ab0cbb9c 100644 --- a/docker/hub/docker-compose.sqlite.yml +++ b/docker/hub/docker-compose.sqlite.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: image: jonaswinkler/paperless-ng:latest - restart: always + restart: unless-stopped depends_on: - broker ports: diff --git a/docker/hub/docker-compose.tika.yml b/docker/hub/docker-compose.tika.yml index 7ad7685d3..f9e44472a 100644 --- a/docker/hub/docker-compose.tika.yml +++ b/docker/hub/docker-compose.tika.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: image: jonaswinkler/paperless-ng:latest - restart: always + restart: unless-stopped depends_on: - broker ports: From 50ba88231f3b3d1ed3794b91191733ed54383438 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:29:05 +0100 Subject: [PATCH 147/148] documentation --- docs/administration.rst | 5 ++--- docs/setup.rst | 17 +++++++---------- docs/usage_overview.rst | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index c3d1d07a6..80f57d849 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -48,9 +48,6 @@ Options available to bare-metal and non-docker installations: Restoring ========= - - - .. _administration-updating: Updating Paperless @@ -84,9 +81,11 @@ B. If you built the image yourself, do the following: .. code:: shell-session $ git pull + $ ./compile-frontend.sh $ docker-compose build $ docker-compose up +Running `docker-compose up` will also apply any new database migrations. If you see everything working, you can start paperless-ng with "-d" to have it run in the background. diff --git a/docs/setup.rst b/docs/setup.rst index 48569a04c..1e46608ae 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -97,9 +97,10 @@ run the above mentioned components yourself. Install Paperless from Docker Hub ================================= -1. Go to the `project page `_ +1. Go to the `/docker/hub directory on the project page `_ and download one of the ``docker-compose.*.yml`` files, depending on which database backend you - want to use. Download the ``docker-compose.env`` file and the ``.env`` file as well and store them + want to use. Rename this file to `docker-compose.yml`. + Download the ``docker-compose.env`` file and the ``.env`` file as well and store them in the same directory. .. hint:: @@ -124,7 +125,6 @@ Install Paperless from Docker Hub .. _Docker installation guide: https://docs.docker.com/engine/installation/ .. _docker-compose installation guide: https://docs.docker.com/compose/install/ - 3. Modify ``docker-compose.yml`` to your preferences. You may want to change the path to the consumption directory in this file. Find the line that specifies where to mount the consumption directory: @@ -201,7 +201,9 @@ Build the docker image yourself depending on which database backend you want to use. Copy ``docker-compose.env.example`` to ``docker-compose.env``. -3. Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run +3. Run the `compile-frontend.sh` script. This requires `node` and `npm`. + +4. Follow steps 2 to 7 of :ref:`setup-docker_hub`. When asked to run ``docker-compose up -d`` to start the containers, do .. code:: shell-session @@ -422,12 +424,7 @@ Migration to paperless-ng is then performed in a few simple steps: 5. Adjust ``docker-compose.yml`` and ``docker-compose.env`` to your needs. See :ref:`setup-docker_hub` for details on which edits are advised. -6. Since ``docker-compose`` would just use the the old paperless image, we need to - manually build a new image: - - .. code:: shell-session - - $ docker-compose build +6. :ref:`Update paperless. ` 7. In order to find your existing documents with the new search feature, you need to invoke a one-time operation that will create the search index: diff --git a/docs/usage_overview.rst b/docs/usage_overview.rst index 7a4fd7740..6c34b2823 100644 --- a/docs/usage_overview.rst +++ b/docs/usage_overview.rst @@ -118,7 +118,7 @@ to share any documents with paperless. This can be combined with any of the mobi scanning apps out there, such as Office Lens. Furthermore, there is the `Paperless App `_ as well, -which no only has document upload, but also document editing and browsing. +which not only has document upload, but also document browsing and download features. .. _usage-email: From 5d1dddfac82091c4a1c7db31b5f2a02509e41865 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 12 Jan 2021 00:40:49 +0100 Subject: [PATCH 148/148] unless-stopped in local dockerfiles --- docker-compose.postgres.yml | 6 +++--- docker-compose.sqlite.yml | 4 ++-- docker-compose.tika.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml index 59ae28939..53fea9e28 100644 --- a/docker-compose.postgres.yml +++ b/docker-compose.postgres.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped db: image: postgres:13 - restart: always + restart: unless-stopped volumes: - pgdata:/var/lib/postgresql/data environment: @@ -16,7 +16,7 @@ services: webserver: build: . - restart: always + restart: unless-stopped depends_on: - db - broker diff --git a/docker-compose.sqlite.yml b/docker-compose.sqlite.yml index 2d21fa3f1..87aa100a6 100644 --- a/docker-compose.sqlite.yml +++ b/docker-compose.sqlite.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: build: . - restart: always + restart: unless-stopped depends_on: - broker ports: diff --git a/docker-compose.tika.yml b/docker-compose.tika.yml index 889713908..d7b7e0da9 100644 --- a/docker-compose.tika.yml +++ b/docker-compose.tika.yml @@ -2,11 +2,11 @@ version: "3.4" services: broker: image: redis:6.0 - restart: always + restart: unless-stopped webserver: build: . - restart: always + restart: unless-stopped depends_on: - broker ports: