From 5b344963b921afdaf06c4c58df9d59b23f4e53d2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 15:58:29 +0100 Subject: [PATCH 001/371] reorganized docker build. --- docker/local/Dockerfile => Dockerfile | 17 ++- ...ostgres.yml => docker-compose.postgres.yml | 0 ...se.sqlite.yml => docker-compose.sqlite.yml | 0 docker/docker-compose.env | 34 ----- scripts/make-release.sh | 125 ------------------ scripts/push-release.sh | 23 ---- 6 files changed, 14 insertions(+), 185 deletions(-) rename docker/local/Dockerfile => Dockerfile (78%) rename docker/local/docker-compose.postgres.yml => docker-compose.postgres.yml (100%) rename docker/local/docker-compose.sqlite.yml => docker-compose.sqlite.yml (100%) delete mode 100644 docker/docker-compose.env delete mode 100755 scripts/make-release.sh delete mode 100755 scripts/push-release.sh diff --git a/docker/local/Dockerfile b/Dockerfile similarity index 78% rename from docker/local/Dockerfile rename to Dockerfile index 461b9e4fc..732b4183c 100644 --- a/docker/local/Dockerfile +++ b/Dockerfile @@ -1,8 +1,18 @@ +FROM node:15 AS frontend + +WORKDIR /usr/src/paperless/src-ui/ + +COPY src-ui/package* ./ +RUN npm install + +COPY src-ui . +RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false + FROM python:3.7-slim WORKDIR /usr/src/paperless/ -COPY requirements.txt ./ +COPY Pipfile* ./ #Dependencies RUN apt-get update \ @@ -33,8 +43,8 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools \ - && pip install --no-cache-dir -r requirements.txt \ + && pip3 install --upgrade supervisor setuptools pipenv \ + && pipenv install --system --deploy --ignore-pipfile --clear \ && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ @@ -49,6 +59,7 @@ COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh # copy app COPY src/ ./src/ +COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/docker/local/docker-compose.postgres.yml b/docker-compose.postgres.yml similarity index 100% rename from docker/local/docker-compose.postgres.yml rename to docker-compose.postgres.yml diff --git a/docker/local/docker-compose.sqlite.yml b/docker-compose.sqlite.yml similarity index 100% rename from docker/local/docker-compose.sqlite.yml rename to docker-compose.sqlite.yml diff --git a/docker/docker-compose.env b/docker/docker-compose.env deleted file mode 100644 index 4271bce6e..000000000 --- a/docker/docker-compose.env +++ /dev/null @@ -1,34 +0,0 @@ -# The UID and GID of the user used to run paperless in the container. Set this -# to your UID and GID on the host so that you have write access to the -# consumption directory. -#USERMAP_UID=1000 -#USERMAP_GID=1000 - -# Additional languages to install for text recognition, separated by a -# whitespace. Note that this is -# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the -# default language used when guessing the language from the OCR output. -# The container installs English, German, Italian, Spanish and French by -# default. -# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster -# for available languages. -#PAPERLESS_OCR_LANGUAGES=tur ces - -############################################################################### -# Paperless-specific settings # -############################################################################### - -# All settings defined in the paperless.conf.example can be used here. The -# Docker setup does not use the configuration file. -# A few commonly adjusted settings are provided below. - -# Adjust this key if you plan to make paperless available publicly. It should -# be a very long sequence of random characters. You don't need to remember it. -#PAPERLESS_SECRET_KEY=change-me - -# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. -#PAPERLESS_TIME_ZONE=America/Los_Angeles - -# The default language to use for OCR. Set this to the language most of your -# documents are written in. -#PAPERLESS_OCR_LANGUAGE=eng diff --git a/scripts/make-release.sh b/scripts/make-release.sh deleted file mode 100755 index f5c9028fa..000000000 --- a/scripts/make-release.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -# Release checklist -# - wait for travis build. -# adjust src/paperless/version.py -# changelog in the documentation -# adjust versions in docker/hub/* -# adjust version in src-ui/src/environments/prod -# If docker-compose was modified: all compose files are the same. - -# Steps: -# run release script "dev", push -# if it works: new tag, merge into master -# on master: make release "lastest", push -# on master: make release "version-tag", push -# publish release files - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" -PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles" - -if [ -d "$PAPERLESS_DIST" ] -then - echo "Removing $PAPERLESS_DIST" - rm "$PAPERLESS_DIST" -r -fi - -mkdir "$PAPERLESS_DIST" -mkdir "$PAPERLESS_DIST_APP" -mkdir "$PAPERLESS_DIST_APP/docker" -mkdir "$PAPERLESS_DIST_APP/scripts" -mkdir "$PAPERLESS_DIST_DOCKERFILES" - -# setup dependencies. - -cd "$PAPERLESS_ROOT" - -pipenv clean -pipenv install --dev -pipenv lock --keep-outdated -r > "$PAPERLESS_DIST_APP/requirements.txt" - -# test if the application works. - -cd "$PAPERLESS_ROOT/src" -pipenv run pytest --cov -pipenv run pycodestyle - -# make the documentation. - -cd "$PAPERLESS_ROOT/docs" -make clean html - -# copy stuff into place - -# the application itself - -cp "$PAPERLESS_ROOT/.env" \ - "$PAPERLESS_ROOT/.dockerignore" \ - "$PAPERLESS_ROOT/CONTRIBUTING.md" \ - "$PAPERLESS_ROOT/LICENSE" \ - "$PAPERLESS_ROOT/Pipfile" \ - "$PAPERLESS_ROOT/Pipfile.lock" \ - "$PAPERLESS_ROOT/README.md" "$PAPERLESS_DIST_APP" - -cp "$PAPERLESS_ROOT/paperless.conf.example" "$PAPERLESS_DIST_APP/paperless.conf" - -# copy python source, templates and static files. -cd "$PAPERLESS_ROOT" -find src -wholename '*/templates/*' -o -wholename '*/static/*' -o -name '*.py' | cpio -pdm "$PAPERLESS_DIST_APP" - -# build the front end. - -cd "$PAPERLESS_ROOT/src-ui" -ng build --prod --output-hashing none --sourceMap=false --output-path "$PAPERLESS_DIST_APP/src/documents/static/frontend" - -# documentation -cp "$PAPERLESS_ROOT/docs/_build/html/" "$PAPERLESS_DIST_APP/docs" -r - -# docker files for building the image yourself -cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP" - -# docker files for pulling from docker hub -cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES" - -# auxiliary files required for the docker image -cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/gunicorn.conf.py" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/imagemagick-policy.xml" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/" - -# auxiliary files for bare metal installs -cp "$PAPERLESS_ROOT/scripts/paperless-webserver.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-consumer.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-scheduler.service" "$PAPERLESS_DIST_APP/scripts/" - -# try to make the docker build. - -cd "$PAPERLESS_DIST_APP" - -docker build . -t "jonaswinkler/paperless-ng:$VERSION" - -# works. package the app! - -cd "$PAPERLESS_DIST" - -tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/ -tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/ diff --git a/scripts/push-release.sh b/scripts/push-release.sh deleted file mode 100755 index cfa63f5cf..000000000 --- a/scripts/push-release.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" - -cd "$PAPERLESS_DIST_APP" - -docker push "jonaswinkler/paperless-ng:$VERSION" From dd6aa2f77539fc66835484f7a95e80cb08a7e026 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 16:01:16 +0100 Subject: [PATCH 002/371] added missing file --- docker-compose.example.env | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docker-compose.example.env diff --git a/docker-compose.example.env b/docker-compose.example.env new file mode 100644 index 000000000..4271bce6e --- /dev/null +++ b/docker-compose.example.env @@ -0,0 +1,34 @@ +# The UID and GID of the user used to run paperless in the container. Set this +# to your UID and GID on the host so that you have write access to the +# consumption directory. +#USERMAP_UID=1000 +#USERMAP_GID=1000 + +# Additional languages to install for text recognition, separated by a +# whitespace. Note that this is +# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the +# default language used when guessing the language from the OCR output. +# The container installs English, German, Italian, Spanish and French by +# default. +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# for available languages. +#PAPERLESS_OCR_LANGUAGES=tur ces + +############################################################################### +# Paperless-specific settings # +############################################################################### + +# All settings defined in the paperless.conf.example can be used here. The +# Docker setup does not use the configuration file. +# A few commonly adjusted settings are provided below. + +# Adjust this key if you plan to make paperless available publicly. It should +# be a very long sequence of random characters. You don't need to remember it. +#PAPERLESS_SECRET_KEY=change-me + +# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. +#PAPERLESS_TIME_ZONE=America/Los_Angeles + +# The default language to use for OCR. Set this to the language most of your +# documents are written in. +#PAPERLESS_OCR_LANGUAGE=eng From 23381f7d170d8dcf1aabcbb26f5dbe04816ca0a8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 13:13:52 +0100 Subject: [PATCH 003/371] Added multi arch docker builds to travis ci --- .travis.yml | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.travis.yml b/.travis.yml index b745d6bd7..307a6073a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,102 @@ jobs: - ng build --prod after_success: true + - stage: build_docker + name: amd64 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 + on: + condition: '"${BUILD_DOCKER}" = 1' + - stage: build_docker + name: arm64v8 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 + arch: arm64 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: build_docker + name: arm32v7 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: publish_manifest + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} + - | + if [ "${TRAVIS_BRANCH}" = "master" ]; then + echo "Master branch detected" + DOCKER_TAG="latest" + else + DOCKER_TAG=${TRAVIS_TAG} + fi + - | + if [ "${DOCKER_TAG}" != "" ]; then + echo "Create Tag ${DOCKER_TAG}" + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} + else + echo "Not a tag and not on master, so not pushing tag/master specific manifest" + fi + on: + condition: '"${BUILD_DOCKER}" = 1' + before_install: - sudo apt-get update -qq From ba18258750e06d4a9f6529baf3722ec467474a88 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 16:19:24 +0100 Subject: [PATCH 004/371] Add libxslt-dev to dependencies. At least one arm based image does not pull in dependencies the same way the amd64 one does, resulting in a missing libxslt shared lib. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..016bf7da9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ + libxslt-dev \ optipng \ pngquant \ qpdf \ From 7817315f8b4b6fb9c95c08f4f788a8c068c4b401 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 17:08:01 +0100 Subject: [PATCH 005/371] Adding some echos to bette runderstand where the build hangs --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 307a6073a..203861391 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,9 +60,12 @@ jobs: after_success: - true script: + - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64 on: From 48b59c8e24081d06d9c38457517cf55cdcc5f79b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 21:53:39 +0100 Subject: [PATCH 006/371] Use qemu also for arm64v8 build - something might be causing trouble on the native hosts --- .travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 203861391..d2534c152 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,14 +60,17 @@ jobs: after_success: - true script: - - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64 + arch: amd64 on: condition: '"${BUILD_DOCKER}" = 1' From ade951a600ee4e00fb1aaeeb8a13fec569640d2e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 23:02:26 +0100 Subject: [PATCH 007/371] Adds jbig2 to the build, fixes #93 --- Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..6c04a1d07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,14 @@ +FROM ubuntu:20.04 AS jbig2 + +WORKDIR /usr/src/jbig2enc + +RUN apt-get update && apt-get install -y --no-install-recommends build-essential automake libtool libleptonica-dev zlib1g-dev git ca-certificates + +RUN git clone https://github.com/agl/jbig2enc . +RUN ./autogen.sh +RUN ./configure && make + + FROM node:15 AS frontend WORKDIR /usr/src/paperless/src-ui/ @@ -57,6 +68,12 @@ COPY docker/gunicorn.conf.py ./ COPY docker/supervisord.conf /etc/supervisord.conf COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh +# copy jbic +COPY --from=jbig2 /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ +COPY --from=jbig2 /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ +COPY --from=jbig2 /usr/src/jbig2enc/src/*.h /usr/local/include/ + + # copy app COPY src/ ./src/ COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ From 7d1f931234147bb4588778fda9d3b1fccf4a7ecd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:49:36 +0100 Subject: [PATCH 008/371] Use arm64-gravis instead of regular arm64. See if that is fast enough. --- .travis.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2534c152..b91e43a28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,17 +60,12 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: amd64 + arch: arm64-graviton2 + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From bf3b110804597af01ade2047527582d9f24ae9b2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:58:35 +0100 Subject: [PATCH 009/371] Remove accidentially added quote --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b91e43a28..5f838755d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 From 162626209c82ce8f8f286790302a279afdac6f86 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:27:19 +0100 Subject: [PATCH 010/371] Tell me, what kind of arm64 is it that produces amd64 docker images? It's weird --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f838755d..f99a52eba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,12 +60,13 @@ jobs: after_success: - true script: + - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: vm + virt: lxc on: condition: '"${BUILD_DOCKER}" = 1' @@ -85,8 +86,8 @@ jobs: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile + - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 From e8ea373eb2c4e3a686e07d2fc8919b0b15bb3281 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:42:21 +0100 Subject: [PATCH 011/371] Enable manifests support --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f99a52eba..6d26d6ccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,8 @@ jobs: # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: condition: '"${BUILD_DOCKER}" = 1' From 46ecdefde84eb581b038da1cdc2728cc3f11ea8d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:43:51 +0100 Subject: [PATCH 012/371] Remove steps not necessary to build docker images --- .travis.yml | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d26d6ccb..c04f765d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: - - name: "Paperless on Python 3.6" - python: "3.6" - - - name: "Paperless on Python 3.7" - python: "3.7" - - - name: "Paperless on Python 3.8" - python: "3.8" - - - name: "Documentation" - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true +# - name: "Paperless on Python 3.6" +# python: "3.6" +# +# - name: "Paperless on Python 3.7" +# python: "3.7" +# +# - name: "Paperless on Python 3.8" +# python: "3.8" +# +# - name: "Documentation" +# script: +# - cd docs/ +# - make html +# after_success: true +# +# - name: "Front end" +# language: node_js +# node_js: +# - 15 +# before_install: true +# install: +# - cd src-ui/ +# - npm install -g @angular/cli +# - npm install +# script: +# - ng build --prod +# after_success: true - stage: build_docker name: amd64 docker build From c751fe752079bf1367944e81c26a4d718a4531be Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:15:06 +0100 Subject: [PATCH 013/371] Switching back to virt: vm as there seems to be a prob with the lxc ones --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c04f765d0..9803a4d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ jobs: - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: lxc + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From 1f62b6d0d4c44516818a3e8e5d6e3efaa50f97f2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:18:11 +0100 Subject: [PATCH 014/371] arm64-graviton2 build requires group:edge --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9803a4d3b..8422229c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 virt: vm + group: edge on: condition: '"${BUILD_DOCKER}" = 1' From 50c5a23de8ae1cd86f15260b459972b27d1a1d32 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 18:46:47 +0100 Subject: [PATCH 015/371] add basic ansible role for debian deployment Currently only Debian 10 buster is supported. Other Debian versions, Ubuntu and derivates should be easy to integrate. Database deployment is considered out-of-scope and deferred to the user. Provides basic upgrade support between releases. --- ansible/defaults/main.yml | 39 +++++ ansible/tasks/main.yml | 350 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+) create mode 100644 ansible/defaults/main.yml create mode 100644 ansible/tasks/main.yml diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml new file mode 100644 index 000000000..0fa03b47a --- /dev/null +++ b/ansible/defaults/main.yml @@ -0,0 +1,39 @@ +--- +paperlessng_version: 0.9.8 +paperlessng_directory: /opt/paperless-ng +paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" +paperlessng_data_dir: "{{ paperlessng_directory }}/data" +paperlessng_media_root: "{{ paperlessng_directory }}/media" +paperlessng_static_dir: "{{ paperlessng_directory }}/static" +paperlessng_filename_format: +paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" + +paperlessng_ocr_languages: + - eng +paperlessng_time_zone: Europe/Berlin +paperlessng_ocrmypdf_args: --optimize 1 +# TODO Does optimze==1 really work with jbig2enc? +# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 +# Documentation states -O1 only applies lossless transformations +# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +paperlessng_use_jbig2enc: true + +paperlessng_superuser_name: paperlessng +paperlessng_superuser_email: paperlessng@example.com +paperlessng_superuser_password: paperlessng + +paperlessng_system_user: paperlessng +paperlessng_system_group: paperlessng + +paperlessng_listen_address: 127.0.0.1 +paperlessng_listen_port: 8000 + +paperlessng_redis_host: localhost +paperlessng_redis_port: 6379 + +paperlessng_db_type: sqlite # or postgresql +paperlessng_db_host: localhost +paperlessng_db_port: 5432 +paperlessng_db_name: paperlessng +paperlessng_db_user: paperlessng +paperlessng_db_pass: paperlessng \ No newline at end of file diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml new file mode 100644 index 000000000..b427482b2 --- /dev/null +++ b/ansible/tasks/main.yml @@ -0,0 +1,350 @@ +--- +- name: verify operating system + fail: + msg: Sorry, only Debian 10 supported at the moment. + when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + +- name: install base dependencies + apt: + update_cache: yes + pkg: + # paperless-ng + - python3-dev + - python3-pip + - imagemagick + - unpaper + - ghostscript + - optipng + - tesseract-ocr + - gnupg + - libpoppler-cpp-dev + - libmagic-dev + - libpq-dev + # OCRmyPDF + - icc-profiles-free + - qpdf + - liblept5 + - libxml2 + - pngquant + - zlib1g + # dev + - build-essential + - python3-setuptools + - python3-wheel + - python3-virtualenv + +- name: install ocr languages + apt: + pkg: "{{ paperlessng_ocr_languages | map('regex_replace', '^(.*)$', 'tesseract-ocr-\\1') | list }}" + +- name: set up notesalexp repository key (for jbig2enc) + apt_key: + url: https://notesalexp.org/debian/alexp_key.asc + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository (for jbig2enc) + apt_repository: + repo: deb https://notesalexp.org/debian/buster/ buster main + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository pinning + copy: + content: | + Package: * + Pin: release o=notesalexp.org + Pin-Priority: 1 + + Package: jbig2enc + Pin: release o=notesalexp.org + Pin-Priority: 500 + dest: /etc/apt/preferences.d/notesalexp + when: paperlessng_use_jbig2enc + +- name: install jbig2enc + apt: + pkg: jbig2enc + update_cache: yes + when: paperlessng_use_jbig2enc + +- name: install redis + apt: + pkg: redis-server + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: enable redis + systemd: + name: redis-server + enabled: yes + masked: no + state: started + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: check for paperless-ng installation + command: + cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' + changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + failed_when: false + ignore_errors: yes + register: paperlessng_current_version + +- name: backup current paperless-ng installation + copy: + src: "{{ paperlessng_directory }}" + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: download paperless-ng + get_url: + url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: create paperless-ng directories + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: 0750 + recurse: yes + with_items: + - "{{ paperlessng_directory }}" + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + +- name: create temporary directory + tempfile: + state: directory + register: tempdir + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: extract paperless-ng + unarchive: + src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + dest: "{{ tempdir.path }}" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: move paperless-ng + command: + cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" + args: + warn: false + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: remove temporary directory + file: + path: "{{ tempdir.path }}" + state: absent + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: configure paperless-ng + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_REDIS=" + line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}" + - regexp: "^#?PAPERLESS_CONSUMPTION_DIR=" + line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}" + - regexp: "^#?PAPERLESS_DATA_DIR=" + line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}" + - regexp: "^#?PAPERLESS_MEDIA_ROOT=" + line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}" + - regexp: "^#?PAPERLESS_STATICDIR=" + line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}" + - regexp: "^#?PAPERLESS_FILENAME_FORMAT=" + line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" + - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" + line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # TODO JSON dict required in conf? + # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#?PAPERLESS_TIME_ZONE=" + line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" + no_log: true + +- name: configure paperless-ng database [sqlite] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "^#?PAPERLESS_DBHOST=" + state: absent + when: paperlessng_db_type == 'sqlite' + +- name: configure paperless-ng database [postgresql] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_DBHOST=" + line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}" + - regexp: "^#?PAPERLESS_DBPORT=" + line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}" + - regexp: "^#?PAPERLESS_DBNAME=" + line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}" + - regexp: "^#?PAPERLESS_DBUSER=" + line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}" + - regexp: "^#?PAPERLESS_DBPASS=" + line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" + when: paperlessng_db_type == 'postgresql' + no_log: true + +- name: create paperlessng venv + command: + cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" + creates: "{{ paperlessng_virtualenv }}" + +- name: install paperlessng requirements + pip: + requirements: "{{ paperlessng_directory }}/requirements.txt" + virtualenv: "{{ paperlessng_virtualenv }}" + extra_args: --upgrade + +- name: collect static files + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + +- name: create database schema + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + register: database_schema + changed_when: '"No migrations to apply." not in database_schema.stdout' + +- name: create first paperless user + # "manage.py createsuperuser" only works on interactive TTYs + command: | + {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') + " + register: superuser + changed_when: superuser.stdout == 'changed' + no_log: true + +- name: configure ghostscript for PDF + lineinfile: + path: "/etc/ImageMagick-6/policy.xml" + regexp: '' + line: '' + +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + +- name: configure systemd services + ini_file: + path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" + section: "{{ item[1].section }}" + option: "{{ item[1].option }}" + value: "{{ item[1].value }}" + with_nested: + - [ + paperless-consumer.service, + paperless-scheduler.service, + paperless-webserver.service, + ] + - [ + { + section: "Service", + option: "User", + value: "{{ paperlessng_system_user }}", + }, + { + section: "Service", + option: "Group", + value: "{{ paperlessng_system_group }}", + }, + { + section: "Service", + option: "WorkingDirectory", + value: "{{ paperlessng_directory }}/src", + }, + ] + +- name: configure paperless-consumer service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-consumer.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py document_consumer" + +- name: configure paperless-scheduler service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-scheduler.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py qcluster" + +- name: configure paperless-webserver service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}" + +- name: copy systemd services + copy: + src: "{{ paperlessng_directory }}/scripts/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + remote_src: yes + with_items: + - paperless-consumer.service + - paperless-scheduler.service + - paperless-webserver.service + register: paperless_services + +- name: reload systemd daemon + systemd: + name: "{{ item }}" + state: restarted + daemon_reload: yes + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver + when: paperless_services.changed + +- name: enable paperlessng services + systemd: + name: "{{ item }}" + enabled: yes + masked: no + state: started + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver From e48294a74b18283ce4c60288eed907a1c0c8dbb1 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 19:30:53 +0100 Subject: [PATCH 016/371] Update to 0.9.9 --- ansible/defaults/main.yml | 5 +++-- ansible/tasks/main.yml | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 0fa03b47a..fbeef9871 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.8 +paperlessng_version: 0.9.9 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" @@ -32,8 +32,9 @@ paperlessng_redis_host: localhost paperlessng_redis_port: 6379 paperlessng_db_type: sqlite # or postgresql +# Below entries only apply for paperlessng_db_type=='postgresql' paperlessng_db_host: localhost paperlessng_db_port: 5432 paperlessng_db_name: paperlessng paperlessng_db_user: paperlessng -paperlessng_db_pass: paperlessng \ No newline at end of file +paperlessng_db_pass: paperlessng diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b427482b2..96b1e469b 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + #no_log: true - name: configure paperless-ng database [sqlite] lineinfile: @@ -209,17 +209,21 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + args: + chdir: "{{ paperlessng_directory }}/src" - name: create database schema - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" + args: + chdir: "{{ paperlessng_directory }}/src" register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' - name: create first paperless user # "manage.py createsuperuser" only works on interactive TTYs command: | - {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " from django.contrib.auth.models import User from django.contrib.auth.hashers import get_hasher @@ -242,6 +246,8 @@ User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') " + args: + chdir: "{{ paperlessng_directory }}/src" register: superuser changed_when: superuser.stdout == 'changed' no_log: true From ccfa14b9b7c48ed60fc49ce64d673dde02edb308 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 09:14:40 +0100 Subject: [PATCH 017/371] Remove arm32v6 from docker manifest , it is not getting created --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8422229c7..281635ace 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ jobs: - true script: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} @@ -124,7 +124,7 @@ jobs: - | if [ "${DOCKER_TAG}" != "" ]; then echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} From 8135de49f00e01170c1450f380fb1789626d68af Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 12:13:12 +0100 Subject: [PATCH 018/371] Re-enable test steps --- .travis.yml | 55 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 281635ace..e5f3cd91e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: -# - name: "Paperless on Python 3.6" -# python: "3.6" -# -# - name: "Paperless on Python 3.7" -# python: "3.7" -# -# - name: "Paperless on Python 3.8" -# python: "3.8" -# -# - name: "Documentation" -# script: -# - cd docs/ -# - make html -# after_success: true -# -# - name: "Front end" -# language: node_js -# node_js: -# - 15 -# before_install: true -# install: -# - cd src-ui/ -# - npm install -g @angular/cli -# - npm install -# script: -# - ng build --prod -# after_success: true + - name: "Paperless on Python 3.6" + python: "3.6" + + - name: "Paperless on Python 3.7" + python: "3.7" + + - name: "Paperless on Python 3.8" + python: "3.8" + + - name: "Documentation" + script: + - cd docs/ + - make html + after_success: true + + - name: "Front end" + language: node_js + node_js: + - 15 + before_install: true + install: + - cd src-ui/ + - npm install -g @angular/cli + - npm install + script: + - ng build --prod + after_success: true - stage: build_docker name: amd64 docker build @@ -60,7 +60,6 @@ jobs: after_success: - true script: - - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . From 92fa97873543cbd499ceaca8787765f231027071 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:30:09 +0100 Subject: [PATCH 019/371] Allow running role on all Debian releases Dynamically template current release string where required. --- ansible/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 96b1e469b..5b0bb5802 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian 10 supported at the moment. - when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + msg: Sorry, only Debian supported at the moment. + when: ansible_distribution != 'Debian' - name: install base dependencies apt: @@ -45,7 +45,7 @@ - name: set up notesalexp repository (for jbig2enc) apt_repository: - repo: deb https://notesalexp.org/debian/buster/ buster main + repo: "deb https://notesalexp.org/debian/{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} main" state: present when: paperlessng_use_jbig2enc @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - #no_log: true + no_log: true - name: configure paperless-ng database [sqlite] lineinfile: From 089a8c0498c1e3264209284a7f5804c093b7d3d2 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:39:16 +0100 Subject: [PATCH 020/371] Fix fresh installation We can't backup a nonexistent folder. --- ansible/tasks/main.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 5b0bb5802..ba83b4975 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,6 +81,19 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + - name: check for paperless-ng installation command: cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' @@ -94,7 +107,7 @@ src: "{{ paperlessng_directory }}" dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' - name: download paperless-ng get_url: @@ -258,19 +271,6 @@ regexp: '' line: '' -- name: create paperless group - group: - name: "{{ paperlessng_system_group }}" - -- name: create paperless user - user: - name: "{{ paperlessng_system_user }}" - groups: - - "{{ paperlessng_system_group }}" - shell: /usr/sbin/nologin - # GNUPG_HOME required due to paperless db.py - create_home: yes - - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" From 227934a7f0941e76c8c6b51f552aea25ab2e3222 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 14:05:35 +0100 Subject: [PATCH 021/371] Do not clear static files on every run Django collectstatic knows when files change. --- ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index ba83b4975..9c8fd67f5 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -222,7 +222,7 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" From 02e81f7ab5942afac95d1346ab743f17be4a04f2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 23 Dec 2020 17:03:00 +0100 Subject: [PATCH 022/371] update Dockerfile --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f0112e529..686496475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ - libxslt-dev \ + libxslt1-dev \ optipng \ pngquant \ qpdf \ @@ -55,9 +55,11 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools pipenv \ - && pipenv install --system --deploy --ignore-pipfile --clear \ - && apt-get -y purge build-essential libqpdf-dev \ + && pip3 install --upgrade supervisor pipenv \ + && pipenv install --system --deploy --ignore-pipfile \ + && pipenv --clear \ + && pip3 uninstall -y pipenv \ + && apt-get -y purge build-essential libqpdf-dev libxslt1-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord From 0560ac2a957957030b695b7645d2d35f7809b683 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sun, 27 Dec 2020 20:01:21 +0100 Subject: [PATCH 023/371] Use conditions to prevent builds on branches / tags other than master, dev and ng-* --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5f3cd91e..cf18825e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ jobs: - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -68,8 +68,7 @@ jobs: virt: vm group: edge on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -94,8 +93,7 @@ jobs: env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -131,7 +129,7 @@ jobs: echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From ef9631ae241889cfca5ce7f18b71d2575ae5a673 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 09:47:17 +0100 Subject: [PATCH 024/371] Drop all permissions to paperlessng user Also make role idempotent --- ansible/tasks/main.yml | 105 ++++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 9c8fd67f5..77635b3d1 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,11 +81,11 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' -- name: create paperless group +- name: create paperless system group group: name: "{{ paperlessng_system_group }}" -- name: create paperless user +- name: create paperless system user user: name: "{{ paperlessng_system_user }}" groups: @@ -105,31 +105,10 @@ - name: backup current paperless-ng installation copy: src: "{{ paperlessng_directory }}" - dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' -- name: download paperless-ng - get_url: - url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" - dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - -- name: create paperless-ng directories - file: - path: "{{ item }}" - state: directory - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: 0750 - recurse: yes - with_items: - - "{{ paperlessng_directory }}" - - "{{ paperlessng_consumption_dir }}" - - "{{ paperlessng_data_dir }}" - - "{{ paperlessng_media_root }}" - - "{{ paperlessng_static_dir }}" - - name: create temporary directory tempfile: state: directory @@ -138,16 +117,28 @@ - name: extract paperless-ng unarchive: - src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - dest: "{{ tempdir.path }}" + src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" remote_src: yes + dest: "{{ tempdir.path }}" + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: change permissions of paperless-ng + command: + cmd: "{{ item }}" + with_items: + - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" + - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - command: - cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" - args: - warn: false + copy: + src: "{{ tempdir.path }}/paperless-ng/" + remote_src: yes + dest: "{{ paperlessng_directory }}" + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: preserve + directory_mode: preserve when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -156,6 +147,20 @@ state: absent when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' +- name: create paperless-ng directories and set permissions + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: "750" + with_items: + - "{{ paperlessng_directory }}" # ansible `copy:` does not set correct permissions on `dest:` for recursive copies + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + - name: configure paperless-ng lineinfile: path: "{{ paperlessng_directory }}/paperless.conf" @@ -176,10 +181,10 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # TODO JSON dict required in conf? - # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + # - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # # TODO JSON dict required in conf + # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: true @@ -211,29 +216,45 @@ no_log: true - name: create paperlessng venv + become: yes + become_user: "{{ paperlessng_system_user }}" command: cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" creates: "{{ paperlessng_virtualenv }}" + register: venv - name: install paperlessng requirements + become: yes + become_user: "{{ paperlessng_system_user }}" pip: requirements: "{{ paperlessng_directory }}/requirements.txt" - virtualenv: "{{ paperlessng_virtualenv }}" + executable: "{{ paperlessng_virtualenv }}/bin/pip3" extra_args: --upgrade + when: paperlessng_current_version.stdout != paperlessng_version | string - name: collect static files + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string + register: static_files + changed_when: "'188 unmodified' not in static_files.stdout" - name: create database schema + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' -- name: create first paperless user +- name: configure paperless superuser + become: yes + become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs command: | {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " @@ -265,6 +286,16 @@ changed_when: superuser.stdout == 'changed' no_log: true +- name: set ownership and permissions on paperlessng venv + file: + path: "{{ paperlessng_virtualenv }}" + state: directory + recurse: yes + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: g-w,o-rwx + when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string + - name: configure ghostscript for PDF lineinfile: path: "/etc/ImageMagick-6/policy.xml" @@ -325,8 +356,8 @@ - name: copy systemd services copy: src: "{{ paperlessng_directory }}/scripts/{{ item }}" - dest: "/etc/systemd/system/{{ item }}" remote_src: yes + dest: "/etc/systemd/system/{{ item }}" with_items: - paperless-consumer.service - paperless-scheduler.service From 1276419ec6e6691280d98bfb30081f926166e6e5 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 11:28:19 +0100 Subject: [PATCH 025/371] Add molecule test for role Only test default installation with jbig2enc and sqlite --- .travis.yml | 10 ++++++++++ ansible/molecule/default/converge.yml | 7 +++++++ ansible/molecule/default/molecule.yml | 21 +++++++++++++++++++++ ansible/molecule/default/verify.yml | 14 ++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 ansible/molecule/default/converge.yml create mode 100644 ansible/molecule/default/molecule.yml create mode 100644 ansible/molecule/default/verify.yml diff --git a/.travis.yml b/.travis.yml index b745d6bd7..e36b1ca99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,16 @@ jobs: - ng build --prod after_success: true + - name: "Ansible role" + sudo: required + services: + - docker + install: + - python3 -m pip install molecule[ansible,docker] + script: + - cd ansible/ + - molecule test + after_success: true before_install: - sudo apt-get update -qq diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml new file mode 100644 index 000000000..82067f417 --- /dev/null +++ b/ansible/molecule/default/converge.yml @@ -0,0 +1,7 @@ +--- +- name: Converge + hosts: all + tasks: + - name: "Include ansible" + include_role: + name: "ansible" diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml new file mode 100644 index 000000000..1bf938d98 --- /dev/null +++ b/ansible/molecule/default/molecule.yml @@ -0,0 +1,21 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: debian_buster + image: jrei/systemd-debian:10 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies +provisioner: + name: ansible +verifier: + name: ansible diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml new file mode 100644 index 000000000..4c3b86044 --- /dev/null +++ b/ansible/molecule/default/verify.yml @@ -0,0 +1,14 @@ +--- +# This is an example playbook to execute Ansible tests. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: check if webserver is up + uri: + url: http://localhost:8000 + status_code: [200, 302] + return_content: yes + register: landingpage + failed_when: "'Sign in' not in landingpage.content" From 61834581d5b840eedf35177cee29efc33100330b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:22:35 +0100 Subject: [PATCH 026/371] Move build condition into separate stages list --- .travis.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf18825e8..26153588e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,12 @@ language: python dist: focal os: linux +stages: + - name: build_docker + if: branch =~ /^(master|dev|ng-.*)$/ + - name: publish_manifest + if: branch =~ /^(master|dev|ng-.*)$/ + jobs: include: - name: "Paperless on Python 3.6" @@ -47,8 +53,6 @@ jobs: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -67,8 +71,6 @@ jobs: arch: arm64-graviton2 virt: vm group: edge - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -92,8 +94,6 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -128,8 +128,6 @@ jobs: else echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From ba9b5c50d2a6cfac6c8615f5a9a603f4e0d33b5d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:24:39 +0100 Subject: [PATCH 027/371] Add stage definitions of test jobs and add test to list of stages Defining the stages instead of having travis guess them and implicitly build an order ensures that the test stage is executed before the docker images are built. --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 26153588e..f4873cf3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ dist: focal os: linux stages: + - name: test - name: build_docker if: branch =~ /^(master|dev|ng-.*)$/ - name: publish_manifest @@ -12,21 +13,26 @@ stages: jobs: include: - name: "Paperless on Python 3.6" + stage: test python: "3.6" - name: "Paperless on Python 3.7" + stage: test python: "3.7" - name: "Paperless on Python 3.8" + stage: test python: "3.8" - name: "Documentation" + stage: test script: - cd docs/ - make html after_success: true - name: "Front end" + stage: test language: node_js node_js: - 15 From bdf2e298435b2e3139224f91a880842de3193eae Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 12:21:10 +0100 Subject: [PATCH 028/371] Verify role for Ubuntu 20.04 --- ansible/molecule/default/molecule.yml | 14 ++++++++++++++ ansible/tasks/main.yml | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 1bf938d98..27f37ba63 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -4,6 +4,20 @@ dependency: driver: name: docker platforms: + - name: ubuntu_focal + image: jrei/systemd-ubuntu:20.04 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # ubuntu 18.04 bionic works except that + # the default redis configuration expects IPv6 which is not enabled in docker by default + # the default Python environment is configured for ASCII instead of UTF-8 + # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies - name: debian_buster image: jrei/systemd-debian:10 privileged: true diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 77635b3d1..c94534895 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian supported at the moment. - when: ansible_distribution != 'Debian' + msg: Sorry, only Debian and Ubuntu supported at the moment. + when: not(ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu') - name: install base dependencies apt: From bf3ffc29a9aee03a160612032fd78f31226732bb Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 20:59:49 +0100 Subject: [PATCH 029/371] Make role compatible with ansible 2.7 Recursive remote copy is supported starting with 2.8 only Indentation behaviour in literal yaml strings seems to have changed Regex logic for ImageMagic was flawed (no idea why this worked before) --- ansible/tasks/main.yml | 69 +++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index c94534895..b1932a33c 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -28,6 +28,7 @@ - pngquant - zlib1g # dev + - sudo - build-essential - python3-setuptools - python3-wheel @@ -122,23 +123,19 @@ dest: "{{ tempdir.path }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' -- name: change permissions of paperless-ng +- name: change owner and permissions of paperless-ng command: cmd: "{{ item }}" + warn: false with_items: + - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}" - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - copy: - src: "{{ tempdir.path }}/paperless-ng/" - remote_src: yes - dest: "{{ paperlessng_directory }}" - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: preserve - directory_mode: preserve + command: + cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -256,30 +253,31 @@ become: yes become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs + vars: + creation_script: | + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " - from django.contrib.auth.models import User - from django.contrib.auth.hashers import get_hasher - - if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): - user = User.objects.get(username='{{ paperlessng_superuser_name }}') - old = user.__dict__.copy() - - user.is_superuser = True - user.email = '{{ paperlessng_superuser_email }}' - user.set_password('{{ paperlessng_superuser_password }}') - user.save() - new = user.__dict__ - - algorithm, iterations, old_salt, old_hash = old['password'].split('$') - new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) - _, _, _, new_hash = new_password_old_salt.split('$') - if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): - print('changed') - else: - User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') - print('changed') - " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" args: chdir: "{{ paperlessng_directory }}/src" register: superuser @@ -298,9 +296,10 @@ - name: configure ghostscript for PDF lineinfile: - path: "/etc/ImageMagick-6/policy.xml" - regexp: '' - line: '' + path: /etc/ImageMagick-6/policy.xml + regexp: '(\s+)' + line: '\1' + backrefs: yes - name: configure systemd services ini_file: From f075384b449aa390926dae1c8f0e4e4f7e411146 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 21:55:59 +0100 Subject: [PATCH 030/371] Move paperless.conf to /etc, drop permissions --- ansible/tasks/main.yml | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b1932a33c..2ee2e4db2 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -158,9 +158,14 @@ - "{{ paperlessng_media_root }}" - "{{ paperlessng_static_dir }}" +- name: rename initial config + command: + cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template" + removes: "{{ paperlessng_directory }}/paperless.conf" + - name: configure paperless-ng lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -184,18 +189,18 @@ # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + no_log: yes - name: configure paperless-ng database [sqlite] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "^#?PAPERLESS_DBHOST=" state: absent when: paperlessng_db_type == 'sqlite' - name: configure paperless-ng database [postgresql] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -210,7 +215,17 @@ - regexp: "^#?PAPERLESS_DBPASS=" line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" when: paperlessng_db_type == 'postgresql' - no_log: true + no_log: yes + +- name: deploy paperless-ng configuration + copy: + src: "{{ paperlessng_directory }}/paperless.conf.template" + remote_src: yes + dest: /etc/paperless.conf + owner: root + group: root + mode: '0644' + register: configuration - name: create paperlessng venv become: yes @@ -232,9 +247,7 @@ - name: collect static files become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input" when: paperlessng_current_version.stdout != paperlessng_version | string register: static_files changed_when: "'188 unmodified' not in static_files.stdout" @@ -242,9 +255,7 @@ - name: create database schema become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' @@ -276,13 +287,10 @@ else: User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') - command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\"" register: superuser changed_when: superuser.stdout == 'changed' - no_log: true + no_log: yes - name: set ownership and permissions on paperlessng venv file: @@ -372,7 +380,7 @@ - paperless-consumer - paperless-scheduler - paperless-webserver - when: paperless_services.changed + when: paperless_services.changed or configuration.changed - name: enable paperlessng services systemd: From bb569b4e7875298049efe3e4c9b4cf29018267f9 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 22:43:52 +0100 Subject: [PATCH 031/371] Integrate OCRmyPDF args into ansible config --- ansible/defaults/main.yml | 11 ++++++----- ansible/tasks/main.yml | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index fbeef9871..2504f18dc 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -11,12 +11,13 @@ paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" paperlessng_ocr_languages: - eng paperlessng_time_zone: Europe/Berlin -paperlessng_ocrmypdf_args: --optimize 1 -# TODO Does optimze==1 really work with jbig2enc? -# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 -# Documentation states -O1 only applies lossless transformations -# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +# see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr +paperlessng_ocrmypdf_args: + - "deskew": true + - "clean": true + - "optimize": 1 paperlessng_use_jbig2enc: true +paperlessng_big2enc_lossy: false paperlessng_superuser_name: paperlessng paperlessng_superuser_email: paperlessng@example.com diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 2ee2e4db2..4a243f322 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,10 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - # - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # # TODO JSON dict required in conf - # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From 14f87f5aee52753ada73ffcee8ab1f110e25af4c Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:30:59 +0100 Subject: [PATCH 032/371] Harden systemd service files, drop perms further --- ansible/tasks/main.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 4a243f322..f0fd84d67 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -310,7 +310,7 @@ - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" - section: "{{ item[1].section }}" + section: "Service" option: "{{ item[1].option }}" value: "{{ item[1].value }}" with_nested: @@ -320,21 +320,35 @@ paperless-webserver.service, ] - [ + # https://www.freedesktop.org/software/systemd/man/systemd.exec.html { - section: "Service", option: "User", value: "{{ paperlessng_system_user }}", }, { - section: "Service", option: "Group", value: "{{ paperlessng_system_group }}", }, { - section: "Service", option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", }, + { + option: "ProtectSystem", + value: "full", + }, + { + option: "NoNewPrivileges", + value: "true", + }, + { + option: "PrivateUsers", + value: "true", + }, + { + option: "PrivateDevices", + value: "true", + } ] - name: configure paperless-consumer service From e4e4efcba7e287d7133ea194d5785d88ff937963 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:54:22 +0100 Subject: [PATCH 033/371] Fix creation of user arg json.loads is picky in that is expects true json, not yaml from ansible --- ansible/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index f0fd84d67..604b29203 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,8 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" + - regexp: "^#PAPERLESS_OCR_USER_ARGS=" + line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From a74404170ddd2a9cd66d0e46c58b041ca9841bb4 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 09:53:29 +0100 Subject: [PATCH 034/371] Update to 0.9.10 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 2504f18dc..adfdb2b87 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.9 +paperlessng_version: 0.9.10 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From be56707a8a508953d605d4f6bb554ef27646dba1 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 10:01:57 +0100 Subject: [PATCH 035/371] Change default OCRmyPDF args Clean is hardcoded as True, anyway Deskew breaks tesseract redo --- ansible/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index adfdb2b87..a2c560701 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -13,8 +13,7 @@ paperlessng_ocr_languages: paperlessng_time_zone: Europe/Berlin # see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr paperlessng_ocrmypdf_args: - - "deskew": true - - "clean": true + #- "deskew": true - "optimize": 1 paperlessng_use_jbig2enc: true paperlessng_big2enc_lossy: false From fc31195fa0bd307e2a640dce7ef98cbc6e3df7fb Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:47:54 +0100 Subject: [PATCH 036/371] Update to 0.9.11 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index a2c560701..da38ca300 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.10 +paperlessng_version: 0.9.11 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From b855222ee150e419930054ad3ea750a584c2b58b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:24:57 +0100 Subject: [PATCH 037/371] Removing travis in favor of github actions --- .travis.yml | 153 ---------------------------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f4873cf3c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,153 +0,0 @@ -language: python - -dist: focal -os: linux - -stages: - - name: test - - name: build_docker - if: branch =~ /^(master|dev|ng-.*)$/ - - name: publish_manifest - if: branch =~ /^(master|dev|ng-.*)$/ - -jobs: - include: - - name: "Paperless on Python 3.6" - stage: test - python: "3.6" - - - name: "Paperless on Python 3.7" - stage: test - python: "3.7" - - - name: "Paperless on Python 3.8" - stage: test - python: "3.8" - - - name: "Documentation" - stage: test - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - stage: test - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true - - - stage: build_docker - name: amd64 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - - stage: build_docker - name: arm64v8 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64-graviton2 - virt: vm - group: edge - - stage: build_docker - name: arm32v7 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - - stage: publish_manifest - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} - - | - if [ "${TRAVIS_BRANCH}" = "master" ]; then - echo "Master branch detected" - DOCKER_TAG="latest" - else - DOCKER_TAG=${TRAVIS_TAG} - fi - - | - if [ "${DOCKER_TAG}" != "" ]; then - echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} - else - echo "Not a tag and not on master, so not pushing tag/master specific manifest" - fi - - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - -install: - - pip install --upgrade pipenv - - pipenv install --system --dev - -script: - - cd src/ - - pipenv run pytest --cov - - pipenv run pycodestyle - -after_success: - - pipenv run coveralls From 916ea79e5749f295ad4652fe46862015cbd259c7 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:27:22 +0100 Subject: [PATCH 038/371] Experimenting with github actions --- .github/workflows/docker-publish.yml | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..f2da6b3b8 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,63 @@ +name: Docker + +on: + push: + branches: travis-multiarch-builds + +jobs: + # Build and push image to docker hub. + buildx: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + fi + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./Dockerfile . + - name: install buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v3.1.0 + + - name: Docker Buildx (build) + run: | + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Login + if: success() && github.event_name != 'pull_request' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + - name: Docker Buildx (push) + if: success() && github.event_name != 'pull_request' + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Check Manifest + if: always() && github.event_name != 'pull_request' + run: | + docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + - name: Clear + if: always() && github.event_name != 'pull_request' + run: | + rm -f ${HOME}/.docker/config.json \ No newline at end of file From 3f01a9e1e274105296fc52add82b6fd486abb988 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 19:32:04 +0100 Subject: [PATCH 039/371] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f2da6b3b8..5ee210ef2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,7 +15,7 @@ jobs: - name: Prepare id: prepare run: | - DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_IMAGE=moztr/paperless-ng DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then @@ -60,4 +60,4 @@ jobs: - name: Clear if: always() && github.event_name != 'pull_request' run: | - rm -f ${HOME}/.docker/config.json \ No newline at end of file + rm -f ${HOME}/.docker/config.json From 97d068c3bbd79002eca7147190ec0d61234f1278 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 1 Jan 2021 12:53:16 +0100 Subject: [PATCH 040/371] Removed separate build step It looks like part of the build step is repeated in the push step due to bad caching possibly. --- .github/workflows/docker-publish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5ee210ef2..53d53e25c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,10 +38,6 @@ jobs: - name: install buildx id: buildx uses: crazy-max/ghaction-docker-buildx@v3.1.0 - - - name: Docker Buildx (build) - run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} - name: Docker Login if: success() && github.event_name != 'pull_request' env: From 8ab251c5f7d86fce37bd92f6cb18e070c0f67221 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:04:46 +0100 Subject: [PATCH 041/371] Execute python tests --- .github/workflows/docker-publish.yml | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 53d53e25c..a2897c05b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -5,6 +5,47 @@ on: branches: travis-multiarch-builds jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python-version }} + - name: Update system, install system requirements + run: | + apt-get update -qq + apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + - name: Upgrade pip, Install dependencies + run: | + pip install --upgrade pip pipenv sphinx + pipenv lock -r > requirements.txt + pip install -r requirements.txt + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W + - name: Execute Tests + run: | + cd src/ + pytest --cov + pycodestyle + - name: Execute coveralls + run: | + coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 41c1b711c2d29692c4d366dd2ba789b91bb2c15c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:08:19 +0100 Subject: [PATCH 042/371] Use sudo for apt --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a2897c05b..f6ac55c1e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,8 +28,8 @@ jobs: key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | - apt-get update -qq - apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip, Install dependencies run: | pip install --upgrade pip pipenv sphinx From b8b0b61e67725a215e341f2ed967343b7e5cd81f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:15:10 +0100 Subject: [PATCH 043/371] Execute pip commands one after the other to find issue --- .github/workflows/docker-publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f6ac55c1e..a810f134f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -30,10 +30,14 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip, Install dependencies + - name: Upgrade pip run: | pip install --upgrade pip pipenv sphinx + - name: Gather pip requirements + run: | pipenv lock -r > requirements.txt + - name: Insall pip requirements + run: | pip install -r requirements.txt - name: Run sphinx run: | From ef28ae811767a51a94d1cf0b94584c852f9ab383 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:19:58 +0100 Subject: [PATCH 044/371] Move sphinx step below tests step as it used to be --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a810f134f..e020fb66b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,17 +36,17 @@ jobs: - name: Gather pip requirements run: | pipenv lock -r > requirements.txt - - name: Insall pip requirements + - name: Install pip requirements run: | pip install -r requirements.txt - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - name: Execute Tests run: | cd src/ pytest --cov pycodestyle + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W - name: Execute coveralls run: | coveralls From 4a45a77753addc28f9b845a51978d8729b1c2db8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:23:49 +0100 Subject: [PATCH 045/371] Add pytest to install step --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e020fb66b..c074b1d2f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From d292f710077ad6fca94f64902b3ed9a2dc70888a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:31:20 +0100 Subject: [PATCH 046/371] Add pytest-cov to install --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c074b1d2f..9bba2e031 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest + pip install --upgrade pip pipenv sphinx pytest pytest-cov - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 90f4ccfb941c4d973094d84b1dc8ce7f0605ad48 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:41:21 +0100 Subject: [PATCH 047/371] Remove cache to see if that helps with pytest-cov issues --- .github/workflows/docker-publish.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9bba2e031..090cfa937 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,15 +17,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | sudo apt-get update -qq From 4bea504690611de0340150a75b205f31d8638665 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:44:08 +0100 Subject: [PATCH 048/371] Try o make pipenv also install dev dependencies --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 090cfa937..24c3fe062 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov + pip install --upgrade pip pipenv sphinx - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 4684d199538c2b49ba5372f3db7bd99e99c8524b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:23:30 +0100 Subject: [PATCH 049/371] github-actions: readding explicit install explicitly install pytest, pytest-cov and coveralls. I think travis already did that and we need to explicitly do so now. removed installing dev-dependencies, do it explicitly. (just trying to reproduce the travis-ci build) --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 24c3fe062..afbb2571f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | - pipenv lock --dev -r > requirements.txt + pipenv lock -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 376efcde27f6f3c0b9f55695ea8013a2af63e650 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:27:38 +0100 Subject: [PATCH 050/371] github-actions: removing unsupported parameter I don't understand what the "-n auto" parameter does, pytest doesn't either (in the version installed) - so removing it to see if that produces something usable. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index 2a1a348bd..f43c9adf6 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html -n auto +addopts = --pythonwarnings=all --cov --cov-report=html env = PAPERLESS_DISABLE_DBHANDLER=true From 28209bb4746f048c42d2285aa36328e91bf57558 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:30:12 +0100 Subject: [PATCH 051/371] github-actions: also install wheel There's an error about pip wheel not being available and falling back to some legacy version. Let's see if this helps. --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index afbb2571f..a7a399c3f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,7 +23,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From a21fb36cd27e381d5297113ddb9ca442e138ca7c Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:33:56 +0100 Subject: [PATCH 052/371] github-actions: readding previously removed "-n auto" possibly this is fixed by the wheel dependency that was missing before. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index f43c9adf6..2a1a348bd 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html +addopts = --pythonwarnings=all --cov --cov-report=html -n auto env = PAPERLESS_DISABLE_DBHANDLER=true From 654ee4e62e774aad562bfd708fe873e21fbcee1c Mon Sep 17 00:00:00 2001 From: jayme-github Date: Sat, 2 Jan 2021 14:40:56 +0100 Subject: [PATCH 053/371] Add option to ignore certain dates in parse_date PAPERLESS_IGNORE_DATES allows to specify a comma separated list of dates to ignore during date parsing (from filename and content). This can be used so specify dates that do appear often in documents but are usually not the documents creation date (like your date of birth). --- src/documents/parsers.py | 15 +++++++++++---- src/documents/tests/test_date_parsing.py | 15 +++++++++++++++ src/paperless/settings.py | 8 ++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/documents/parsers.py b/src/documents/parsers.py index e14607bd0..cf413a449 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -209,6 +209,13 @@ def parse_date(filename, text): } ) + def __filter(date): + if date and date.year > 1900 and \ + date <= timezone.now() and \ + date.date() not in settings.IGNORE_DATES: + return date + return None + date = None # if filename date parsing is enabled, search there first: @@ -222,7 +229,8 @@ def parse_date(filename, text): # Skip all matches that do not parse to a proper date continue - if date and date.year > 1900 and date <= timezone.now(): + date = __filter(date) + if date is not None: return date # Iterate through all regex matches in text and try to parse the date @@ -235,10 +243,9 @@ def parse_date(filename, text): # Skip all matches that do not parse to a proper date continue - if date and date.year > 1900 and date <= timezone.now(): + date = __filter(date) + if date is not None: break - else: - date = None return date diff --git a/src/documents/tests/test_date_parsing.py b/src/documents/tests/test_date_parsing.py index 357b0937e..9cbb19c2b 100644 --- a/src/documents/tests/test_date_parsing.py +++ b/src/documents/tests/test_date_parsing.py @@ -138,3 +138,18 @@ class TestDate(TestCase): @override_settings(FILENAME_DATE_ORDER="YMD") def test_filename_date_parse_invalid(self, *args): self.assertIsNone(parse_date("/tmp/20 408000l 2475 - test.pdf", "No date in here")) + + @override_settings(IGNORE_DATES=(datetime.date(2019, 11, 3), datetime.date(2020, 1, 17))) + def test_ignored_dates(self, *args): + text = ( + "lorem ipsum 110319, 20200117 and lorem 13.02.2018 lorem " + "ipsum" + ) + date = parse_date("", text) + self.assertEqual( + date, + datetime.datetime( + 2018, 2, 13, 0, 0, + tzinfo=tz.gettz(settings.TIME_ZONE) + ) + ) \ No newline at end of file diff --git a/src/paperless/settings.py b/src/paperless/settings.py index e8b44e8cd..5191803d0 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -4,6 +4,7 @@ import multiprocessing import os import re +import dateparser from dotenv import load_dotenv from django.utils.translation import gettext_lazy as _ @@ -444,3 +445,10 @@ PAPERLESS_TIKA_ENDPOINT = os.getenv("PAPERLESS_TIKA_ENDPOINT", "http://localhost PAPERLESS_TIKA_GOTENBERG_ENDPOINT = os.getenv( "PAPERLESS_TIKA_GOTENBERG_ENDPOINT", "http://localhost:3000" ) + +# List dates that should be ignored when trying to parse date from document text +IGNORE_DATES = set() +for s in os.getenv("PAPERLESS_IGNORE_DATES", "").split(","): + d = dateparser.parse(s) + if d: + IGNORE_DATES.add(d.date()) From 74cf5373b969e1204845f7da1f5cc5962a4a362c Mon Sep 17 00:00:00 2001 From: "transifex-integration[bot]" <43880903+transifex-integration[bot]@users.noreply.github.com> Date: Sat, 2 Jan 2021 23:58:17 +0000 Subject: [PATCH 054/371] Apply translations in de at least 80% translated for the source file '/src/locale/en-us/LC_MESSAGES/django.po' on the 'de' language. Manual sync of partially translated files: untranslated content is included with an empty translation or source language content depending on file format From d93ec0d5c7e919b6d70c820e3bb0704fed3f9250 Mon Sep 17 00:00:00 2001 From: "transifex-integration[bot]" <43880903+transifex-integration[bot]@users.noreply.github.com> Date: Sat, 2 Jan 2021 23:58:23 +0000 Subject: [PATCH 055/371] Translate /src-ui/messages.xlf in de at least 80% translated for the source file '/src-ui/messages.xlf' on the 'de' language. Manual sync of partially translated files: untranslated content is included with an empty translation or source language content depending on file format From 7b56ad9dad620cb359b128e83b7a5aa49067dacb Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Sun, 3 Jan 2021 00:37:19 -0800 Subject: [PATCH 056/371] Allow authentication via HTTP_REMOTE_USER --- docs/configuration.rst | 18 ++++++++++++------ paperless.conf.example | 1 + src/paperless/auth.py | 19 +++++++++++++++++++ src/paperless/settings.py | 7 +++++++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 5ccb80b3a..c72027574 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -162,6 +162,12 @@ PAPERLESS_COOKIE_PREFIX= Defaults to ``""``, which does not alter the cookie names. +PAPERLESS_ENABLE_HTTP_REMOTE_USER= + Allows authentication via HTTP_REMOTE_USER which is used by some SSO + applications. + + Defaults to `false` which disables this feature. + .. _configuration-ocr: OCR settings @@ -210,20 +216,20 @@ PAPERLESS_OCR_MODE= into images and puts the OCRed text on top. This works for all documents, however, the resulting document may be significantly larger and text won't appear as sharp when zoomed in. - + The default is ``skip``, which only performs OCR when necessary and always creates archived documents. PAPERLESS_OCR_OUTPUT_TYPE= Specify the the type of PDF documents that paperless should produce. - + * ``pdf``: Modify the PDF document as little as possible. * ``pdfa``: Convert PDF documents into PDF/A-2b documents, which is a subset of the entire PDF specification and meant for storing documents long term. * ``pdfa-1``, ``pdfa-2``, ``pdfa-3`` to specify the exact version of PDF/A you wish to use. - + If not specified, ``pdfa`` is used. Remember that paperless also keeps the original input file as well as the archived version. @@ -275,9 +281,9 @@ PAPERLESS_OCR_USER_ARG= .. code:: json - {"deskew": true, "optimize": 3, "unpaper_args": "--pre-rotate 90"} - - + {"deskew": true, "optimize": 3, "unpaper_args": "--pre-rotate 90"} + + Software tweaks ############### diff --git a/paperless.conf.example b/paperless.conf.example index 139453cf3..c55b7f5f4 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -31,6 +31,7 @@ #PAPERLESS_STATIC_URL=/static/ #PAPERLESS_AUTO_LOGIN_USERNAME= #PAPERLESS_COOKIE_PREFIX= +#PAPERLESS_ENABLE_HTTP_REMOTE_USER=false # OCR settings diff --git a/src/paperless/auth.py b/src/paperless/auth.py index ece5d0eba..d92dc7671 100644 --- a/src/paperless/auth.py +++ b/src/paperless/auth.py @@ -2,6 +2,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.utils.deprecation import MiddlewareMixin from rest_framework import authentication +from rest_framework import exceptions class AutoLoginMiddleware(MiddlewareMixin): @@ -26,3 +27,21 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication): return (user, None) else: return None + + +class HttpRemoteUserAuthentication(authentication.BaseAuthentication): + """ This class allows authentication via HTTP_REMOTE_USER which is set for + example by certain SSO applications. + """ + + def authenticate(self, request): + username = request.META.get('HTTP_REMOTE_USER') + if not username: + return None + + try: + user = User.objects.get(username=username) + except User.DoesNotExist: + raise exceptions.AuthenticationFailed('No such user') + + return (user, None) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 5af1be85e..f522c4c0b 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -112,6 +112,13 @@ if DEBUG: 'paperless.auth.AngularApiAuthenticationOverride' ) +ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER") + +if ENABLE_HTTP_REMOTE_USER: + REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append( + 'paperless.auth.HttpRemoteUserAuthentication' + ) + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', From 2aa2086dfbdbdef5839f35e08d2cc8a996ca4446 Mon Sep 17 00:00:00 2001 From: jayme-github Date: Sun, 3 Jan 2021 14:35:28 +0100 Subject: [PATCH 057/371] Add missing config options to example file --- paperless.conf.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paperless.conf.example b/paperless.conf.example index d9d0f5b06..2e46b91de 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -50,11 +50,14 @@ #PAPERLESS_TIME_ZONE=UTC #PAPERLESS_CONSUMER_POLLING=10 #PAPERLESS_CONSUMER_DELETE_DUPLICATES=false +#PAPERLESS_CONSUMER_RECURSIVE=false +#PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false #PAPERLESS_OPTIMIZE_THUMBNAILS=true #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh #PAPERLESS_FILENAME_DATE_ORDER=YMD #PAPERLESS_FILENAME_PARSE_TRANSFORMS=[] #PAPERLESS_THUMBNAIL_FONT_NAME= +#PAPERLESS_IGNORE_DATES= # Binaries From be2061b74d9ad99a4caf0568aa49275bc5eb2eb7 Mon Sep 17 00:00:00 2001 From: jayme-github Date: Sun, 3 Jan 2021 14:47:04 +0100 Subject: [PATCH 058/371] Add PAPERLESS_IGNORE_DATES to docs --- docs/configuration.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 49c95bff1..5036d6aa4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -438,6 +438,19 @@ PAPERLESS_THUMBNAIL_FONT_NAME= Defaults to ``/usr/share/fonts/liberation/LiberationSerif-Regular.ttf``. +PAPERLESS_IGNORE_DATES= + Paperless parses a documents creation date from filename and file content. + You may specify a comma separated list of dates that should be ignored during + this process. This is useful for special dates (like date of birth) that appear + in documents regularly but are very unlikely to be the documents creation date. + + You may specify dates in a multitude of formats supported by dateparser (see + https://dateparser.readthedocs.io/en/latest/#popular-formats) but as the dates + need to be comma separated, the options are limited. + Example: "2020-12-02,22.04.1999" + + Defaults to an empty string to not ignore any dates. + Binaries ######## From 426ad30a52a5326d808ab1ba242d97ab85cc9d4d Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Sun, 3 Jan 2021 21:21:39 -0800 Subject: [PATCH 059/371] Refactor to extend RemoteUserMiddleware & add authentication for Django --- src/paperless/auth.py | 16 +++------------- src/paperless/settings.py | 21 ++++++++++++++------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/paperless/auth.py b/src/paperless/auth.py index d92dc7671..cd717e56b 100644 --- a/src/paperless/auth.py +++ b/src/paperless/auth.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.utils.deprecation import MiddlewareMixin from rest_framework import authentication -from rest_framework import exceptions +from django.contrib.auth.middleware import RemoteUserMiddleware class AutoLoginMiddleware(MiddlewareMixin): @@ -29,19 +29,9 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication): return None -class HttpRemoteUserAuthentication(authentication.BaseAuthentication): +class HttpRemoteUserMiddleware(RemoteUserMiddleware): """ This class allows authentication via HTTP_REMOTE_USER which is set for example by certain SSO applications. """ - def authenticate(self, request): - username = request.META.get('HTTP_REMOTE_USER') - if not username: - return None - - try: - user = User.objects.get(username=username) - except User.DoesNotExist: - raise exceptions.AuthenticationFailed('No such user') - - return (user, None) + header = 'HTTP_REMOTE_USER' diff --git a/src/paperless/settings.py b/src/paperless/settings.py index dd0d4a7d9..afbc667e0 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -115,13 +115,6 @@ if DEBUG: 'paperless.auth.AngularApiAuthenticationOverride' ) -ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER") - -if ENABLE_HTTP_REMOTE_USER: - REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append( - 'paperless.auth.HttpRemoteUserAuthentication' - ) - MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', @@ -135,6 +128,20 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] +ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER") + +if ENABLE_HTTP_REMOTE_USER: + MIDDLEWARE.append( + 'paperless.auth.HttpRemoteUserMiddleware' + ) + AUTHENTICATION_BACKENDS = [ + 'django.contrib.auth.backends.RemoteUserBackend', + 'django.contrib.auth.backends.ModelBackend' + ] + REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append( + 'rest_framework.authentication.RemoteUserAuthentication' + ) + ROOT_URLCONF = 'paperless.urls' FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME") From 51fd19c3156b5056db8478ea525e5f2d07255631 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:48:05 +0100 Subject: [PATCH 060/371] Switch to github actions --- .github/workflows/ansible.yml | 33 +++++++++++++++++++++++++++++++++ .travis.yml | 10 ---------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/ansible.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml new file mode 100644 index 000000000..41c13c774 --- /dev/null +++ b/.github/workflows/ansible.yml @@ -0,0 +1,33 @@ +--- +name: Ansible Role + +on: push + +jobs: + test: + # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + runs-on: ubuntu-latest + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/heads/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Set up Docker + uses: docker-practice/actions-setup-docker@master + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install molecule[ansible,docker] + ansible --version + docker --version + molecule --version + python --version + - name: Test with molecule + run: | + cd ansible + molecule test + working-directory: "${{ github.repository }}" diff --git a/.travis.yml b/.travis.yml index e36b1ca99..b745d6bd7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,16 +33,6 @@ jobs: - ng build --prod after_success: true - - name: "Ansible role" - sudo: required - services: - - docker - install: - - python3 -m pip install molecule[ansible,docker] - script: - - cd ansible/ - - molecule test - after_success: true before_install: - sudo apt-get update -qq From 586db3bc5e9bdd7962721f8bc0efee2af1e178a2 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 13:21:42 +0100 Subject: [PATCH 061/371] Deploy role to ansible galaxy Happens only when a new tag is pushed --- .github/workflows/ansible.yml | 32 +++++++++++++++++++++++++---- ansible/README.md | 38 +++++++++++++++++++++++++++++++++++ ansible/meta/main.yml | 17 ++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 ansible/README.md create mode 100644 ansible/meta/main.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 41c13c774..67383ecf8 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -1,14 +1,14 @@ --- name: Ansible Role -on: push +on: [push, pull_request] jobs: - test: # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + test: runs-on: ubuntu-latest # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context - if: contains(github.ref, 'refs/heads/') + if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) steps: - name: Check out the codebase uses: actions/checkout@v2 @@ -16,7 +16,7 @@ jobs: path: "${{ github.repository }}" - name: Set up Python uses: actions/setup-python@v2 - - name: Set up Docker + - name: Set up Docker uses: docker-practice/actions-setup-docker@master - name: Install dependencies run: | @@ -31,3 +31,27 @@ jobs: cd ansible molecule test working-directory: "${{ github.repository }}" + # https://galaxy.ansible.com/docs/contributing/importing.html + release: + runs-on: ubuntu-latest + needs: + - test + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/tags/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python3 -m pip install --upgrade ansible-base + ansible --version + python --version + - name: Trigger a new import on Galaxy + run: | + cd ansible + ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) + working-directory: "${{ github.repository }}" diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/ansible/meta/main.yml b/ansible/meta/main.yml new file mode 100644 index 000000000..78a4c1a41 --- /dev/null +++ b/ansible/meta/main.yml @@ -0,0 +1,17 @@ +dependencies: [] + +galaxy_info: + author: C0nsultant + description: Bare-metal deployment of paperless-ng DMS + license: license (GPLv3) + min_ansible_version: 2.7 + + platforms: + - name: Debian + versions: + - buster + - name: Ubuntu + versions: + - focal + + galaxy_tags: [EDMS, django, python, web] From 7e36986a26110373f26559818e44d2282f260837 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Mon, 4 Jan 2021 00:20:10 -0800 Subject: [PATCH 062/371] Clearing filters on saved views should reset them to initial rules --- .../document-list.component.html | 4 +- .../document-list/document-list.component.ts | 45 +++++++++++++++---- .../filter-editor.component.html | 6 +-- .../filter-editor/filter-editor.component.ts | 28 +++++------- 4 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index e4d7256e1..90893baee 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -5,7 +5,7 @@  Select - +
@@ -78,7 +78,7 @@
- +
diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index 1f29cb901..685d40a44 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -30,10 +30,6 @@ export class DocumentListComponent implements OnInit { displayMode = 'smallCards' // largeCards, smallCards, details - get isFiltered() { - return this.list.filterRules?.length > 0 - } - getTitle() { return this.list.savedViewTitle || $localize`Documents` } @@ -62,18 +58,16 @@ export class DocumentListComponent implements OnInit { this.router.navigate(["404"]) return } - this.list.savedView = view - this.list.reload() }) } else { this.list.savedView = null - this.list.reload() } + + this.list.reload() }) } - loadViewConfig(view: PaperlessSavedView) { this.list.load(view) this.list.reload() @@ -99,6 +93,7 @@ export class DocumentListComponent implements OnInit { sort_reverse: this.list.sortReverse, sort_field: this.list.sortField } + this.savedViewService.create(savedView).subscribe(() => { modal.close() this.toastService.showInfo($localize`View "${savedView.name}" created successfully.`) @@ -109,6 +104,40 @@ export class DocumentListComponent implements OnInit { }) } + resetFilters(): void { + if (this.list.savedViewId) { + this.savedViewService.getCached(this.list.savedViewId).subscribe(viewUntouched => { + this.list.filterRules = viewUntouched.filter_rules + }) + } else { + this.list.filterRules = [] + } + this.list.reload() + } + + get filterRulesModified(): boolean { + if (this.list.savedView == null) { + return this.list.filterRules.length > 0 // documents list is modified if it has any filters + } else { + // compare savedView current filters vs original + let modified = false + this.savedViewService.getCached(this.list.savedViewId).subscribe(view => { + let filterRulesInitial = view.filter_rules + + if (this.list.filterRules.length !== filterRulesInitial.length) modified = true + else { + this.list.filterRules.forEach(rule => { + if (filterRulesInitial.find(fri => fri.rule_type == rule.rule_type && fri.value == rule.value) == undefined) modified = true + }) + filterRulesInitial.forEach(rule => { + if (this.list.filterRules.find(fr => fr.rule_type == rule.rule_type && fr.value == rule.value) == undefined) modified = true + }) + } + }) + return modified + } + } + clickTag(tagID: number) { this.list.selectNone() setTimeout(() => { diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html index efbf6ce7e..02a51238d 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html @@ -41,11 +41,11 @@
-
diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index e6565beac..febb9dd84 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -40,7 +40,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy { case FILTER_HAS_TAG: return $localize`Tag: ${this.tags.find(t => t.id == +rule.value)?.name}` - + case FILTER_HAS_ANY_TAG: if (rule.value == "false") { return $localize`Without any tag` @@ -127,7 +127,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy { } else { this.tagSelectionModel.getSelectedItems().filter(tag => tag.id).forEach(tag => { filterRules.push({rule_type: FILTER_HAS_TAG, value: tag.id?.toString()}) - }) + }) } this.correspondentSelectionModel.getSelectedItems().forEach(correspondent => { filterRules.push({rule_type: FILTER_CORRESPONDENT, value: correspondent.id?.toString()}) @@ -153,16 +153,16 @@ export class FilterEditorComponent implements OnInit, OnDestroy { @Output() filterRulesChange = new EventEmitter() + @Output() + reset = new EventEmitter() + + @Input() + rulesModified: boolean = false + updateRules() { this.filterRulesChange.next(this.filterRules) } - hasFilters() { - return this._titleFilter || - this.dateAddedAfter || this.dateAddedBefore || this.dateCreatedAfter || this.dateCreatedBefore || - this.tagSelectionModel.selectionSize() || this.correspondentSelectionModel.selectionSize() || this.documentTypeSelectionModel.selectionSize() - } - get titleFilter() { return this._titleFilter } @@ -194,16 +194,8 @@ export class FilterEditorComponent implements OnInit, OnDestroy { this.titleFilterDebounce.complete() } - clearSelected() { - this._titleFilter = "" - this.tagSelectionModel.clear(false) - this.documentTypeSelectionModel.clear(false) - this.correspondentSelectionModel.clear(false) - this.dateAddedBefore = null - this.dateAddedAfter = null - this.dateCreatedBefore = null - this.dateCreatedAfter = null - this.updateRules() + resetSelected() { + this.reset.next() } toggleTag(tagId: number) { From 2e641b0e3037fe76044a51129c0dc4b08624f6c2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:18:56 +0100 Subject: [PATCH 063/371] Explicitly set ubuntu-20.04 as OS and install dev dependencies --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a7a399c3f..d29fd7740 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,8 +9,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] - python-version: [3.7, 3.8, 3.9] + os: [ubuntu-20.04] + python-version: 3.7 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From e91befabc06f0e611c5d6a63d9df35d8b346bd36 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:22:35 +0100 Subject: [PATCH 064/371] Revert job strategy matrix change --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d29fd7740..cfda3d10d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: 3.7 + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 05f57249d6fdfa89fba0e3966528a62116d0dbde Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:24:48 +0100 Subject: [PATCH 065/371] Only build python 3.7 and say so in the Pipfile too --- .github/workflows/docker-publish.yml | 2 +- Pipfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cfda3d10d..0d2c8bd68 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7, 3.8, 3.9] + python-version: [3.7] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/Pipfile b/Pipfile index 48759307c..e0f1d83d8 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.6" +python_version = "3.7" [packages] dateparser = "~=0.7.6" From 69ac7b9bcf0efbfeadbfe4210c991e601674ce7e Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:00:35 +0100 Subject: [PATCH 066/371] Revert python version in Pipfile to 3.6 --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index e0f1d83d8..48759307c 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.7" +python_version = "3.6" [packages] dateparser = "~=0.7.6" From c9de5928320a74ad016128bfae5531c0220d208d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:10:16 +0100 Subject: [PATCH 067/371] Try to reproduce exactly what's done in the travis build to test --- .github/workflows/docker-publish.yml | 30 ++++++++-------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0d2c8bd68..66bc73ea8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,37 +10,23 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Update system, install system requirements + - name: Tests run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip - run: | - pip install --upgrade pip pipenv - - name: Gather pip requirements - run: | - pipenv lock --dev -r > requirements.txt - - name: Install pip requirements - run: | - pip install -r requirements.txt - - name: Execute Tests - run: | + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev cd src/ - pytest --cov - pycodestyle - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - - name: Execute coveralls - run: | - coveralls + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 7e99d7515f82dfbd3de680a3250796bcd907c7cb Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:17:13 +0100 Subject: [PATCH 068/371] Try manually replacing the python-version --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 66bc73ea8..264552168 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,6 +21,8 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + echo "Replacing python version with ${matrix.python-version}" + sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From b41dd2780e4e6cb50a9eb77f5d1034f365a13b08 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:19:00 +0100 Subject: [PATCH 069/371] Print the env --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 264552168..9904d8623 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -19,6 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | + env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng echo "Replacing python version with ${matrix.python-version}" From 336b908e8f57b41be05cf69cf024c6bc62bb627f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:28:47 +0100 Subject: [PATCH 070/371] Only build for 3.6 --- .github/workflows/docker-publish.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9904d8623..23c8aef20 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6, 3.7, 3.8] + python-version: [3.6] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -19,11 +19,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | - env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - echo "Replacing python version with ${matrix.python-version}" - sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From 574ab7d696bdcda4967cbf22d41b8e0ad679c8e1 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:44:02 +0100 Subject: [PATCH 071/371] Add GITHUB_TOKEN to python tests --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 23c8aef20..01bf72595 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From b0588321e0ff3d38b4272f0bb8e65124e3e04807 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:45:12 +0100 Subject: [PATCH 072/371] Fix yaml --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 01bf72595..c0a1565c6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,7 +18,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From 056b9638abf7fa5aca75e3783b5fcf9b25ee631e Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Mon, 4 Jan 2021 19:31:18 -0800 Subject: [PATCH 073/371] Fix some inconsistent elements for dark mode --- src-ui/src/theme_dark.scss | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src-ui/src/theme_dark.scss b/src-ui/src/theme_dark.scss index 88c2b8014..e65d4564e 100644 --- a/src-ui/src/theme_dark.scss +++ b/src-ui/src/theme_dark.scss @@ -174,6 +174,17 @@ $border-color-dark-mode: #47494f; color: $text-color-dark-mode; border-color: $border-color-dark-mode; + .des, + .asc { + background-color: transparent !important; + color: $text-color-dark-mode; + border-color: $border-color-dark-mode; + + &::after { + filter: invert(0.8); /* arrow is a black inline png bkgd image (!) so use filter */ + } + } + tr:hover { background-color: $bg-light-dark-mode; color: $text-color-dark-mode-accent; @@ -250,13 +261,18 @@ $border-color-dark-mode: #47494f; background-color: $bg-dark-mode !important; } - .form-control, + .form-control:not(.is-invalid):not(.btn), + input:not(.is-invalid), + textarea:not(.is-invalid) { + border-color: $border-color-dark-mode; /* we dont want to override controls that get highlighting for errors */ + } + + .form-control:not(.btn), input, select, textarea { background-color: $bg-dark-mode; color: $text-color-dark-mode; - border-color: $border-color-dark-mode; &::placeholder { color: $text-color-dark-mode; @@ -325,6 +341,12 @@ $border-color-dark-mode: #47494f; .progress { background-color: $border-color-dark-mode; } + + .alert-danger { + color: $text-color-dark-mode-accent; + background-color: darken($danger-dark-mode, 20%); + border-color: darken($danger-dark-mode, 20%); + } } body.color-scheme-dark { From 73682d22d6e5bbebda79856c9850917f2b0938b2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 13:50:27 +0100 Subject: [PATCH 074/371] test cases --- .../commands/document_thumbnails.py | 10 +++- .../tests/test_management_thumbnails.py | 52 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 src/documents/tests/test_management_thumbnails.py diff --git a/src/documents/management/commands/document_thumbnails.py b/src/documents/management/commands/document_thumbnails.py index 01df14624..6f6451599 100644 --- a/src/documents/management/commands/document_thumbnails.py +++ b/src/documents/management/commands/document_thumbnails.py @@ -13,8 +13,14 @@ from ...parsers import get_parser_class_for_mime_type def _process_document(doc_in): document = Document.objects.get(id=doc_in) - parser = get_parser_class_for_mime_type(document.mime_type)( - logging_group=None) + parser_class = get_parser_class_for_mime_type(document.mime_type) + + if parser_class: + parser = parser_class(logging_group=None) + else: + print(f"{document} No parser for mime type {document.mime_type}") + return + try: thumb = parser.get_optimised_thumbnail( document.source_path, document.mime_type) diff --git a/src/documents/tests/test_management_thumbnails.py b/src/documents/tests/test_management_thumbnails.py new file mode 100644 index 000000000..7ecdf0489 --- /dev/null +++ b/src/documents/tests/test_management_thumbnails.py @@ -0,0 +1,52 @@ +import os +import shutil +from unittest import mock + +from django.core.management import call_command +from django.test import TestCase + +from documents.management.commands.document_thumbnails import _process_document +from documents.models import Document, Tag, Correspondent, DocumentType +from documents.tests.utils import DirectoriesMixin + + +class TestMakeThumbnails(DirectoriesMixin, TestCase): + + def make_models(self): + self.d1 = Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf", filename="test.pdf") + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), self.d1.source_path) + + self.d2 = Document.objects.create(checksum="Ass", title="A", content="first document", mime_type="application/pdf", filename="test2.pdf") + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), self.d2.source_path) + + def setUp(self) -> None: + super(TestMakeThumbnails, self).setUp() + self.make_models() + + def test_process_document(self): + self.assertFalse(os.path.isfile(self.d1.thumbnail_path)) + _process_document(self.d1.id) + self.assertTrue(os.path.isfile(self.d1.thumbnail_path)) + + @mock.patch("documents.management.commands.document_thumbnails.shutil.move") + def test_process_document_invalid_mime_type(self, m): + self.d1.mime_type = "asdasdasd" + self.d1.save() + + _process_document(self.d1.id) + + m.assert_not_called() + + def test_command(self): + self.assertFalse(os.path.isfile(self.d1.thumbnail_path)) + self.assertFalse(os.path.isfile(self.d2.thumbnail_path)) + call_command('document_thumbnails') + self.assertTrue(os.path.isfile(self.d1.thumbnail_path)) + self.assertTrue(os.path.isfile(self.d2.thumbnail_path)) + + def test_command_documentid(self): + self.assertFalse(os.path.isfile(self.d1.thumbnail_path)) + self.assertFalse(os.path.isfile(self.d2.thumbnail_path)) + call_command('document_thumbnails', '-d', f"{self.d1.id}") + self.assertTrue(os.path.isfile(self.d1.thumbnail_path)) + self.assertFalse(os.path.isfile(self.d2.thumbnail_path)) From ac2cac6edc4f25451b45a6a27862cc901fe0f02d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 14:57:56 +0100 Subject: [PATCH 075/371] fix missing translation. --- .../document-list.component.html | 24 +++++-------------- .../tag-edit-dialog.component.html | 2 +- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index bb687632d..8f4ae1b6c 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -104,49 +104,37 @@ [currentSortField]="list.sortField" [currentSortReverse]="list.sortReverse" (sort)="onSort($event)" - i18n> - ASN - + i18n>ASN - Correspondent - + i18n>Correspondent - Title - + i18n>Title - Document type - + i18n>Document type - Created - + i18n>Created - Added - + i18n>Added diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html index ad40659fb..57f3b19fd 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html @@ -6,7 +6,7 @@
- +
From 0ee6426eb53a06709998c19c63e7a01c522ea640 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 22:11:42 +0100 Subject: [PATCH 076/371] fixes #278 --- .../src/app/components/common/input/abstract-input.ts | 11 ++++++++++- .../components/common/input/text/text.component.html | 2 +- .../document-detail/document-detail.component.html | 2 +- .../document-detail/document-detail.component.ts | 7 ++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src-ui/src/app/components/common/input/abstract-input.ts b/src-ui/src/app/components/common/input/abstract-input.ts index f6039d2ec..5821b6682 100644 --- a/src-ui/src/app/components/common/input/abstract-input.ts +++ b/src-ui/src/app/components/common/input/abstract-input.ts @@ -1,10 +1,13 @@ -import { Directive, Input, OnInit } from '@angular/core'; +import { Directive, ElementRef, Input, OnInit, ViewChild } from '@angular/core'; import { ControlValueAccessor } from '@angular/forms'; import { v4 as uuidv4 } from 'uuid'; @Directive() export class AbstractInputComponent implements OnInit, ControlValueAccessor { + @ViewChild("inputField") + inputField: ElementRef + constructor() { } onChange = (newValue: T) => {}; @@ -24,6 +27,12 @@ export class AbstractInputComponent implements OnInit, ControlValueAccessor { this.disabled = isDisabled; } + focus() { + if (this.inputField && this.inputField.nativeElement) { + this.inputField.nativeElement.focus() + } + } + @Input() title: string diff --git a/src-ui/src/app/components/common/input/text/text.component.html b/src-ui/src/app/components/common/input/text/text.component.html index 78aa76577..fe8607e61 100644 --- a/src-ui/src/app/components/common/input/text/text.component.html +++ b/src-ui/src/app/components/common/input/text/text.component.html @@ -1,6 +1,6 @@
- + {{hint}}
{{error}} diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index a724ae031..dbd1c0ba7 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -56,7 +56,7 @@ Details - + { this.networkActive = false From 63d2dcc1f72c6dd8606408ea1a1d016843f3d5af Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 23:36:31 +0100 Subject: [PATCH 077/371] fix some translations --- .../common/confirm-dialog/confirm-dialog.component.html | 2 +- .../src/app/components/common/input/tags/tags.component.html | 2 +- .../components/document-detail/document-detail.component.html | 2 +- .../app/components/manage/settings/settings.component.html | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html index 565829f21..8a8a297d9 100644 --- a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html @@ -9,7 +9,7 @@

{{message}}

- + + From 6e84668884c63f66aec379ce4c0c712c81a44f0c Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 6 Jan 2021 01:16:16 +0100 Subject: [PATCH 078/371] fix login/logout pages --- src/documents/templates/registration/logged_out.html | 2 +- src/documents/templates/registration/login.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/documents/templates/registration/logged_out.html b/src/documents/templates/registration/logged_out.html index 4a3cb60bb..1448b641e 100644 --- a/src/documents/templates/registration/logged_out.html +++ b/src/documents/templates/registration/logged_out.html @@ -36,7 +36,7 @@ diff --git a/src/documents/templates/registration/login.html b/src/documents/templates/registration/login.html index 25a707bed..7f1fc79dd 100644 --- a/src/documents/templates/registration/login.html +++ b/src/documents/templates/registration/login.html @@ -37,7 +37,7 @@