From d702147b77b3f2716c8d69b31e5fea5371506c04 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 15:58:29 +0100 Subject: [PATCH 001/369] reorganized docker build. --- docker/local/Dockerfile => Dockerfile | 17 ++- ...ostgres.yml => docker-compose.postgres.yml | 0 ...se.sqlite.yml => docker-compose.sqlite.yml | 0 docker/docker-compose.env | 34 ----- scripts/make-release.sh | 125 ------------------ scripts/push-release.sh | 23 ---- 6 files changed, 14 insertions(+), 185 deletions(-) rename docker/local/Dockerfile => Dockerfile (78%) rename docker/local/docker-compose.postgres.yml => docker-compose.postgres.yml (100%) rename docker/local/docker-compose.sqlite.yml => docker-compose.sqlite.yml (100%) delete mode 100644 docker/docker-compose.env delete mode 100755 scripts/make-release.sh delete mode 100755 scripts/push-release.sh diff --git a/docker/local/Dockerfile b/Dockerfile similarity index 78% rename from docker/local/Dockerfile rename to Dockerfile index 461b9e4fc..732b4183c 100644 --- a/docker/local/Dockerfile +++ b/Dockerfile @@ -1,8 +1,18 @@ +FROM node:15 AS frontend + +WORKDIR /usr/src/paperless/src-ui/ + +COPY src-ui/package* ./ +RUN npm install + +COPY src-ui . +RUN node_modules/.bin/ng build --prod --output-hashing none --sourceMap=false + FROM python:3.7-slim WORKDIR /usr/src/paperless/ -COPY requirements.txt ./ +COPY Pipfile* ./ #Dependencies RUN apt-get update \ @@ -33,8 +43,8 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools \ - && pip install --no-cache-dir -r requirements.txt \ + && pip3 install --upgrade supervisor setuptools pipenv \ + && pipenv install --system --deploy --ignore-pipfile --clear \ && apt-get -y purge build-essential libqpdf-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ @@ -49,6 +59,7 @@ COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh # copy app COPY src/ ./src/ +COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ # add users, setup scripts RUN addgroup --gid 1000 paperless \ diff --git a/docker/local/docker-compose.postgres.yml b/docker-compose.postgres.yml similarity index 100% rename from docker/local/docker-compose.postgres.yml rename to docker-compose.postgres.yml diff --git a/docker/local/docker-compose.sqlite.yml b/docker-compose.sqlite.yml similarity index 100% rename from docker/local/docker-compose.sqlite.yml rename to docker-compose.sqlite.yml diff --git a/docker/docker-compose.env b/docker/docker-compose.env deleted file mode 100644 index 4271bce6e..000000000 --- a/docker/docker-compose.env +++ /dev/null @@ -1,34 +0,0 @@ -# The UID and GID of the user used to run paperless in the container. Set this -# to your UID and GID on the host so that you have write access to the -# consumption directory. -#USERMAP_UID=1000 -#USERMAP_GID=1000 - -# Additional languages to install for text recognition, separated by a -# whitespace. Note that this is -# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the -# default language used when guessing the language from the OCR output. -# The container installs English, German, Italian, Spanish and French by -# default. -# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster -# for available languages. -#PAPERLESS_OCR_LANGUAGES=tur ces - -############################################################################### -# Paperless-specific settings # -############################################################################### - -# All settings defined in the paperless.conf.example can be used here. The -# Docker setup does not use the configuration file. -# A few commonly adjusted settings are provided below. - -# Adjust this key if you plan to make paperless available publicly. It should -# be a very long sequence of random characters. You don't need to remember it. -#PAPERLESS_SECRET_KEY=change-me - -# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. -#PAPERLESS_TIME_ZONE=America/Los_Angeles - -# The default language to use for OCR. Set this to the language most of your -# documents are written in. -#PAPERLESS_OCR_LANGUAGE=eng diff --git a/scripts/make-release.sh b/scripts/make-release.sh deleted file mode 100755 index f5c9028fa..000000000 --- a/scripts/make-release.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -# Release checklist -# - wait for travis build. -# adjust src/paperless/version.py -# changelog in the documentation -# adjust versions in docker/hub/* -# adjust version in src-ui/src/environments/prod -# If docker-compose was modified: all compose files are the same. - -# Steps: -# run release script "dev", push -# if it works: new tag, merge into master -# on master: make release "lastest", push -# on master: make release "version-tag", push -# publish release files - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" -PAPERLESS_DIST_DOCKERFILES="$PAPERLESS_DIST/paperless-ng-dockerfiles" - -if [ -d "$PAPERLESS_DIST" ] -then - echo "Removing $PAPERLESS_DIST" - rm "$PAPERLESS_DIST" -r -fi - -mkdir "$PAPERLESS_DIST" -mkdir "$PAPERLESS_DIST_APP" -mkdir "$PAPERLESS_DIST_APP/docker" -mkdir "$PAPERLESS_DIST_APP/scripts" -mkdir "$PAPERLESS_DIST_DOCKERFILES" - -# setup dependencies. - -cd "$PAPERLESS_ROOT" - -pipenv clean -pipenv install --dev -pipenv lock --keep-outdated -r > "$PAPERLESS_DIST_APP/requirements.txt" - -# test if the application works. - -cd "$PAPERLESS_ROOT/src" -pipenv run pytest --cov -pipenv run pycodestyle - -# make the documentation. - -cd "$PAPERLESS_ROOT/docs" -make clean html - -# copy stuff into place - -# the application itself - -cp "$PAPERLESS_ROOT/.env" \ - "$PAPERLESS_ROOT/.dockerignore" \ - "$PAPERLESS_ROOT/CONTRIBUTING.md" \ - "$PAPERLESS_ROOT/LICENSE" \ - "$PAPERLESS_ROOT/Pipfile" \ - "$PAPERLESS_ROOT/Pipfile.lock" \ - "$PAPERLESS_ROOT/README.md" "$PAPERLESS_DIST_APP" - -cp "$PAPERLESS_ROOT/paperless.conf.example" "$PAPERLESS_DIST_APP/paperless.conf" - -# copy python source, templates and static files. -cd "$PAPERLESS_ROOT" -find src -wholename '*/templates/*' -o -wholename '*/static/*' -o -name '*.py' | cpio -pdm "$PAPERLESS_DIST_APP" - -# build the front end. - -cd "$PAPERLESS_ROOT/src-ui" -ng build --prod --output-hashing none --sourceMap=false --output-path "$PAPERLESS_DIST_APP/src/documents/static/frontend" - -# documentation -cp "$PAPERLESS_ROOT/docs/_build/html/" "$PAPERLESS_DIST_APP/docs" -r - -# docker files for building the image yourself -cp "$PAPERLESS_ROOT/docker/local/"* "$PAPERLESS_DIST_APP" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_APP" - -# docker files for pulling from docker hub -cp "$PAPERLESS_ROOT/docker/hub/"* "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/.env" "$PAPERLESS_DIST_DOCKERFILES" -cp "$PAPERLESS_ROOT/docker/docker-compose.env" "$PAPERLESS_DIST_DOCKERFILES" - -# auxiliary files required for the docker image -cp "$PAPERLESS_ROOT/docker/docker-entrypoint.sh" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/gunicorn.conf.py" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/imagemagick-policy.xml" "$PAPERLESS_DIST_APP/docker/" -cp "$PAPERLESS_ROOT/docker/supervisord.conf" "$PAPERLESS_DIST_APP/docker/" - -# auxiliary files for bare metal installs -cp "$PAPERLESS_ROOT/scripts/paperless-webserver.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-consumer.service" "$PAPERLESS_DIST_APP/scripts/" -cp "$PAPERLESS_ROOT/scripts/paperless-scheduler.service" "$PAPERLESS_DIST_APP/scripts/" - -# try to make the docker build. - -cd "$PAPERLESS_DIST_APP" - -docker build . -t "jonaswinkler/paperless-ng:$VERSION" - -# works. package the app! - -cd "$PAPERLESS_DIST" - -tar -cJf "paperless-ng-$VERSION.tar.xz" paperless-ng/ -tar -cJf "paperless-ng-$VERSION-dockerfiles.tar.xz" paperless-ng-dockerfiles/ diff --git a/scripts/push-release.sh b/scripts/push-release.sh deleted file mode 100755 index cfa63f5cf..000000000 --- a/scripts/push-release.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - - -VERSION=$1 - -if [ -z "$VERSION" ] -then - echo "Need a version string." - exit 1 -fi - -# source root directory of paperless -PAPERLESS_ROOT=$(git rev-parse --show-toplevel) - -# output directory -PAPERLESS_DIST="$PAPERLESS_ROOT/dist" -PAPERLESS_DIST_APP="$PAPERLESS_DIST/paperless-ng" - -cd "$PAPERLESS_DIST_APP" - -docker push "jonaswinkler/paperless-ng:$VERSION" From 7326a246225baa4840f4827e3469488c8365d938 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 16:01:16 +0100 Subject: [PATCH 002/369] added missing file --- docker-compose.example.env | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docker-compose.example.env diff --git a/docker-compose.example.env b/docker-compose.example.env new file mode 100644 index 000000000..4271bce6e --- /dev/null +++ b/docker-compose.example.env @@ -0,0 +1,34 @@ +# The UID and GID of the user used to run paperless in the container. Set this +# to your UID and GID on the host so that you have write access to the +# consumption directory. +#USERMAP_UID=1000 +#USERMAP_GID=1000 + +# Additional languages to install for text recognition, separated by a +# whitespace. Note that this is +# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the +# default language used when guessing the language from the OCR output. +# The container installs English, German, Italian, Spanish and French by +# default. +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# for available languages. +#PAPERLESS_OCR_LANGUAGES=tur ces + +############################################################################### +# Paperless-specific settings # +############################################################################### + +# All settings defined in the paperless.conf.example can be used here. The +# Docker setup does not use the configuration file. +# A few commonly adjusted settings are provided below. + +# Adjust this key if you plan to make paperless available publicly. It should +# be a very long sequence of random characters. You don't need to remember it. +#PAPERLESS_SECRET_KEY=change-me + +# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. +#PAPERLESS_TIME_ZONE=America/Los_Angeles + +# The default language to use for OCR. Set this to the language most of your +# documents are written in. +#PAPERLESS_OCR_LANGUAGE=eng From 2fff2277476c58adb03e856207082780c5b393c8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 13:13:52 +0100 Subject: [PATCH 003/369] Added multi arch docker builds to travis ci --- .travis.yml | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.travis.yml b/.travis.yml index b745d6bd7..307a6073a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,102 @@ jobs: - ng build --prod after_success: true + - stage: build_docker + name: amd64 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 + on: + condition: '"${BUILD_DOCKER}" = 1' + - stage: build_docker + name: arm64v8 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 + arch: arm64 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: build_docker + name: arm32v7 docker build + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . + - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + on: + condition: '"${BUILD_DOCKER}" = 1' + + - stage: publish_manifest + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support + services: + - docker + before_install: + - true + install: + - true + after_success: + - true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} + - | + if [ "${TRAVIS_BRANCH}" = "master" ]; then + echo "Master branch detected" + DOCKER_TAG="latest" + else + DOCKER_TAG=${TRAVIS_TAG} + fi + - | + if [ "${DOCKER_TAG}" != "" ]; then + echo "Create Tag ${DOCKER_TAG}" + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 + docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 + docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} + else + echo "Not a tag and not on master, so not pushing tag/master specific manifest" + fi + on: + condition: '"${BUILD_DOCKER}" = 1' + before_install: - sudo apt-get update -qq From 35523bfcaa960d4e305d731c83ea3df94fb5e4c4 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 16:19:24 +0100 Subject: [PATCH 004/369] Add libxslt-dev to dependencies. At least one arm based image does not pull in dependencies the same way the amd64 one does, resulting in a missing libxslt shared lib. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..016bf7da9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ + libxslt-dev \ optipng \ pngquant \ qpdf \ From e3e826e906e4ca1ba8f4f0028594523af307bc61 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 17:08:01 +0100 Subject: [PATCH 005/369] Adding some echos to bette runderstand where the build hangs --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 307a6073a..203861391 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,9 +60,12 @@ jobs: after_success: - true script: + - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short + - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . + - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64 on: From 133c1bf2a2c760b646a7d200d2976ae78e41cc29 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 21 Dec 2020 21:53:39 +0100 Subject: [PATCH 006/369] Use qemu also for arm64v8 build - something might be causing trouble on the native hosts --- .travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 203861391..d2534c152 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,14 +60,17 @@ jobs: after_success: - true script: - - echo "Starting arm64v8 build, login to docker" - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # register binfmt stuff for qemu-static binaries so we can use userland-emulation + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, + # which is amd64. + - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile + - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - echo "Build ..." - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - echo "Push" - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64 + arch: amd64 on: condition: '"${BUILD_DOCKER}" = 1' From bc30d28cbbf595e80d0eb9720515b150a7d2e425 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 23:02:26 +0100 Subject: [PATCH 007/369] Adds jbig2 to the build, fixes #93 --- Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Dockerfile b/Dockerfile index 732b4183c..6c04a1d07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,14 @@ +FROM ubuntu:20.04 AS jbig2 + +WORKDIR /usr/src/jbig2enc + +RUN apt-get update && apt-get install -y --no-install-recommends build-essential automake libtool libleptonica-dev zlib1g-dev git ca-certificates + +RUN git clone https://github.com/agl/jbig2enc . +RUN ./autogen.sh +RUN ./configure && make + + FROM node:15 AS frontend WORKDIR /usr/src/paperless/src-ui/ @@ -57,6 +68,12 @@ COPY docker/gunicorn.conf.py ./ COPY docker/supervisord.conf /etc/supervisord.conf COPY docker/docker-entrypoint.sh /sbin/docker-entrypoint.sh +# copy jbic +COPY --from=jbig2 /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/ +COPY --from=jbig2 /usr/src/jbig2enc/src/jbig2 /usr/local/bin/ +COPY --from=jbig2 /usr/src/jbig2enc/src/*.h /usr/local/include/ + + # copy app COPY src/ ./src/ COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ /usr/src/paperless/src/documents/static/frontend/ From b2accdbde78ef801c5733407ea623bc94e010a0f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:49:36 +0100 Subject: [PATCH 008/369] Use arm64-gravis instead of regular arm64. See if that is fast enough. --- .travis.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index d2534c152..b91e43a28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,17 +60,12 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:5a14c8bf5020253f322b8f1f6bec4c34cafb0097acf1c1155506ee17b3c71119/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:d75eb820f62221ce8e40c5d8dbe988aa417e88553ef095a4a7591d7318da8486/g' Dockerfile + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: amd64 + arch: arm64-graviton2 + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From 30984765e4e49a988fdbf32c0d42903561fab87a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 12:58:35 +0100 Subject: [PATCH 009/369] Remove accidentially added quote --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b91e43a28..5f838755d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ jobs: after_success: - true script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin" + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 From e13e202e3d7cec54f89f7dd9e016c3ef3c735011 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:27:19 +0100 Subject: [PATCH 010/369] Tell me, what kind of arm64 is it that produces amd64 docker images? It's weird --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f838755d..f99a52eba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,12 +60,13 @@ jobs: after_success: - true script: + - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: vm + virt: lxc on: condition: '"${BUILD_DOCKER}" = 1' @@ -85,8 +86,8 @@ jobs: - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, # which is amd64. - - sed -i 's/FROM node:15/FROM node@sha256:e9bf2028bd59399afd3f6665f427a07b7ddfee07cffdf2061c39d991a3b3e332/g' Dockerfile - - sed -i 's/FROM python:3.7-slim/FROM python@sha256:843dd86fa35b923095b5db60c6e2e296013d647c8327aeba0e7638e7f0a43373/g' Dockerfile + - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile + - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 From 12a8f96f28f2b7f5825e884a25b6802eedbfdfd5 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:42:21 +0100 Subject: [PATCH 011/369] Enable manifests support --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f99a52eba..6d26d6ccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,8 @@ jobs: # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 + env: + - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: condition: '"${BUILD_DOCKER}" = 1' From 8c07c8833cccb2ac7487ffb1100677b886a22592 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 13:43:51 +0100 Subject: [PATCH 012/369] Remove steps not necessary to build docker images --- .travis.yml | 54 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d26d6ccb..c04f765d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: - - name: "Paperless on Python 3.6" - python: "3.6" - - - name: "Paperless on Python 3.7" - python: "3.7" - - - name: "Paperless on Python 3.8" - python: "3.8" - - - name: "Documentation" - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true +# - name: "Paperless on Python 3.6" +# python: "3.6" +# +# - name: "Paperless on Python 3.7" +# python: "3.7" +# +# - name: "Paperless on Python 3.8" +# python: "3.8" +# +# - name: "Documentation" +# script: +# - cd docs/ +# - make html +# after_success: true +# +# - name: "Front end" +# language: node_js +# node_js: +# - 15 +# before_install: true +# install: +# - cd src-ui/ +# - npm install -g @angular/cli +# - npm install +# script: +# - ng build --prod +# after_success: true - stage: build_docker name: amd64 docker build From d9c1c15dd6478e52fbb9482050a77056da854ed8 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:15:06 +0100 Subject: [PATCH 013/369] Switching back to virt: vm as there seems to be a prob with the lxc ones --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c04f765d0..9803a4d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,7 @@ jobs: - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 - virt: lxc + virt: vm on: condition: '"${BUILD_DOCKER}" = 1' From 5b584670d6e642ad02da42d3bcbe050b02d2080b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Tue, 22 Dec 2020 14:18:11 +0100 Subject: [PATCH 014/369] arm64-graviton2 build requires group:edge --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9803a4d3b..8422229c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,6 +67,7 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 arch: arm64-graviton2 virt: vm + group: edge on: condition: '"${BUILD_DOCKER}" = 1' From 43ad2e59b18a5c5d6cf1dc5129edb8ffc119c412 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 18:46:47 +0100 Subject: [PATCH 015/369] add basic ansible role for debian deployment Currently only Debian 10 buster is supported. Other Debian versions, Ubuntu and derivates should be easy to integrate. Database deployment is considered out-of-scope and deferred to the user. Provides basic upgrade support between releases. --- ansible/defaults/main.yml | 39 +++++ ansible/tasks/main.yml | 350 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 389 insertions(+) create mode 100644 ansible/defaults/main.yml create mode 100644 ansible/tasks/main.yml diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml new file mode 100644 index 000000000..0fa03b47a --- /dev/null +++ b/ansible/defaults/main.yml @@ -0,0 +1,39 @@ +--- +paperlessng_version: 0.9.8 +paperlessng_directory: /opt/paperless-ng +paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" +paperlessng_data_dir: "{{ paperlessng_directory }}/data" +paperlessng_media_root: "{{ paperlessng_directory }}/media" +paperlessng_static_dir: "{{ paperlessng_directory }}/static" +paperlessng_filename_format: +paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" + +paperlessng_ocr_languages: + - eng +paperlessng_time_zone: Europe/Berlin +paperlessng_ocrmypdf_args: --optimize 1 +# TODO Does optimze==1 really work with jbig2enc? +# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 +# Documentation states -O1 only applies lossless transformations +# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +paperlessng_use_jbig2enc: true + +paperlessng_superuser_name: paperlessng +paperlessng_superuser_email: paperlessng@example.com +paperlessng_superuser_password: paperlessng + +paperlessng_system_user: paperlessng +paperlessng_system_group: paperlessng + +paperlessng_listen_address: 127.0.0.1 +paperlessng_listen_port: 8000 + +paperlessng_redis_host: localhost +paperlessng_redis_port: 6379 + +paperlessng_db_type: sqlite # or postgresql +paperlessng_db_host: localhost +paperlessng_db_port: 5432 +paperlessng_db_name: paperlessng +paperlessng_db_user: paperlessng +paperlessng_db_pass: paperlessng \ No newline at end of file diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml new file mode 100644 index 000000000..b427482b2 --- /dev/null +++ b/ansible/tasks/main.yml @@ -0,0 +1,350 @@ +--- +- name: verify operating system + fail: + msg: Sorry, only Debian 10 supported at the moment. + when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + +- name: install base dependencies + apt: + update_cache: yes + pkg: + # paperless-ng + - python3-dev + - python3-pip + - imagemagick + - unpaper + - ghostscript + - optipng + - tesseract-ocr + - gnupg + - libpoppler-cpp-dev + - libmagic-dev + - libpq-dev + # OCRmyPDF + - icc-profiles-free + - qpdf + - liblept5 + - libxml2 + - pngquant + - zlib1g + # dev + - build-essential + - python3-setuptools + - python3-wheel + - python3-virtualenv + +- name: install ocr languages + apt: + pkg: "{{ paperlessng_ocr_languages | map('regex_replace', '^(.*)$', 'tesseract-ocr-\\1') | list }}" + +- name: set up notesalexp repository key (for jbig2enc) + apt_key: + url: https://notesalexp.org/debian/alexp_key.asc + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository (for jbig2enc) + apt_repository: + repo: deb https://notesalexp.org/debian/buster/ buster main + state: present + when: paperlessng_use_jbig2enc + +- name: set up notesalexp repository pinning + copy: + content: | + Package: * + Pin: release o=notesalexp.org + Pin-Priority: 1 + + Package: jbig2enc + Pin: release o=notesalexp.org + Pin-Priority: 500 + dest: /etc/apt/preferences.d/notesalexp + when: paperlessng_use_jbig2enc + +- name: install jbig2enc + apt: + pkg: jbig2enc + update_cache: yes + when: paperlessng_use_jbig2enc + +- name: install redis + apt: + pkg: redis-server + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: enable redis + systemd: + name: redis-server + enabled: yes + masked: no + state: started + when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' + +- name: check for paperless-ng installation + command: + cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' + changed_when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + failed_when: false + ignore_errors: yes + register: paperlessng_current_version + +- name: backup current paperless-ng installation + copy: + src: "{{ paperlessng_directory }}" + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: download paperless-ng + get_url: + url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" + dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: create paperless-ng directories + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: 0750 + recurse: yes + with_items: + - "{{ paperlessng_directory }}" + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + +- name: create temporary directory + tempfile: + state: directory + register: tempdir + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: extract paperless-ng + unarchive: + src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz + dest: "{{ tempdir.path }}" + remote_src: yes + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: move paperless-ng + command: + cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" + args: + warn: false + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: remove temporary directory + file: + path: "{{ tempdir.path }}" + state: absent + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: configure paperless-ng + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_REDIS=" + line: "PAPERLESS_REDIS=redis://{{ paperlessng_redis_host }}:{{ paperlessng_redis_port }}" + - regexp: "^#?PAPERLESS_CONSUMPTION_DIR=" + line: "PAPERLESS_CONSUMPTION_DIR={{ paperlessng_consumption_dir }}" + - regexp: "^#?PAPERLESS_DATA_DIR=" + line: "PAPERLESS_DATA_DIR={{ paperlessng_data_dir }}" + - regexp: "^#?PAPERLESS_MEDIA_ROOT=" + line: "PAPERLESS_MEDIA_ROOT={{ paperlessng_media_root }}" + - regexp: "^#?PAPERLESS_STATICDIR=" + line: "PAPERLESS_STATICDIR={{ paperlessng_static_dir }}" + - regexp: "^#?PAPERLESS_FILENAME_FORMAT=" + line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" + - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" + line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # TODO JSON dict required in conf? + # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#?PAPERLESS_TIME_ZONE=" + line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" + no_log: true + +- name: configure paperless-ng database [sqlite] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "^#?PAPERLESS_DBHOST=" + state: absent + when: paperlessng_db_type == 'sqlite' + +- name: configure paperless-ng database [postgresql] + lineinfile: + path: "{{ paperlessng_directory }}/paperless.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - regexp: "^#?PAPERLESS_DBHOST=" + line: "PAPERLESS_DBHOST={{ paperlessng_db_host }}" + - regexp: "^#?PAPERLESS_DBPORT=" + line: "PAPERLESS_DBPORT={{ paperlessng_db_port }}" + - regexp: "^#?PAPERLESS_DBNAME=" + line: "PAPERLESS_DBNAME={{ paperlessng_db_name }}" + - regexp: "^#?PAPERLESS_DBUSER=" + line: "PAPERLESS_DBUSER={{ paperlessng_db_user }}" + - regexp: "^#?PAPERLESS_DBPASS=" + line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" + when: paperlessng_db_type == 'postgresql' + no_log: true + +- name: create paperlessng venv + command: + cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" + creates: "{{ paperlessng_virtualenv }}" + +- name: install paperlessng requirements + pip: + requirements: "{{ paperlessng_directory }}/requirements.txt" + virtualenv: "{{ paperlessng_virtualenv }}" + extra_args: --upgrade + +- name: collect static files + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + +- name: create database schema + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + register: database_schema + changed_when: '"No migrations to apply." not in database_schema.stdout' + +- name: create first paperless user + # "manage.py createsuperuser" only works on interactive TTYs + command: | + {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') + " + register: superuser + changed_when: superuser.stdout == 'changed' + no_log: true + +- name: configure ghostscript for PDF + lineinfile: + path: "/etc/ImageMagick-6/policy.xml" + regexp: '' + line: '' + +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + +- name: configure systemd services + ini_file: + path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" + section: "{{ item[1].section }}" + option: "{{ item[1].option }}" + value: "{{ item[1].value }}" + with_nested: + - [ + paperless-consumer.service, + paperless-scheduler.service, + paperless-webserver.service, + ] + - [ + { + section: "Service", + option: "User", + value: "{{ paperlessng_system_user }}", + }, + { + section: "Service", + option: "Group", + value: "{{ paperlessng_system_group }}", + }, + { + section: "Service", + option: "WorkingDirectory", + value: "{{ paperlessng_directory }}/src", + }, + ] + +- name: configure paperless-consumer service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-consumer.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py document_consumer" + +- name: configure paperless-scheduler service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-scheduler.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/python3 manage.py qcluster" + +- name: configure paperless-webserver service + ini_file: + path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service" + section: "Service" + option: "ExecStart" + value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}" + +- name: copy systemd services + copy: + src: "{{ paperlessng_directory }}/scripts/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + remote_src: yes + with_items: + - paperless-consumer.service + - paperless-scheduler.service + - paperless-webserver.service + register: paperless_services + +- name: reload systemd daemon + systemd: + name: "{{ item }}" + state: restarted + daemon_reload: yes + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver + when: paperless_services.changed + +- name: enable paperlessng services + systemd: + name: "{{ item }}" + enabled: yes + masked: no + state: started + with_items: + - paperless-consumer + - paperless-scheduler + - paperless-webserver From 093c4f1b9651c0ac50d9bb46c0c4f1b33e17c5e4 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 22 Dec 2020 19:30:53 +0100 Subject: [PATCH 016/369] Update to 0.9.9 --- ansible/defaults/main.yml | 5 +++-- ansible/tasks/main.yml | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 0fa03b47a..fbeef9871 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.8 +paperlessng_version: 0.9.9 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" @@ -32,8 +32,9 @@ paperlessng_redis_host: localhost paperlessng_redis_port: 6379 paperlessng_db_type: sqlite # or postgresql +# Below entries only apply for paperlessng_db_type=='postgresql' paperlessng_db_host: localhost paperlessng_db_port: 5432 paperlessng_db_name: paperlessng paperlessng_db_user: paperlessng -paperlessng_db_pass: paperlessng \ No newline at end of file +paperlessng_db_pass: paperlessng diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b427482b2..96b1e469b 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + #no_log: true - name: configure paperless-ng database [sqlite] lineinfile: @@ -209,17 +209,21 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + args: + chdir: "{{ paperlessng_directory }}/src" - name: create database schema - command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" + args: + chdir: "{{ paperlessng_directory }}/src" register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' - name: create first paperless user # "manage.py createsuperuser" only works on interactive TTYs command: | - {{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " from django.contrib.auth.models import User from django.contrib.auth.hashers import get_hasher @@ -242,6 +246,8 @@ User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') " + args: + chdir: "{{ paperlessng_directory }}/src" register: superuser changed_when: superuser.stdout == 'changed' no_log: true From 571736816ebda82675edd416845c72c3f809fa4b Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 09:14:40 +0100 Subject: [PATCH 017/369] Remove arm32v6 from docker manifest , it is not getting created --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8422229c7..281635ace 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,7 +110,7 @@ jobs: - true script: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} @@ -124,7 +124,7 @@ jobs: - | if [ "${DOCKER_TAG}" != "" ]; then echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v6 + docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} From e541992c1216f9dd2ab4d3c5cf146428a3bfd89d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Wed, 23 Dec 2020 12:13:12 +0100 Subject: [PATCH 018/369] Re-enable test steps --- .travis.yml | 55 ++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 281635ace..e5f3cd91e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,33 +5,33 @@ os: linux jobs: include: -# - name: "Paperless on Python 3.6" -# python: "3.6" -# -# - name: "Paperless on Python 3.7" -# python: "3.7" -# -# - name: "Paperless on Python 3.8" -# python: "3.8" -# -# - name: "Documentation" -# script: -# - cd docs/ -# - make html -# after_success: true -# -# - name: "Front end" -# language: node_js -# node_js: -# - 15 -# before_install: true -# install: -# - cd src-ui/ -# - npm install -g @angular/cli -# - npm install -# script: -# - ng build --prod -# after_success: true + - name: "Paperless on Python 3.6" + python: "3.6" + + - name: "Paperless on Python 3.7" + python: "3.7" + + - name: "Paperless on Python 3.8" + python: "3.8" + + - name: "Documentation" + script: + - cd docs/ + - make html + after_success: true + + - name: "Front end" + language: node_js + node_js: + - 15 + before_install: true + install: + - cd src-ui/ + - npm install -g @angular/cli + - npm install + script: + - ng build --prod + after_success: true - stage: build_docker name: amd64 docker build @@ -60,7 +60,6 @@ jobs: after_success: - true script: - - uname -a - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . From f4e2c46d92dae55d3fba5d4feff3add01a1b38a0 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:30:09 +0100 Subject: [PATCH 019/369] Allow running role on all Debian releases Dynamically template current release string where required. --- ansible/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 96b1e469b..5b0bb5802 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian 10 supported at the moment. - when: not(ansible_distribution == 'Debian' and ansible_distribution_version == '10') + msg: Sorry, only Debian supported at the moment. + when: ansible_distribution != 'Debian' - name: install base dependencies apt: @@ -45,7 +45,7 @@ - name: set up notesalexp repository (for jbig2enc) apt_repository: - repo: deb https://notesalexp.org/debian/buster/ buster main + repo: "deb https://notesalexp.org/debian/{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} main" state: present when: paperlessng_use_jbig2enc @@ -169,7 +169,7 @@ line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - #no_log: true + no_log: true - name: configure paperless-ng database [sqlite] lineinfile: From 76e95d75f81ea3ef6140f423c5a31108016a9398 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 13:39:16 +0100 Subject: [PATCH 020/369] Fix fresh installation We can't backup a nonexistent folder. --- ansible/tasks/main.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 5b0bb5802..ba83b4975 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,6 +81,19 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' +- name: create paperless group + group: + name: "{{ paperlessng_system_group }}" + +- name: create paperless user + user: + name: "{{ paperlessng_system_user }}" + groups: + - "{{ paperlessng_system_group }}" + shell: /usr/sbin/nologin + # GNUPG_HOME required due to paperless db.py + create_home: yes + - name: check for paperless-ng installation command: cmd: 'grep -Po "(?<=Paperless-ng )\d+\.\d+\.\d+" {{ paperlessng_directory }}/docs/changelog.html' @@ -94,7 +107,7 @@ src: "{{ paperlessng_directory }}" dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' - name: download paperless-ng get_url: @@ -258,19 +271,6 @@ regexp: '' line: '' -- name: create paperless group - group: - name: "{{ paperlessng_system_group }}" - -- name: create paperless user - user: - name: "{{ paperlessng_system_user }}" - groups: - - "{{ paperlessng_system_group }}" - shell: /usr/sbin/nologin - # GNUPG_HOME required due to paperless db.py - create_home: yes - - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" From 36ac1eeb57b9314b2356218567c4c74652ed4009 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Wed, 23 Dec 2020 14:05:35 +0100 Subject: [PATCH 021/369] Do not clear static files on every run Django collectstatic knows when files change. --- ansible/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index ba83b4975..9c8fd67f5 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -222,7 +222,7 @@ extra_args: --upgrade - name: collect static files - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --clear --no-input" + command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" From c72c489cdcc1e532459f1b7eea59a4f546dbd669 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 23 Dec 2020 17:03:00 +0100 Subject: [PATCH 022/369] update Dockerfile --- Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f0112e529..686496475 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN apt-get update \ libpq-dev \ libqpdf-dev \ libxml2 \ - libxslt-dev \ + libxslt1-dev \ optipng \ pngquant \ qpdf \ @@ -55,9 +55,11 @@ RUN apt-get update \ tzdata \ unpaper \ zlib1g \ - && pip3 install --upgrade supervisor setuptools pipenv \ - && pipenv install --system --deploy --ignore-pipfile --clear \ - && apt-get -y purge build-essential libqpdf-dev \ + && pip3 install --upgrade supervisor pipenv \ + && pipenv install --system --deploy --ignore-pipfile \ + && pipenv --clear \ + && pip3 uninstall -y pipenv \ + && apt-get -y purge build-essential libqpdf-dev libxslt1-dev \ && apt-get -y autoremove --purge \ && rm -rf /var/lib/apt/lists/* \ && mkdir /var/log/supervisord /var/run/supervisord From 22ae74d409b6c62148849757e7f615be17457da1 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sun, 27 Dec 2020 20:01:21 +0100 Subject: [PATCH 023/369] Use conditions to prevent builds on branches / tags other than master, dev and ng-* --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5f3cd91e..cf18825e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ jobs: - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -68,8 +68,7 @@ jobs: virt: vm group: edge on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -94,8 +93,7 @@ jobs: env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support on: - condition: '"${BUILD_DOCKER}" = 1' - + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -131,7 +129,7 @@ jobs: echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi on: - condition: '"${BUILD_DOCKER}" = 1' + condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From 2b761cb3c67bfb7d23c404e465450d3216d999fe Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 09:47:17 +0100 Subject: [PATCH 024/369] Drop all permissions to paperlessng user Also make role idempotent --- ansible/tasks/main.yml | 105 ++++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 9c8fd67f5..77635b3d1 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -81,11 +81,11 @@ state: started when: paperlessng_redis_host == 'localhost' or paperlessng_redis_host == '127.0.0.1' -- name: create paperless group +- name: create paperless system group group: name: "{{ paperlessng_system_group }}" -- name: create paperless user +- name: create paperless system user user: name: "{{ paperlessng_system_user }}" groups: @@ -105,31 +105,10 @@ - name: backup current paperless-ng installation copy: src: "{{ paperlessng_directory }}" - dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" remote_src: yes + dest: "{{ paperlessng_directory }}-{{ ansible_date_time.iso8601 }}/" when: '"No such file or directory" not in paperlessng_current_version.stderr and paperlessng_current_version.stdout != paperlessng_version | string' -- name: download paperless-ng - get_url: - url: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" - dest: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - -- name: create paperless-ng directories - file: - path: "{{ item }}" - state: directory - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: 0750 - recurse: yes - with_items: - - "{{ paperlessng_directory }}" - - "{{ paperlessng_consumption_dir }}" - - "{{ paperlessng_data_dir }}" - - "{{ paperlessng_media_root }}" - - "{{ paperlessng_static_dir }}" - - name: create temporary directory tempfile: state: directory @@ -138,16 +117,28 @@ - name: extract paperless-ng unarchive: - src: /opt/paperless-ng-{{ paperlessng_version }}.tar.xz - dest: "{{ tempdir.path }}" + src: "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-{{ paperlessng_version }}/paperless-ng-{{ paperlessng_version }}.tar.xz" remote_src: yes + dest: "{{ tempdir.path }}" + when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' + +- name: change permissions of paperless-ng + command: + cmd: "{{ item }}" + with_items: + - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" + - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - command: - cmd: "cp -R {{ tempdir.path }}/paperless-ng/. {{ paperlessng_directory }}" - args: - warn: false + copy: + src: "{{ tempdir.path }}/paperless-ng/" + remote_src: yes + dest: "{{ paperlessng_directory }}" + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: preserve + directory_mode: preserve when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -156,6 +147,20 @@ state: absent when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' +- name: create paperless-ng directories and set permissions + file: + path: "{{ item }}" + state: directory + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: "750" + with_items: + - "{{ paperlessng_directory }}" # ansible `copy:` does not set correct permissions on `dest:` for recursive copies + - "{{ paperlessng_consumption_dir }}" + - "{{ paperlessng_data_dir }}" + - "{{ paperlessng_media_root }}" + - "{{ paperlessng_static_dir }}" + - name: configure paperless-ng lineinfile: path: "{{ paperlessng_directory }}/paperless.conf" @@ -176,10 +181,10 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # TODO JSON dict required in conf? - # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + # - regexp: "^#PAPERLESS_OCR_USER_ARG=" + # # TODO JSON dict required in conf + # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings + # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: true @@ -211,29 +216,45 @@ no_log: true - name: create paperlessng venv + become: yes + become_user: "{{ paperlessng_system_user }}" command: cmd: "python3 -m virtualenv {{ paperlessng_virtualenv }} -p /usr/bin/python3" creates: "{{ paperlessng_virtualenv }}" + register: venv - name: install paperlessng requirements + become: yes + become_user: "{{ paperlessng_system_user }}" pip: requirements: "{{ paperlessng_directory }}/requirements.txt" - virtualenv: "{{ paperlessng_virtualenv }}" + executable: "{{ paperlessng_virtualenv }}/bin/pip3" extra_args: --upgrade + when: paperlessng_current_version.stdout != paperlessng_version | string - name: collect static files + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string + register: static_files + changed_when: "'188 unmodified' not in static_files.stdout" - name: create database schema + become: yes + become_user: "{{ paperlessng_system_user }}" command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" args: chdir: "{{ paperlessng_directory }}/src" + when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' -- name: create first paperless user +- name: configure paperless superuser + become: yes + become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs command: | {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " @@ -265,6 +286,16 @@ changed_when: superuser.stdout == 'changed' no_log: true +- name: set ownership and permissions on paperlessng venv + file: + path: "{{ paperlessng_virtualenv }}" + state: directory + recurse: yes + owner: "{{ paperlessng_system_user }}" + group: "{{ paperlessng_system_group }}" + mode: g-w,o-rwx + when: venv.changed or paperlessng_current_version.stdout != paperlessng_version | string + - name: configure ghostscript for PDF lineinfile: path: "/etc/ImageMagick-6/policy.xml" @@ -325,8 +356,8 @@ - name: copy systemd services copy: src: "{{ paperlessng_directory }}/scripts/{{ item }}" - dest: "/etc/systemd/system/{{ item }}" remote_src: yes + dest: "/etc/systemd/system/{{ item }}" with_items: - paperless-consumer.service - paperless-scheduler.service From c9d90d8ae018b7d8fae961a355a9dee31a76c5ad Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 11:28:19 +0100 Subject: [PATCH 025/369] Add molecule test for role Only test default installation with jbig2enc and sqlite --- .travis.yml | 10 ++++++++++ ansible/molecule/default/converge.yml | 7 +++++++ ansible/molecule/default/molecule.yml | 21 +++++++++++++++++++++ ansible/molecule/default/verify.yml | 14 ++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 ansible/molecule/default/converge.yml create mode 100644 ansible/molecule/default/molecule.yml create mode 100644 ansible/molecule/default/verify.yml diff --git a/.travis.yml b/.travis.yml index b745d6bd7..e36b1ca99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,16 @@ jobs: - ng build --prod after_success: true + - name: "Ansible role" + sudo: required + services: + - docker + install: + - python3 -m pip install molecule[ansible,docker] + script: + - cd ansible/ + - molecule test + after_success: true before_install: - sudo apt-get update -qq diff --git a/ansible/molecule/default/converge.yml b/ansible/molecule/default/converge.yml new file mode 100644 index 000000000..82067f417 --- /dev/null +++ b/ansible/molecule/default/converge.yml @@ -0,0 +1,7 @@ +--- +- name: Converge + hosts: all + tasks: + - name: "Include ansible" + include_role: + name: "ansible" diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml new file mode 100644 index 000000000..1bf938d98 --- /dev/null +++ b/ansible/molecule/default/molecule.yml @@ -0,0 +1,21 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: debian_buster + image: jrei/systemd-debian:10 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # debian 9 stretch only has Python 3.5 which is EOL and breaks multiple dependencies +provisioner: + name: ansible +verifier: + name: ansible diff --git a/ansible/molecule/default/verify.yml b/ansible/molecule/default/verify.yml new file mode 100644 index 000000000..4c3b86044 --- /dev/null +++ b/ansible/molecule/default/verify.yml @@ -0,0 +1,14 @@ +--- +# This is an example playbook to execute Ansible tests. + +- name: Verify + hosts: all + gather_facts: false + tasks: + - name: check if webserver is up + uri: + url: http://localhost:8000 + status_code: [200, 302] + return_content: yes + register: landingpage + failed_when: "'Sign in' not in landingpage.content" From d258fdc783110b8abf0c5e1e5d57fbbdb03a8b73 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:22:35 +0100 Subject: [PATCH 026/369] Move build condition into separate stages list --- .travis.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf18825e8..26153588e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,12 @@ language: python dist: focal os: linux +stages: + - name: build_docker + if: branch =~ /^(master|dev|ng-.*)$/ + - name: publish_manifest + if: branch =~ /^(master|dev|ng-.*)$/ + jobs: include: - name: "Paperless on Python 3.6" @@ -47,8 +53,6 @@ jobs: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm64v8 docker build services: @@ -67,8 +71,6 @@ jobs: arch: arm64-graviton2 virt: vm group: edge - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: build_docker name: arm32v7 docker build services: @@ -92,8 +94,6 @@ jobs: - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' - stage: publish_manifest env: - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support @@ -128,8 +128,6 @@ jobs: else echo "Not a tag and not on master, so not pushing tag/master specific manifest" fi - on: - condition: '"${BUILD_DOCKER}" = 1 and ( branch =~ /^(master|dev|ng-.*)$/)' before_install: From 80df0cc06a64984a8ddd344b314d45d6c3e07d63 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 12:24:39 +0100 Subject: [PATCH 027/369] Add stage definitions of test jobs and add test to list of stages Defining the stages instead of having travis guess them and implicitly build an order ensures that the test stage is executed before the docker images are built. --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 26153588e..f4873cf3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ dist: focal os: linux stages: + - name: test - name: build_docker if: branch =~ /^(master|dev|ng-.*)$/ - name: publish_manifest @@ -12,21 +13,26 @@ stages: jobs: include: - name: "Paperless on Python 3.6" + stage: test python: "3.6" - name: "Paperless on Python 3.7" + stage: test python: "3.7" - name: "Paperless on Python 3.8" + stage: test python: "3.8" - name: "Documentation" + stage: test script: - cd docs/ - make html after_success: true - name: "Front end" + stage: test language: node_js node_js: - 15 From 39dc72b36dc30646c30326f401ecd9f4978bc20c Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Mon, 28 Dec 2020 12:21:10 +0100 Subject: [PATCH 028/369] Verify role for Ubuntu 20.04 --- ansible/molecule/default/molecule.yml | 14 ++++++++++++++ ansible/tasks/main.yml | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ansible/molecule/default/molecule.yml b/ansible/molecule/default/molecule.yml index 1bf938d98..27f37ba63 100644 --- a/ansible/molecule/default/molecule.yml +++ b/ansible/molecule/default/molecule.yml @@ -4,6 +4,20 @@ dependency: driver: name: docker platforms: + - name: ubuntu_focal + image: jrei/systemd-ubuntu:20.04 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /tmp + - /run + - /run/lock + override_command: False + # ubuntu 18.04 bionic works except that + # the default redis configuration expects IPv6 which is not enabled in docker by default + # the default Python environment is configured for ASCII instead of UTF-8 + # ubuntu 16.04 xenial only has Python 3.5 which is EOL and breaks multiple dependencies - name: debian_buster image: jrei/systemd-debian:10 privileged: true diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 77635b3d1..c94534895 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: verify operating system fail: - msg: Sorry, only Debian supported at the moment. - when: ansible_distribution != 'Debian' + msg: Sorry, only Debian and Ubuntu supported at the moment. + when: not(ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu') - name: install base dependencies apt: From dd3d55c6b94754395233df0e98f1e86b0ed6a27f Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 20:59:49 +0100 Subject: [PATCH 029/369] Make role compatible with ansible 2.7 Recursive remote copy is supported starting with 2.8 only Indentation behaviour in literal yaml strings seems to have changed Regex logic for ImageMagic was flawed (no idea why this worked before) --- ansible/tasks/main.yml | 69 +++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index c94534895..b1932a33c 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -28,6 +28,7 @@ - pngquant - zlib1g # dev + - sudo - build-essential - python3-setuptools - python3-wheel @@ -122,23 +123,19 @@ dest: "{{ tempdir.path }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' -- name: change permissions of paperless-ng +- name: change owner and permissions of paperless-ng command: cmd: "{{ item }}" + warn: false with_items: + - "chown -R {{ paperlessng_system_user }}:{{ paperlessng_system_group }} {{ tempdir.path }}" - "find {{ tempdir.path }} -type d -exec chmod 0750 {} ;" - "find {{ tempdir.path }} -type f -exec chmod 0640 {} ;" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: move paperless-ng - copy: - src: "{{ tempdir.path }}/paperless-ng/" - remote_src: yes - dest: "{{ paperlessng_directory }}" - owner: "{{ paperlessng_system_user }}" - group: "{{ paperlessng_system_group }}" - mode: preserve - directory_mode: preserve + command: + cmd: "cp -a {{ tempdir.path }}/paperless-ng/ {{ paperlessng_directory }}" when: '"No such file or directory" in paperlessng_current_version.stderr or paperlessng_current_version.stdout != paperlessng_version | string' - name: remove temporary directory @@ -256,30 +253,31 @@ become: yes become_user: "{{ paperlessng_system_user }}" # "manage.py createsuperuser" only works on interactive TTYs + vars: + creation_script: | + from django.contrib.auth.models import User + from django.contrib.auth.hashers import get_hasher + + if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): + user = User.objects.get(username='{{ paperlessng_superuser_name }}') + old = user.__dict__.copy() + + user.is_superuser = True + user.email = '{{ paperlessng_superuser_email }}' + user.set_password('{{ paperlessng_superuser_password }}') + user.save() + new = user.__dict__ + + algorithm, iterations, old_salt, old_hash = old['password'].split('$') + new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) + _, _, _, new_hash = new_password_old_salt.split('$') + if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): + print('changed') + else: + User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') + print('changed') command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c " - from django.contrib.auth.models import User - from django.contrib.auth.hashers import get_hasher - - if User.objects.filter(username='{{ paperlessng_superuser_name }}').exists(): - user = User.objects.get(username='{{ paperlessng_superuser_name }}') - old = user.__dict__.copy() - - user.is_superuser = True - user.email = '{{ paperlessng_superuser_email }}' - user.set_password('{{ paperlessng_superuser_password }}') - user.save() - new = user.__dict__ - - algorithm, iterations, old_salt, old_hash = old['password'].split('$') - new_password_old_salt = get_hasher(algorithm).encode(password='{{ paperlessng_superuser_password }}', salt=old_salt, iterations=int(iterations)) - _, _, _, new_hash = new_password_old_salt.split('$') - if not (old_hash == new_hash and old['is_superuser'] == new['is_superuser'] and old['email'] == new['email']): - print('changed') - else: - User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') - print('changed') - " + {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" args: chdir: "{{ paperlessng_directory }}/src" register: superuser @@ -298,9 +296,10 @@ - name: configure ghostscript for PDF lineinfile: - path: "/etc/ImageMagick-6/policy.xml" - regexp: '' - line: '' + path: /etc/ImageMagick-6/policy.xml + regexp: '(\s+)' + line: '\1' + backrefs: yes - name: configure systemd services ini_file: From f1bd6f1a4c3d95098bc4f27ee9af17f3accc5caf Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 21:55:59 +0100 Subject: [PATCH 030/369] Move paperless.conf to /etc, drop permissions --- ansible/tasks/main.yml | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index b1932a33c..2ee2e4db2 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -158,9 +158,14 @@ - "{{ paperlessng_media_root }}" - "{{ paperlessng_static_dir }}" +- name: rename initial config + command: + cmd: "mv {{ paperlessng_directory }}/paperless.conf {{ paperlessng_directory }}/paperless.conf.template" + removes: "{{ paperlessng_directory }}/paperless.conf" + - name: configure paperless-ng lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -184,18 +189,18 @@ # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" - no_log: true + no_log: yes - name: configure paperless-ng database [sqlite] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "^#?PAPERLESS_DBHOST=" state: absent when: paperlessng_db_type == 'sqlite' - name: configure paperless-ng database [postgresql] lineinfile: - path: "{{ paperlessng_directory }}/paperless.conf" + path: "{{ paperlessng_directory }}/paperless.conf.template" regexp: "{{ item.regexp }}" line: "{{ item.line }}" with_items: @@ -210,7 +215,17 @@ - regexp: "^#?PAPERLESS_DBPASS=" line: "PAPERLESS_DBPASS={{ paperlessng_db_pass }}" when: paperlessng_db_type == 'postgresql' - no_log: true + no_log: yes + +- name: deploy paperless-ng configuration + copy: + src: "{{ paperlessng_directory }}/paperless.conf.template" + remote_src: yes + dest: /etc/paperless.conf + owner: root + group: root + mode: '0644' + register: configuration - name: create paperlessng venv become: yes @@ -232,9 +247,7 @@ - name: collect static files become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py collectstatic --no-input" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py collectstatic --no-input" when: paperlessng_current_version.stdout != paperlessng_version | string register: static_files changed_when: "'188 unmodified' not in static_files.stdout" @@ -242,9 +255,7 @@ - name: create database schema become: yes become_user: "{{ paperlessng_system_user }}" - command: "{{ paperlessng_virtualenv }}/bin/python3 manage.py migrate" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py migrate" when: paperlessng_current_version.stdout != paperlessng_version | string register: database_schema changed_when: '"No migrations to apply." not in database_schema.stdout' @@ -276,13 +287,10 @@ else: User.objects.create_superuser('{{ paperlessng_superuser_name }}', '{{ paperlessng_superuser_email }}', '{{ paperlessng_superuser_password }}') print('changed') - command: | - {{ paperlessng_virtualenv }}/bin/python3 manage.py shell -c "{{ creation_script }}" - args: - chdir: "{{ paperlessng_directory }}/src" + command: "{{ paperlessng_virtualenv }}/bin/python3 {{ paperlessng_directory }}/src/manage.py shell -c \"{{ creation_script }}\"" register: superuser changed_when: superuser.stdout == 'changed' - no_log: true + no_log: yes - name: set ownership and permissions on paperlessng venv file: @@ -372,7 +380,7 @@ - paperless-consumer - paperless-scheduler - paperless-webserver - when: paperless_services.changed + when: paperless_services.changed or configuration.changed - name: enable paperlessng services systemd: From ce8c968c79152953dc046438050eab04421fdbcb Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 22:43:52 +0100 Subject: [PATCH 031/369] Integrate OCRmyPDF args into ansible config --- ansible/defaults/main.yml | 11 ++++++----- ansible/tasks/main.yml | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index fbeef9871..2504f18dc 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -11,12 +11,13 @@ paperlessng_virtualenv: "{{ paperlessng_directory }}/.venv" paperlessng_ocr_languages: - eng paperlessng_time_zone: Europe/Berlin -paperlessng_ocrmypdf_args: --optimize 1 -# TODO Does optimze==1 really work with jbig2enc? -# https://ocrmypdf.readthedocs.io/en/latest/jbig2.html#lossy-mode-jbig2 -# Documentation states -O1 only applies lossless transformations -# https://ocrmypdf.readthedocs.io/en/latest/optimizer.html#lossless-optimizations +# see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr +paperlessng_ocrmypdf_args: + - "deskew": true + - "clean": true + - "optimize": 1 paperlessng_use_jbig2enc: true +paperlessng_big2enc_lossy: false paperlessng_superuser_name: paperlessng paperlessng_superuser_email: paperlessng@example.com diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 2ee2e4db2..4a243f322 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,10 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - # - regexp: "^#PAPERLESS_OCR_USER_ARG=" - # # TODO JSON dict required in conf - # # https://paperless-ng.readthedocs.io/en/latest/configuration.html#ocr-settings - # line: "PAPERLESS_OCR_USER_ARG=\"{{ paperlessng_ocrmypdf_args }}{{ ' --jbig2-lossy' if paperlessng_use_jbig2enc else '' }}\"" + - regexp: "^#PAPERLESS_OCR_USER_ARG=" + line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From 4d076908908ae3da79e95a09cfc006b2e941e798 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:30:59 +0100 Subject: [PATCH 032/369] Harden systemd service files, drop perms further --- ansible/tasks/main.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index 4a243f322..f0fd84d67 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -310,7 +310,7 @@ - name: configure systemd services ini_file: path: "{{ paperlessng_directory }}/scripts/{{ item[0] }}" - section: "{{ item[1].section }}" + section: "Service" option: "{{ item[1].option }}" value: "{{ item[1].value }}" with_nested: @@ -320,21 +320,35 @@ paperless-webserver.service, ] - [ + # https://www.freedesktop.org/software/systemd/man/systemd.exec.html { - section: "Service", option: "User", value: "{{ paperlessng_system_user }}", }, { - section: "Service", option: "Group", value: "{{ paperlessng_system_group }}", }, { - section: "Service", option: "WorkingDirectory", value: "{{ paperlessng_directory }}/src", }, + { + option: "ProtectSystem", + value: "full", + }, + { + option: "NoNewPrivileges", + value: "true", + }, + { + option: "PrivateUsers", + value: "true", + }, + { + option: "PrivateDevices", + value: "true", + } ] - name: configure paperless-consumer service From 6b16307636f361c0b45941ed1ad3b5ff5c898a65 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Tue, 29 Dec 2020 23:54:22 +0100 Subject: [PATCH 033/369] Fix creation of user arg json.loads is picky in that is expects true json, not yaml from ansible --- ansible/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index f0fd84d67..604b29203 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -183,8 +183,8 @@ line: "PAPERLESS_FILENAME_FORMAT={{ paperlessng_filename_format }}" - regexp: "^#?PAPERLESS_OCR_LANGUAGE=" line: "PAPERLESS_OCR_LANGUAGE={{ paperlessng_ocr_languages | join('+') }}" - - regexp: "^#PAPERLESS_OCR_USER_ARG=" - line: "PAPERLESS_OCR_USER_ARG={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) }}" + - regexp: "^#PAPERLESS_OCR_USER_ARGS=" + line: "PAPERLESS_OCR_USER_ARGS={{ paperlessng_ocrmypdf_args | combine({'jbig2_lossy': true} if paperlessng_big2enc_lossy else {}) | to_json }}" - regexp: "^#?PAPERLESS_TIME_ZONE=" line: "PAPERLESS_TIME_ZONE={{ paperlessng_time_zone }}" no_log: yes From 61e6819b33a719a0397814daa0379acae2e2ef47 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 09:53:29 +0100 Subject: [PATCH 034/369] Update to 0.9.10 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index 2504f18dc..adfdb2b87 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.9 +paperlessng_version: 0.9.10 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From 0a66b9ff8c586a007028af099a9d89d24c6fc5bd Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 10:01:57 +0100 Subject: [PATCH 035/369] Change default OCRmyPDF args Clean is hardcoded as True, anyway Deskew breaks tesseract redo --- ansible/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index adfdb2b87..a2c560701 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -13,8 +13,7 @@ paperlessng_ocr_languages: paperlessng_time_zone: Europe/Berlin # see https://ocrmypdf.readthedocs.io/en/latest/api.html#ocrmypdf.ocr paperlessng_ocrmypdf_args: - - "deskew": true - - "clean": true + #- "deskew": true - "optimize": 1 paperlessng_use_jbig2enc: true paperlessng_big2enc_lossy: false From cbfd1062934d9597bab66cb612fb64cfda3d3e10 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:47:54 +0100 Subject: [PATCH 036/369] Update to 0.9.11 --- ansible/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/defaults/main.yml b/ansible/defaults/main.yml index a2c560701..da38ca300 100644 --- a/ansible/defaults/main.yml +++ b/ansible/defaults/main.yml @@ -1,5 +1,5 @@ --- -paperlessng_version: 0.9.10 +paperlessng_version: 0.9.11 paperlessng_directory: /opt/paperless-ng paperlessng_consumption_dir: "{{ paperlessng_directory }}/consumption" paperlessng_data_dir: "{{ paperlessng_directory }}/data" From 9aed0aceea2a780191a8be970dc83949367ac50a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:24:57 +0100 Subject: [PATCH 037/369] Removing travis in favor of github actions --- .travis.yml | 153 ---------------------------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f4873cf3c..000000000 --- a/.travis.yml +++ /dev/null @@ -1,153 +0,0 @@ -language: python - -dist: focal -os: linux - -stages: - - name: test - - name: build_docker - if: branch =~ /^(master|dev|ng-.*)$/ - - name: publish_manifest - if: branch =~ /^(master|dev|ng-.*)$/ - -jobs: - include: - - name: "Paperless on Python 3.6" - stage: test - python: "3.6" - - - name: "Paperless on Python 3.7" - stage: test - python: "3.7" - - - name: "Paperless on Python 3.8" - stage: test - python: "3.8" - - - name: "Documentation" - stage: test - script: - - cd docs/ - - make html - after_success: true - - - name: "Front end" - stage: test - language: node_js - node_js: - - 15 - before_install: true - install: - - cd src-ui/ - - npm install -g @angular/cli - - npm install - script: - - ng build --prod - after_success: true - - - stage: build_docker - name: amd64 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 - - stage: build_docker - name: arm64v8 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 - arch: arm64-graviton2 - virt: vm - group: edge - - stage: build_docker - name: arm32v7 docker build - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - # register binfmt stuff for qemu-static binaries so we can use userland-emulation - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - # replace the multi-arch reference with a specific, arm32v7 version. else docker will use the platform specific one, - # which is amd64. - - sed -i "s/FROM node:15/FROM node@$(docker manifest inspect node:15 | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - - sed -i "s/FROM python:3.7-slim/FROM python@$(docker manifest inspect python:3.7-slim | jq -r '.manifests [] | select (.platform.variant == "v7") | .digest')/g" Dockerfile - # travis_wait 60 tells travis to wait for up to 60 minutes - default is 20, which is too short - - travis_wait 60 docker build -f Dockerfile --tag=${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 . - - docker push ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - - stage: publish_manifest - env: - - DOCKER_CLI_EXPERIMENTAL=enabled # required for manifest support - services: - - docker - before_install: - - true - install: - - true - after_success: - - true - script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - docker manifest create ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - - docker manifest annotate ${DOCKER_REPO}:${TRAVIS_COMMIT} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - - docker manifest push --purge ${DOCKER_REPO}:${TRAVIS_COMMIT} - - | - if [ "${TRAVIS_BRANCH}" = "master" ]; then - echo "Master branch detected" - DOCKER_TAG="latest" - else - DOCKER_TAG=${TRAVIS_TAG} - fi - - | - if [ "${DOCKER_TAG}" != "" ]; then - echo "Create Tag ${DOCKER_TAG}" - docker manifest create ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-amd64 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm32v7 --os linux --arch arm --variant v7 - docker manifest annotate ${DOCKER_REPO}:${DOCKER_TAG} ${DOCKER_REPO}:${TRAVIS_COMMIT}-arm64v8 --os linux --arch arm64 --variant v8 - docker manifest push --purge ${DOCKER_REPO}:${DOCKER_TAG} - else - echo "Not a tag and not on master, so not pushing tag/master specific manifest" - fi - - -before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - -install: - - pip install --upgrade pipenv - - pipenv install --system --dev - -script: - - cd src/ - - pipenv run pytest --cov - - pipenv run pycodestyle - -after_success: - - pipenv run coveralls From 645ba2d425c9f1be8852fa6edb434b774abc2080 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 13:27:22 +0100 Subject: [PATCH 038/369] Experimenting with github actions --- .github/workflows/docker-publish.yml | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..f2da6b3b8 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,63 @@ +name: Docker + +on: + push: + branches: travis-multiarch-builds + +jobs: + # Build and push image to docker hub. + buildx: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Prepare + id: prepare + run: | + DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + fi + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./Dockerfile . + - name: install buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v3.1.0 + + - name: Docker Buildx (build) + run: | + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Login + if: success() && github.event_name != 'pull_request' + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + - name: Docker Buildx (push) + if: success() && github.event_name != 'pull_request' + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - name: Docker Check Manifest + if: always() && github.event_name != 'pull_request' + run: | + docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} + - name: Clear + if: always() && github.event_name != 'pull_request' + run: | + rm -f ${HOME}/.docker/config.json \ No newline at end of file From 94ab1dd5c4971a98c7859c4d2eda0843d4759e76 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Thu, 31 Dec 2020 19:32:04 +0100 Subject: [PATCH 039/369] Update docker-publish.yml --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f2da6b3b8..5ee210ef2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,7 +15,7 @@ jobs: - name: Prepare id: prepare run: | - DOCKER_IMAGE=MarkSchmitt/paperless-ng + DOCKER_IMAGE=moztr/paperless-ng DOCKER_PLATFORMS=linux/amd64,linux/arm/v7,linux/arm64/v8 VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then @@ -60,4 +60,4 @@ jobs: - name: Clear if: always() && github.event_name != 'pull_request' run: | - rm -f ${HOME}/.docker/config.json \ No newline at end of file + rm -f ${HOME}/.docker/config.json From d9992fb318466d1f1d8477f27815fd4f970ed9ad Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Fri, 1 Jan 2021 12:53:16 +0100 Subject: [PATCH 040/369] Removed separate build step It looks like part of the build step is repeated in the push step due to bad caching possibly. --- .github/workflows/docker-publish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5ee210ef2..53d53e25c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,10 +38,6 @@ jobs: - name: install buildx id: buildx uses: crazy-max/ghaction-docker-buildx@v3.1.0 - - - name: Docker Buildx (build) - run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} - name: Docker Login if: success() && github.event_name != 'pull_request' env: From f9430e1bd24ce6ec81bf502f4d7a9e3728e6d6d0 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:04:46 +0100 Subject: [PATCH 041/369] Execute python tests --- .github/workflows/docker-publish.yml | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 53d53e25c..a2897c05b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -5,6 +5,47 @@ on: branches: travis-multiarch-builds jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: Persistent Github pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip${{ matrix.python-version }} + - name: Update system, install system requirements + run: | + apt-get update -qq + apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + - name: Upgrade pip, Install dependencies + run: | + pip install --upgrade pip pipenv sphinx + pipenv lock -r > requirements.txt + pip install -r requirements.txt + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W + - name: Execute Tests + run: | + cd src/ + pytest --cov + pycodestyle + - name: Execute coveralls + run: | + coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 004ecc94b0fd9d6764d8f44dfe1a58f362e8f7b5 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:08:19 +0100 Subject: [PATCH 042/369] Use sudo for apt --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a2897c05b..f6ac55c1e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,8 +28,8 @@ jobs: key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | - apt-get update -qq - apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr + sudo apt-get update -qq + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip, Install dependencies run: | pip install --upgrade pip pipenv sphinx From a83d5c39797178edd171d412c405fbdfbec8029f Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:15:10 +0100 Subject: [PATCH 043/369] Execute pip commands one after the other to find issue --- .github/workflows/docker-publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f6ac55c1e..a810f134f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -30,10 +30,14 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip, Install dependencies + - name: Upgrade pip run: | pip install --upgrade pip pipenv sphinx + - name: Gather pip requirements + run: | pipenv lock -r > requirements.txt + - name: Insall pip requirements + run: | pip install -r requirements.txt - name: Run sphinx run: | From 192d27dd3c80de4adf23201e472a2f83b010daf2 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:19:58 +0100 Subject: [PATCH 044/369] Move sphinx step below tests step as it used to be --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a810f134f..e020fb66b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,17 +36,17 @@ jobs: - name: Gather pip requirements run: | pipenv lock -r > requirements.txt - - name: Insall pip requirements + - name: Install pip requirements run: | pip install -r requirements.txt - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - name: Execute Tests run: | cd src/ pytest --cov pycodestyle + - name: Run sphinx + run: | + sphinx-build -b html ../docs ../docs/_build -W - name: Execute coveralls run: | coveralls From 3bb351186a13977b981d1b062b2218daf253255a Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:23:49 +0100 Subject: [PATCH 045/369] Add pytest to install step --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e020fb66b..c074b1d2f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 0beba2eb1ed97ae5d41b041e48eeb9066fb8a6bd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:31:20 +0100 Subject: [PATCH 046/369] Add pytest-cov to install --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c074b1d2f..9bba2e031 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,7 +32,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest + pip install --upgrade pip pipenv sphinx pytest pytest-cov - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 00c0634dcbd547a25a823881832bdb32b7dcea48 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:41:21 +0100 Subject: [PATCH 047/369] Remove cache to see if that helps with pytest-cov issues --- .github/workflows/docker-publish.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9bba2e031..090cfa937 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,15 +17,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - name: Persistent Github pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip${{ matrix.python-version }} - name: Update system, install system requirements run: | sudo apt-get update -qq From 7026f4143839083035f68a43e6adc1beadb95e8d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 09:44:08 +0100 Subject: [PATCH 048/369] Try o make pipenv also install dev dependencies --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 090cfa937..24c3fe062 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov + pip install --upgrade pip pipenv sphinx - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 9563cf576b572ed1ac0dd60126a66dee900f4735 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:23:30 +0100 Subject: [PATCH 049/369] github-actions: readding explicit install explicitly install pytest, pytest-cov and coveralls. I think travis already did that and we need to explicitly do so now. removed installing dev-dependencies, do it explicitly. (just trying to reproduce the travis-ci build) --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 24c3fe062..afbb2571f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx + pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | - pipenv lock --dev -r > requirements.txt + pipenv lock -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 8d2d040346e4b396858ec509a2413397a40c64b3 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:27:38 +0100 Subject: [PATCH 050/369] github-actions: removing unsupported parameter I don't understand what the "-n auto" parameter does, pytest doesn't either (in the version installed) - so removing it to see if that produces something usable. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index 2a1a348bd..f43c9adf6 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html -n auto +addopts = --pythonwarnings=all --cov --cov-report=html env = PAPERLESS_DISABLE_DBHANDLER=true From 07556450300ebea030506b49d847d1f74952a286 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:30:12 +0100 Subject: [PATCH 051/369] github-actions: also install wheel There's an error about pip wheel not being available and falling back to some legacy version. Let's see if this helps. --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index afbb2571f..a7a399c3f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,7 +23,7 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls - name: Gather pip requirements run: | pipenv lock -r > requirements.txt From 0e73e56312259288d54eb8a9cd899e46bba2dd98 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Sat, 2 Jan 2021 11:33:56 +0100 Subject: [PATCH 052/369] github-actions: readding previously removed "-n auto" possibly this is fixed by the wheel dependency that was missing before. --- src/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setup.cfg b/src/setup.cfg index f43c9adf6..2a1a348bd 100644 --- a/src/setup.cfg +++ b/src/setup.cfg @@ -3,7 +3,7 @@ exclude = migrations, paperless/settings.py, .tox, */tests/* [tool:pytest] DJANGO_SETTINGS_MODULE=paperless.settings -addopts = --pythonwarnings=all --cov --cov-report=html +addopts = --pythonwarnings=all --cov --cov-report=html -n auto env = PAPERLESS_DISABLE_DBHANDLER=true From cd15490e91e7db755ea3a5feb6edd4471d10d45f Mon Sep 17 00:00:00 2001 From: jayme-github Date: Sat, 2 Jan 2021 14:40:56 +0100 Subject: [PATCH 053/369] Add option to ignore certain dates in parse_date PAPERLESS_IGNORE_DATES allows to specify a comma separated list of dates to ignore during date parsing (from filename and content). This can be used so specify dates that do appear often in documents but are usually not the documents creation date (like your date of birth). --- src/documents/parsers.py | 15 +++++++++++---- src/documents/tests/test_date_parsing.py | 15 +++++++++++++++ src/paperless/settings.py | 8 ++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/documents/parsers.py b/src/documents/parsers.py index e14607bd0..cf413a449 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -209,6 +209,13 @@ def parse_date(filename, text): } ) + def __filter(date): + if date and date.year > 1900 and \ + date <= timezone.now() and \ + date.date() not in settings.IGNORE_DATES: + return date + return None + date = None # if filename date parsing is enabled, search there first: @@ -222,7 +229,8 @@ def parse_date(filename, text): # Skip all matches that do not parse to a proper date continue - if date and date.year > 1900 and date <= timezone.now(): + date = __filter(date) + if date is not None: return date # Iterate through all regex matches in text and try to parse the date @@ -235,10 +243,9 @@ def parse_date(filename, text): # Skip all matches that do not parse to a proper date continue - if date and date.year > 1900 and date <= timezone.now(): + date = __filter(date) + if date is not None: break - else: - date = None return date diff --git a/src/documents/tests/test_date_parsing.py b/src/documents/tests/test_date_parsing.py index 357b0937e..9cbb19c2b 100644 --- a/src/documents/tests/test_date_parsing.py +++ b/src/documents/tests/test_date_parsing.py @@ -138,3 +138,18 @@ class TestDate(TestCase): @override_settings(FILENAME_DATE_ORDER="YMD") def test_filename_date_parse_invalid(self, *args): self.assertIsNone(parse_date("/tmp/20 408000l 2475 - test.pdf", "No date in here")) + + @override_settings(IGNORE_DATES=(datetime.date(2019, 11, 3), datetime.date(2020, 1, 17))) + def test_ignored_dates(self, *args): + text = ( + "lorem ipsum 110319, 20200117 and lorem 13.02.2018 lorem " + "ipsum" + ) + date = parse_date("", text) + self.assertEqual( + date, + datetime.datetime( + 2018, 2, 13, 0, 0, + tzinfo=tz.gettz(settings.TIME_ZONE) + ) + ) \ No newline at end of file diff --git a/src/paperless/settings.py b/src/paperless/settings.py index e8b44e8cd..5191803d0 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -4,6 +4,7 @@ import multiprocessing import os import re +import dateparser from dotenv import load_dotenv from django.utils.translation import gettext_lazy as _ @@ -444,3 +445,10 @@ PAPERLESS_TIKA_ENDPOINT = os.getenv("PAPERLESS_TIKA_ENDPOINT", "http://localhost PAPERLESS_TIKA_GOTENBERG_ENDPOINT = os.getenv( "PAPERLESS_TIKA_GOTENBERG_ENDPOINT", "http://localhost:3000" ) + +# List dates that should be ignored when trying to parse date from document text +IGNORE_DATES = set() +for s in os.getenv("PAPERLESS_IGNORE_DATES", "").split(","): + d = dateparser.parse(s) + if d: + IGNORE_DATES.add(d.date()) From 009de5184fc408cc5b993ac46b8b0c34836783bd Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Sun, 3 Jan 2021 00:37:19 -0800 Subject: [PATCH 054/369] Allow authentication via HTTP_REMOTE_USER --- docs/configuration.rst | 18 ++++++++++++------ paperless.conf.example | 1 + src/paperless/auth.py | 19 +++++++++++++++++++ src/paperless/settings.py | 7 +++++++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 5ccb80b3a..c72027574 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -162,6 +162,12 @@ PAPERLESS_COOKIE_PREFIX= Defaults to ``""``, which does not alter the cookie names. +PAPERLESS_ENABLE_HTTP_REMOTE_USER= + Allows authentication via HTTP_REMOTE_USER which is used by some SSO + applications. + + Defaults to `false` which disables this feature. + .. _configuration-ocr: OCR settings @@ -210,20 +216,20 @@ PAPERLESS_OCR_MODE= into images and puts the OCRed text on top. This works for all documents, however, the resulting document may be significantly larger and text won't appear as sharp when zoomed in. - + The default is ``skip``, which only performs OCR when necessary and always creates archived documents. PAPERLESS_OCR_OUTPUT_TYPE= Specify the the type of PDF documents that paperless should produce. - + * ``pdf``: Modify the PDF document as little as possible. * ``pdfa``: Convert PDF documents into PDF/A-2b documents, which is a subset of the entire PDF specification and meant for storing documents long term. * ``pdfa-1``, ``pdfa-2``, ``pdfa-3`` to specify the exact version of PDF/A you wish to use. - + If not specified, ``pdfa`` is used. Remember that paperless also keeps the original input file as well as the archived version. @@ -275,9 +281,9 @@ PAPERLESS_OCR_USER_ARG= .. code:: json - {"deskew": true, "optimize": 3, "unpaper_args": "--pre-rotate 90"} - - + {"deskew": true, "optimize": 3, "unpaper_args": "--pre-rotate 90"} + + Software tweaks ############### diff --git a/paperless.conf.example b/paperless.conf.example index 139453cf3..c55b7f5f4 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -31,6 +31,7 @@ #PAPERLESS_STATIC_URL=/static/ #PAPERLESS_AUTO_LOGIN_USERNAME= #PAPERLESS_COOKIE_PREFIX= +#PAPERLESS_ENABLE_HTTP_REMOTE_USER=false # OCR settings diff --git a/src/paperless/auth.py b/src/paperless/auth.py index ece5d0eba..d92dc7671 100644 --- a/src/paperless/auth.py +++ b/src/paperless/auth.py @@ -2,6 +2,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.utils.deprecation import MiddlewareMixin from rest_framework import authentication +from rest_framework import exceptions class AutoLoginMiddleware(MiddlewareMixin): @@ -26,3 +27,21 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication): return (user, None) else: return None + + +class HttpRemoteUserAuthentication(authentication.BaseAuthentication): + """ This class allows authentication via HTTP_REMOTE_USER which is set for + example by certain SSO applications. + """ + + def authenticate(self, request): + username = request.META.get('HTTP_REMOTE_USER') + if not username: + return None + + try: + user = User.objects.get(username=username) + except User.DoesNotExist: + raise exceptions.AuthenticationFailed('No such user') + + return (user, None) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 5af1be85e..f522c4c0b 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -112,6 +112,13 @@ if DEBUG: 'paperless.auth.AngularApiAuthenticationOverride' ) +ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER") + +if ENABLE_HTTP_REMOTE_USER: + REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append( + 'paperless.auth.HttpRemoteUserAuthentication' + ) + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', From 34421e2cef73f0c322c0833c5824572e629abbbc Mon Sep 17 00:00:00 2001 From: jayme-github Date: Sun, 3 Jan 2021 14:35:28 +0100 Subject: [PATCH 055/369] Add missing config options to example file --- paperless.conf.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paperless.conf.example b/paperless.conf.example index d9d0f5b06..2e46b91de 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -50,11 +50,14 @@ #PAPERLESS_TIME_ZONE=UTC #PAPERLESS_CONSUMER_POLLING=10 #PAPERLESS_CONSUMER_DELETE_DUPLICATES=false +#PAPERLESS_CONSUMER_RECURSIVE=false +#PAPERLESS_CONSUMER_SUBDIRS_AS_TAGS=false #PAPERLESS_OPTIMIZE_THUMBNAILS=true #PAPERLESS_POST_CONSUME_SCRIPT=/path/to/an/arbitrary/script.sh #PAPERLESS_FILENAME_DATE_ORDER=YMD #PAPERLESS_FILENAME_PARSE_TRANSFORMS=[] #PAPERLESS_THUMBNAIL_FONT_NAME= +#PAPERLESS_IGNORE_DATES= # Binaries From 25cc9e722cedec17c46b7517ef11071fb1f806f0 Mon Sep 17 00:00:00 2001 From: jayme-github Date: Sun, 3 Jan 2021 14:47:04 +0100 Subject: [PATCH 056/369] Add PAPERLESS_IGNORE_DATES to docs --- docs/configuration.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 49c95bff1..5036d6aa4 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -438,6 +438,19 @@ PAPERLESS_THUMBNAIL_FONT_NAME= Defaults to ``/usr/share/fonts/liberation/LiberationSerif-Regular.ttf``. +PAPERLESS_IGNORE_DATES= + Paperless parses a documents creation date from filename and file content. + You may specify a comma separated list of dates that should be ignored during + this process. This is useful for special dates (like date of birth) that appear + in documents regularly but are very unlikely to be the documents creation date. + + You may specify dates in a multitude of formats supported by dateparser (see + https://dateparser.readthedocs.io/en/latest/#popular-formats) but as the dates + need to be comma separated, the options are limited. + Example: "2020-12-02,22.04.1999" + + Defaults to an empty string to not ignore any dates. + Binaries ######## From 0286abd92a78e7235c6637a33b98df91a2e53eb0 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Sun, 3 Jan 2021 21:21:39 -0800 Subject: [PATCH 057/369] Refactor to extend RemoteUserMiddleware & add authentication for Django --- src/paperless/auth.py | 16 +++------------- src/paperless/settings.py | 21 ++++++++++++++------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/paperless/auth.py b/src/paperless/auth.py index d92dc7671..cd717e56b 100644 --- a/src/paperless/auth.py +++ b/src/paperless/auth.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.auth.models import User from django.utils.deprecation import MiddlewareMixin from rest_framework import authentication -from rest_framework import exceptions +from django.contrib.auth.middleware import RemoteUserMiddleware class AutoLoginMiddleware(MiddlewareMixin): @@ -29,19 +29,9 @@ class AngularApiAuthenticationOverride(authentication.BaseAuthentication): return None -class HttpRemoteUserAuthentication(authentication.BaseAuthentication): +class HttpRemoteUserMiddleware(RemoteUserMiddleware): """ This class allows authentication via HTTP_REMOTE_USER which is set for example by certain SSO applications. """ - def authenticate(self, request): - username = request.META.get('HTTP_REMOTE_USER') - if not username: - return None - - try: - user = User.objects.get(username=username) - except User.DoesNotExist: - raise exceptions.AuthenticationFailed('No such user') - - return (user, None) + header = 'HTTP_REMOTE_USER' diff --git a/src/paperless/settings.py b/src/paperless/settings.py index dd0d4a7d9..afbc667e0 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -115,13 +115,6 @@ if DEBUG: 'paperless.auth.AngularApiAuthenticationOverride' ) -ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER") - -if ENABLE_HTTP_REMOTE_USER: - REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append( - 'paperless.auth.HttpRemoteUserAuthentication' - ) - MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', @@ -135,6 +128,20 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] +ENABLE_HTTP_REMOTE_USER = __get_boolean("PAPERLESS_ENABLE_HTTP_REMOTE_USER") + +if ENABLE_HTTP_REMOTE_USER: + MIDDLEWARE.append( + 'paperless.auth.HttpRemoteUserMiddleware' + ) + AUTHENTICATION_BACKENDS = [ + 'django.contrib.auth.backends.RemoteUserBackend', + 'django.contrib.auth.backends.ModelBackend' + ] + REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].append( + 'rest_framework.authentication.RemoteUserAuthentication' + ) + ROOT_URLCONF = 'paperless.urls' FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME") From 93e69ebf383e37216f35ccc2687ca758ffeac9e0 Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 11:48:05 +0100 Subject: [PATCH 058/369] Switch to github actions --- .github/workflows/ansible.yml | 33 +++++++++++++++++++++++++++++++++ .travis.yml | 10 ---------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/ansible.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml new file mode 100644 index 000000000..41c13c774 --- /dev/null +++ b/.github/workflows/ansible.yml @@ -0,0 +1,33 @@ +--- +name: Ansible Role + +on: push + +jobs: + test: + # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + runs-on: ubuntu-latest + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/heads/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Set up Docker + uses: docker-practice/actions-setup-docker@master + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install molecule[ansible,docker] + ansible --version + docker --version + molecule --version + python --version + - name: Test with molecule + run: | + cd ansible + molecule test + working-directory: "${{ github.repository }}" diff --git a/.travis.yml b/.travis.yml index e36b1ca99..b745d6bd7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,16 +33,6 @@ jobs: - ng build --prod after_success: true - - name: "Ansible role" - sudo: required - services: - - docker - install: - - python3 -m pip install molecule[ansible,docker] - script: - - cd ansible/ - - molecule test - after_success: true before_install: - sudo apt-get update -qq From b436102a724ad629e660367d2b37b9bb1b76a57a Mon Sep 17 00:00:00 2001 From: Fabian Koller Date: Thu, 31 Dec 2020 13:21:42 +0100 Subject: [PATCH 059/369] Deploy role to ansible galaxy Happens only when a new tag is pushed --- .github/workflows/ansible.yml | 32 +++++++++++++++++++++++++---- ansible/README.md | 38 +++++++++++++++++++++++++++++++++++ ansible/meta/main.yml | 17 ++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 ansible/README.md create mode 100644 ansible/meta/main.yml diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 41c13c774..67383ecf8 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -1,14 +1,14 @@ --- name: Ansible Role -on: push +on: [push, pull_request] jobs: - test: # https://molecule.readthedocs.io/en/latest/ci.html#github-actions + test: runs-on: ubuntu-latest # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context - if: contains(github.ref, 'refs/heads/') + if: github.event_name == 'pull_request' || (github.event_name == 'push' && contains(github.ref, 'refs/heads/')) steps: - name: Check out the codebase uses: actions/checkout@v2 @@ -16,7 +16,7 @@ jobs: path: "${{ github.repository }}" - name: Set up Python uses: actions/setup-python@v2 - - name: Set up Docker + - name: Set up Docker uses: docker-practice/actions-setup-docker@master - name: Install dependencies run: | @@ -31,3 +31,27 @@ jobs: cd ansible molecule test working-directory: "${{ github.repository }}" + # https://galaxy.ansible.com/docs/contributing/importing.html + release: + runs-on: ubuntu-latest + needs: + - test + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#github-context + if: contains(github.ref, 'refs/tags/') + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + with: + path: "${{ github.repository }}" + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python3 -m pip install --upgrade ansible-base + ansible --version + python --version + - name: Trigger a new import on Galaxy + run: | + cd ansible + ansible-galaxy role import --api-key ${{ secrets.GALAXY_API_KEY }} $(echo ${{ github.repository }} | cut -d/ -f1) $(echo ${{ github.repository }} | cut -d/ -f2) + working-directory: "${{ github.repository }}" diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/ansible/meta/main.yml b/ansible/meta/main.yml new file mode 100644 index 000000000..78a4c1a41 --- /dev/null +++ b/ansible/meta/main.yml @@ -0,0 +1,17 @@ +dependencies: [] + +galaxy_info: + author: C0nsultant + description: Bare-metal deployment of paperless-ng DMS + license: license (GPLv3) + min_ansible_version: 2.7 + + platforms: + - name: Debian + versions: + - buster + - name: Ubuntu + versions: + - focal + + galaxy_tags: [EDMS, django, python, web] From cdec22a332aea62d8a3ccbfa444ca53d6e3005d3 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Mon, 4 Jan 2021 00:20:10 -0800 Subject: [PATCH 060/369] Clearing filters on saved views should reset them to initial rules --- .../document-list.component.html | 4 +- .../document-list/document-list.component.ts | 45 +++++++++++++++---- .../filter-editor.component.html | 6 +-- .../filter-editor/filter-editor.component.ts | 28 +++++------- 4 files changed, 52 insertions(+), 31 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index e4d7256e1..90893baee 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -5,7 +5,7 @@  Select - +
@@ -78,7 +78,7 @@
- +
diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index 1f29cb901..685d40a44 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -30,10 +30,6 @@ export class DocumentListComponent implements OnInit { displayMode = 'smallCards' // largeCards, smallCards, details - get isFiltered() { - return this.list.filterRules?.length > 0 - } - getTitle() { return this.list.savedViewTitle || $localize`Documents` } @@ -62,18 +58,16 @@ export class DocumentListComponent implements OnInit { this.router.navigate(["404"]) return } - this.list.savedView = view - this.list.reload() }) } else { this.list.savedView = null - this.list.reload() } + + this.list.reload() }) } - loadViewConfig(view: PaperlessSavedView) { this.list.load(view) this.list.reload() @@ -99,6 +93,7 @@ export class DocumentListComponent implements OnInit { sort_reverse: this.list.sortReverse, sort_field: this.list.sortField } + this.savedViewService.create(savedView).subscribe(() => { modal.close() this.toastService.showInfo($localize`View "${savedView.name}" created successfully.`) @@ -109,6 +104,40 @@ export class DocumentListComponent implements OnInit { }) } + resetFilters(): void { + if (this.list.savedViewId) { + this.savedViewService.getCached(this.list.savedViewId).subscribe(viewUntouched => { + this.list.filterRules = viewUntouched.filter_rules + }) + } else { + this.list.filterRules = [] + } + this.list.reload() + } + + get filterRulesModified(): boolean { + if (this.list.savedView == null) { + return this.list.filterRules.length > 0 // documents list is modified if it has any filters + } else { + // compare savedView current filters vs original + let modified = false + this.savedViewService.getCached(this.list.savedViewId).subscribe(view => { + let filterRulesInitial = view.filter_rules + + if (this.list.filterRules.length !== filterRulesInitial.length) modified = true + else { + this.list.filterRules.forEach(rule => { + if (filterRulesInitial.find(fri => fri.rule_type == rule.rule_type && fri.value == rule.value) == undefined) modified = true + }) + filterRulesInitial.forEach(rule => { + if (this.list.filterRules.find(fr => fr.rule_type == rule.rule_type && fr.value == rule.value) == undefined) modified = true + }) + } + }) + return modified + } + } + clickTag(tagID: number) { this.list.selectNone() setTimeout(() => { diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html index efbf6ce7e..02a51238d 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html @@ -41,11 +41,11 @@
-
diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index e6565beac..febb9dd84 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -40,7 +40,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy { case FILTER_HAS_TAG: return $localize`Tag: ${this.tags.find(t => t.id == +rule.value)?.name}` - + case FILTER_HAS_ANY_TAG: if (rule.value == "false") { return $localize`Without any tag` @@ -127,7 +127,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy { } else { this.tagSelectionModel.getSelectedItems().filter(tag => tag.id).forEach(tag => { filterRules.push({rule_type: FILTER_HAS_TAG, value: tag.id?.toString()}) - }) + }) } this.correspondentSelectionModel.getSelectedItems().forEach(correspondent => { filterRules.push({rule_type: FILTER_CORRESPONDENT, value: correspondent.id?.toString()}) @@ -153,16 +153,16 @@ export class FilterEditorComponent implements OnInit, OnDestroy { @Output() filterRulesChange = new EventEmitter() + @Output() + reset = new EventEmitter() + + @Input() + rulesModified: boolean = false + updateRules() { this.filterRulesChange.next(this.filterRules) } - hasFilters() { - return this._titleFilter || - this.dateAddedAfter || this.dateAddedBefore || this.dateCreatedAfter || this.dateCreatedBefore || - this.tagSelectionModel.selectionSize() || this.correspondentSelectionModel.selectionSize() || this.documentTypeSelectionModel.selectionSize() - } - get titleFilter() { return this._titleFilter } @@ -194,16 +194,8 @@ export class FilterEditorComponent implements OnInit, OnDestroy { this.titleFilterDebounce.complete() } - clearSelected() { - this._titleFilter = "" - this.tagSelectionModel.clear(false) - this.documentTypeSelectionModel.clear(false) - this.correspondentSelectionModel.clear(false) - this.dateAddedBefore = null - this.dateAddedAfter = null - this.dateCreatedBefore = null - this.dateCreatedAfter = null - this.updateRules() + resetSelected() { + this.reset.next() } toggleTag(tagId: number) { From be9f04cf6cb77a3a17a3d6043183f12775e8a2e7 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:18:56 +0100 Subject: [PATCH 061/369] Explicitly set ubuntu-20.04 as OS and install dev dependencies --- .github/workflows/docker-publish.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a7a399c3f..d29fd7740 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,8 +9,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] - python-version: [3.7, 3.8, 3.9] + os: [ubuntu-20.04] + python-version: 3.7 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -23,10 +23,10 @@ jobs: sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - name: Upgrade pip run: | - pip install --upgrade pip pipenv wheel sphinx pytest pytest-cov coveralls + pip install --upgrade pip pipenv - name: Gather pip requirements run: | - pipenv lock -r > requirements.txt + pipenv lock --dev -r > requirements.txt - name: Install pip requirements run: | pip install -r requirements.txt From 2562f1b44397c56c3eeb1200a4ba74fde1843b08 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:22:35 +0100 Subject: [PATCH 062/369] Revert job strategy matrix change --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d29fd7740..cfda3d10d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: 3.7 + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From afdcc0908fb598d5f325aeac2f0d2eac0f8e8614 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 12:24:48 +0100 Subject: [PATCH 063/369] Only build python 3.7 and say so in the Pipfile too --- .github/workflows/docker-publish.yml | 2 +- Pipfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cfda3d10d..0d2c8bd68 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7, 3.8, 3.9] + python-version: [3.7] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/Pipfile b/Pipfile index 48759307c..e0f1d83d8 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.6" +python_version = "3.7" [packages] dateparser = "~=0.7.6" From a0cac680bc2e96845d71572104cbad01d25a764d Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:00:35 +0100 Subject: [PATCH 064/369] Revert python version in Pipfile to 3.6 --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index e0f1d83d8..48759307c 100644 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true name = "piwheels" [requires] -python_version = "3.7" +python_version = "3.6" [packages] dateparser = "~=0.7.6" From 89bec6de1ba4e9db43d049da9b03589eedc0ebd1 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:10:16 +0100 Subject: [PATCH 065/369] Try to reproduce exactly what's done in the travis build to test --- .github/workflows/docker-publish.yml | 30 ++++++++-------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0d2c8bd68..66bc73ea8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,37 +10,23 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.7] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Update system, install system requirements + - name: Tests run: | sudo apt-get update -qq - sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr - - name: Upgrade pip - run: | - pip install --upgrade pip pipenv - - name: Gather pip requirements - run: | - pipenv lock --dev -r > requirements.txt - - name: Install pip requirements - run: | - pip install -r requirements.txt - - name: Execute Tests - run: | + sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + pip install --upgrade pipenv + pipenv install --system --dev cd src/ - pytest --cov - pycodestyle - - name: Run sphinx - run: | - sphinx-build -b html ../docs ../docs/_build -W - - name: Execute coveralls - run: | - coveralls + pipenv run pytest --cov + pipenv run pycodestyle + pipenv run coveralls # Build and push image to docker hub. buildx: runs-on: ubuntu-latest From 94083f8f195b9539fc8fcd0145f6c5d0135c7d91 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:17:13 +0100 Subject: [PATCH 066/369] Try manually replacing the python-version --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 66bc73ea8..264552168 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,6 +21,8 @@ jobs: run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng + echo "Replacing python version with ${matrix.python-version}" + sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From a8fb681bd514bc16803ef3e8b79758417065c5cd Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:19:00 +0100 Subject: [PATCH 067/369] Print the env --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 264552168..9904d8623 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -19,6 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | + env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng echo "Replacing python version with ${matrix.python-version}" From a59570e2d87ffa4e29875a20616a6895ebee3849 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:28:47 +0100 Subject: [PATCH 068/369] Only build for 3.6 --- .github/workflows/docker-publish.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9904d8623..23c8aef20 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: [3.6, 3.7, 3.8] + python-version: [3.6] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -19,11 +19,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Tests run: | - env sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng - echo "Replacing python version with ${matrix.python-version}" - sed -i "s/python_version = \"3.6\"/python_version = \"${matrix.python-version}\"/" Pipfile pip install --upgrade pipenv pipenv install --system --dev cd src/ From 61fc77c49cd718daa192ec065f64370d12cd8c33 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:44:02 +0100 Subject: [PATCH 069/369] Add GITHUB_TOKEN to python tests --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 23c8aef20..01bf72595 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From b9c55eb80cbabacce7b208aa9a9e47b0cfa66453 Mon Sep 17 00:00:00 2001 From: Mark Schmitt <31215086+MarkSchmitt@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:45:12 +0100 Subject: [PATCH 070/369] Fix yaml --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 01bf72595..c0a1565c6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,7 +18,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Tests - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sudo apt-get update -qq sudo apt-get install -qq libpoppler-cpp-dev unpaper tesseract-ocr imagemagick ghostscript optipng From 0ad943c704664ea42e2e8a44ff6250909616274c Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Mon, 4 Jan 2021 19:31:18 -0800 Subject: [PATCH 071/369] Fix some inconsistent elements for dark mode --- src-ui/src/theme_dark.scss | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src-ui/src/theme_dark.scss b/src-ui/src/theme_dark.scss index 88c2b8014..e65d4564e 100644 --- a/src-ui/src/theme_dark.scss +++ b/src-ui/src/theme_dark.scss @@ -174,6 +174,17 @@ $border-color-dark-mode: #47494f; color: $text-color-dark-mode; border-color: $border-color-dark-mode; + .des, + .asc { + background-color: transparent !important; + color: $text-color-dark-mode; + border-color: $border-color-dark-mode; + + &::after { + filter: invert(0.8); /* arrow is a black inline png bkgd image (!) so use filter */ + } + } + tr:hover { background-color: $bg-light-dark-mode; color: $text-color-dark-mode-accent; @@ -250,13 +261,18 @@ $border-color-dark-mode: #47494f; background-color: $bg-dark-mode !important; } - .form-control, + .form-control:not(.is-invalid):not(.btn), + input:not(.is-invalid), + textarea:not(.is-invalid) { + border-color: $border-color-dark-mode; /* we dont want to override controls that get highlighting for errors */ + } + + .form-control:not(.btn), input, select, textarea { background-color: $bg-dark-mode; color: $text-color-dark-mode; - border-color: $border-color-dark-mode; &::placeholder { color: $text-color-dark-mode; @@ -325,6 +341,12 @@ $border-color-dark-mode: #47494f; .progress { background-color: $border-color-dark-mode; } + + .alert-danger { + color: $text-color-dark-mode-accent; + background-color: darken($danger-dark-mode, 20%); + border-color: darken($danger-dark-mode, 20%); + } } body.color-scheme-dark { From 165a7a954e365383b27c25f7f18d40264d8c6e28 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 13:50:27 +0100 Subject: [PATCH 072/369] test cases --- .../commands/document_thumbnails.py | 10 +++- .../tests/test_management_thumbnails.py | 52 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 src/documents/tests/test_management_thumbnails.py diff --git a/src/documents/management/commands/document_thumbnails.py b/src/documents/management/commands/document_thumbnails.py index 01df14624..6f6451599 100644 --- a/src/documents/management/commands/document_thumbnails.py +++ b/src/documents/management/commands/document_thumbnails.py @@ -13,8 +13,14 @@ from ...parsers import get_parser_class_for_mime_type def _process_document(doc_in): document = Document.objects.get(id=doc_in) - parser = get_parser_class_for_mime_type(document.mime_type)( - logging_group=None) + parser_class = get_parser_class_for_mime_type(document.mime_type) + + if parser_class: + parser = parser_class(logging_group=None) + else: + print(f"{document} No parser for mime type {document.mime_type}") + return + try: thumb = parser.get_optimised_thumbnail( document.source_path, document.mime_type) diff --git a/src/documents/tests/test_management_thumbnails.py b/src/documents/tests/test_management_thumbnails.py new file mode 100644 index 000000000..7ecdf0489 --- /dev/null +++ b/src/documents/tests/test_management_thumbnails.py @@ -0,0 +1,52 @@ +import os +import shutil +from unittest import mock + +from django.core.management import call_command +from django.test import TestCase + +from documents.management.commands.document_thumbnails import _process_document +from documents.models import Document, Tag, Correspondent, DocumentType +from documents.tests.utils import DirectoriesMixin + + +class TestMakeThumbnails(DirectoriesMixin, TestCase): + + def make_models(self): + self.d1 = Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf", filename="test.pdf") + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), self.d1.source_path) + + self.d2 = Document.objects.create(checksum="Ass", title="A", content="first document", mime_type="application/pdf", filename="test2.pdf") + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), self.d2.source_path) + + def setUp(self) -> None: + super(TestMakeThumbnails, self).setUp() + self.make_models() + + def test_process_document(self): + self.assertFalse(os.path.isfile(self.d1.thumbnail_path)) + _process_document(self.d1.id) + self.assertTrue(os.path.isfile(self.d1.thumbnail_path)) + + @mock.patch("documents.management.commands.document_thumbnails.shutil.move") + def test_process_document_invalid_mime_type(self, m): + self.d1.mime_type = "asdasdasd" + self.d1.save() + + _process_document(self.d1.id) + + m.assert_not_called() + + def test_command(self): + self.assertFalse(os.path.isfile(self.d1.thumbnail_path)) + self.assertFalse(os.path.isfile(self.d2.thumbnail_path)) + call_command('document_thumbnails') + self.assertTrue(os.path.isfile(self.d1.thumbnail_path)) + self.assertTrue(os.path.isfile(self.d2.thumbnail_path)) + + def test_command_documentid(self): + self.assertFalse(os.path.isfile(self.d1.thumbnail_path)) + self.assertFalse(os.path.isfile(self.d2.thumbnail_path)) + call_command('document_thumbnails', '-d', f"{self.d1.id}") + self.assertTrue(os.path.isfile(self.d1.thumbnail_path)) + self.assertFalse(os.path.isfile(self.d2.thumbnail_path)) From bc31b88bb1cac90047b3d5691977082a184910c8 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 14:57:56 +0100 Subject: [PATCH 073/369] fix missing translation. --- .../document-list.component.html | 24 +++++-------------- .../tag-edit-dialog.component.html | 2 +- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index bb687632d..8f4ae1b6c 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -104,49 +104,37 @@ [currentSortField]="list.sortField" [currentSortReverse]="list.sortReverse" (sort)="onSort($event)" - i18n> - ASN - + i18n>ASN - Correspondent - + i18n>Correspondent - Title - + i18n>Title - Document type - + i18n>Document type - Created - + i18n>Created - Added - + i18n>Added diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html index ad40659fb..57f3b19fd 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html @@ -6,7 +6,7 @@
- +
From 996210df49de5679c1ca0c1186b7b4bf088f6da1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 22:11:42 +0100 Subject: [PATCH 074/369] fixes #278 --- .../src/app/components/common/input/abstract-input.ts | 11 ++++++++++- .../components/common/input/text/text.component.html | 2 +- .../document-detail/document-detail.component.html | 2 +- .../document-detail/document-detail.component.ts | 7 ++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src-ui/src/app/components/common/input/abstract-input.ts b/src-ui/src/app/components/common/input/abstract-input.ts index f6039d2ec..5821b6682 100644 --- a/src-ui/src/app/components/common/input/abstract-input.ts +++ b/src-ui/src/app/components/common/input/abstract-input.ts @@ -1,10 +1,13 @@ -import { Directive, Input, OnInit } from '@angular/core'; +import { Directive, ElementRef, Input, OnInit, ViewChild } from '@angular/core'; import { ControlValueAccessor } from '@angular/forms'; import { v4 as uuidv4 } from 'uuid'; @Directive() export class AbstractInputComponent implements OnInit, ControlValueAccessor { + @ViewChild("inputField") + inputField: ElementRef + constructor() { } onChange = (newValue: T) => {}; @@ -24,6 +27,12 @@ export class AbstractInputComponent implements OnInit, ControlValueAccessor { this.disabled = isDisabled; } + focus() { + if (this.inputField && this.inputField.nativeElement) { + this.inputField.nativeElement.focus() + } + } + @Input() title: string diff --git a/src-ui/src/app/components/common/input/text/text.component.html b/src-ui/src/app/components/common/input/text/text.component.html index 78aa76577..fe8607e61 100644 --- a/src-ui/src/app/components/common/input/text/text.component.html +++ b/src-ui/src/app/components/common/input/text/text.component.html @@ -1,6 +1,6 @@
- + {{hint}}
{{error}} diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index a724ae031..dbd1c0ba7 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -56,7 +56,7 @@ Details - + { this.networkActive = false From 4ffbbb979677753ef3678cae516985211bf1cbae Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Tue, 5 Jan 2021 23:36:31 +0100 Subject: [PATCH 075/369] fix some translations --- .../common/confirm-dialog/confirm-dialog.component.html | 2 +- .../src/app/components/common/input/tags/tags.component.html | 2 +- .../components/document-detail/document-detail.component.html | 2 +- .../app/components/manage/settings/settings.component.html | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html index 565829f21..8a8a297d9 100644 --- a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html @@ -9,7 +9,7 @@

{{message}}

- + + From 73fb163ad56ab80b739602bbce2e591e29618fa1 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Wed, 6 Jan 2021 01:16:16 +0100 Subject: [PATCH 076/369] fix login/logout pages --- src/documents/templates/registration/logged_out.html | 2 +- src/documents/templates/registration/login.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/documents/templates/registration/logged_out.html b/src/documents/templates/registration/logged_out.html index 4a3cb60bb..1448b641e 100644 --- a/src/documents/templates/registration/logged_out.html +++ b/src/documents/templates/registration/logged_out.html @@ -36,7 +36,7 @@ diff --git a/src/documents/templates/registration/login.html b/src/documents/templates/registration/login.html index 25a707bed..7f1fc79dd 100644 --- a/src/documents/templates/registration/login.html +++ b/src/documents/templates/registration/login.html @@ -37,7 +37,7 @@