From e99bbb4e02c0a1f4b0a90e2b09693023941c61e1 Mon Sep 17 00:00:00 2001 From: syntonym Date: Fri, 11 Jan 2019 19:22:51 +0100 Subject: [PATCH 01/14] Catches OSError on IMAP connection error When something goes wrong with the imaplib.IMAP4_SSL connection (like the host is temporarely down or the DNS does not resolve) it generates an OSError which is currently not catched and handled. Now OSErrors are translated to MailFetcherErrors which get logged and the IMAP connection is retried in the next IMAP check. Fixes #474 --- src/documents/mail.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/documents/mail.py b/src/documents/mail.py index afa1b4362..d54b387b7 100644 --- a/src/documents/mail.py +++ b/src/documents/mail.py @@ -216,7 +216,11 @@ class MailFetcher(Loggable): return r def _connect(self): - self._connection = imaplib.IMAP4_SSL(self._host, self._port) + try: + self._connection = imaplib.IMAP4_SSL(self._host, self._port) + except OSError as e: + msg = "Problem connecting to {}: {}".format(self._host, e.strerror) + raise MailFetcherError(msg) def _login(self): From 30f97126a7889e5885bdc8f615ecebc49606868d Mon Sep 17 00:00:00 2001 From: Christian Wiwie Date: Fri, 18 Jan 2019 20:33:35 +0100 Subject: [PATCH 02/14] Update requirements.rst --- docs/requirements.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/requirements.rst b/docs/requirements.rst index b6cbad213..2bfc5cdec 100644 --- a/docs/requirements.rst +++ b/docs/requirements.rst @@ -12,6 +12,7 @@ should work) that has the following software installed: * `Imagemagick`_ version 6.7.5 or higher * `unpaper`_ * `libpoppler-cpp-dev`_ PDF rendering library +* `optipng`_ .. _Python3: https://python.org/ .. _GNU Privacy Guard: https://gnupg.org @@ -19,6 +20,7 @@ should work) that has the following software installed: .. _Imagemagick: http://imagemagick.org/ .. _unpaper: https://www.flameeyes.eu/projects/unpaper .. _libpoppler-cpp-dev: https://poppler.freedesktop.org/ +.. _optipng: http://optipng.sourceforge.net/ Notably, you should confirm how you access your Python3 installation. Many Linux distributions will install Python3 in parallel to Python2, using the From 64d7add51959f197dacaf2e7cbb885d455d90504 Mon Sep 17 00:00:00 2001 From: CkuT Date: Mon, 21 Jan 2019 20:04:57 +0100 Subject: [PATCH 03/14] Add Cache-Control header for thumbnails This drastically optimizes admin interface loading by telling the browser to cache thumbnails. The max-age recommendation is 1 year according to rfc2616 Closes #411 --- src/documents/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/documents/views.py b/src/documents/views.py index 186b733a5..f72ca22b1 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -2,6 +2,7 @@ from django.http import HttpResponse, HttpResponseBadRequest from django.views.generic import DetailView, FormView, TemplateView from django_filters.rest_framework import DjangoFilterBackend from django.conf import settings +from django.utils import cache from paperless.db import GnuPG from paperless.mixins import SessionOrBasicAuthMixin @@ -56,10 +57,12 @@ class FetchView(SessionOrBasicAuthMixin, DetailView): } if self.kwargs["kind"] == "thumb": - return HttpResponse( + response = HttpResponse( self._get_raw_data(self.object.thumbnail_file), content_type=content_types[Document.TYPE_PNG] ) + cache.patch_cache_control(response, max_age=31536000, private=True) + return response response = HttpResponse( self._get_raw_data(self.object.source_file), From d43bd3610bb9b52b737cb0bde27a984e68350931 Mon Sep 17 00:00:00 2001 From: tsia Date: Tue, 22 Jan 2019 20:51:20 +0100 Subject: [PATCH 04/14] added fields to the correspondent and tag REST API --- src/documents/serialisers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index bd00fe466..b497a1dd0 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -7,7 +7,7 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Correspondent - fields = ("id", "slug", "name") + fields = ("id", "slug", "name", "match", "matching_algorithm", "is_insensitive") class TagSerializer(serializers.HyperlinkedModelSerializer): @@ -15,7 +15,7 @@ class TagSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Tag fields = ( - "id", "slug", "name", "colour", "match", "matching_algorithm") + "id", "slug", "name", "colour", "match", "matching_algorithm", "is_insensitive") class CorrespondentField(serializers.HyperlinkedRelatedField): From 5422e5855aef3755681af1e7529564255aa48ff2 Mon Sep 17 00:00:00 2001 From: Brian Cribbs Date: Tue, 22 Jan 2019 15:18:14 -0500 Subject: [PATCH 05/14] adding information about NFS mounts and inotify --- docs/setup.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/setup.rst b/docs/setup.rst index e86fe9efe..07dd555de 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -143,6 +143,15 @@ Docker Method instructions in comments in the file. The only change that is a hard requirement is to specify where the consumption directory should mount.[#dockercomposeyml]_ + + .. caution:: + + If you are using NFS mounts for the consume directory you also need to + change the command to turn off inotify as it doesn't work with NFS + + `command: ["document_consumer", "--no-inotify"]` + + 5. Modify ``docker-compose.env`` and adapt the following environment variables: ``PAPERLESS_PASSPHRASE`` From d2fc4e0768c04fc43af6a51d4346f9c5c2248ae7 Mon Sep 17 00:00:00 2001 From: bmsleight Date: Sat, 26 Jan 2019 13:15:45 +0000 Subject: [PATCH 06/14] Add note runserver PAPERLESS_DEBUG='true' --- docs/setup.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/setup.rst b/docs/setup.rst index e86fe9efe..75bc5eb78 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -86,6 +86,7 @@ Standard (Bare Metal) You should now be able to visit your (empty) installation at `Paperless webserver`_ or whatever you chose before. You can login with the user/pass you created in #5. + If using runserver within /etc/paperless.conf make sure PAPERLESS_DEBUG="true" 7. In a separate window, change to the ``src`` directory in this repo again, but this time, you should start the consumer script with From fa8903fc7d7366389966f70a891b4a176565741c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sat, 26 Jan 2019 14:22:21 +0100 Subject: [PATCH 07/14] Tabs are not allowed in Yaml files --- docker-compose.yml.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml.example b/docker-compose.yml.example index fb905832e..312467b91 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -17,9 +17,9 @@ services: volumes: - data:/usr/src/paperless/data - media:/usr/src/paperless/media - # You have to adapt the local path you want the consumption + # You have to adapt the local path you want the consumption # directory to mount to by modifying the part before the ':'. - - ./consume:/consume + - ./consume:/consume env_file: docker-compose.env # The reason the line is here is so that the webserver that doesn't do # any text recognition and doesn't have to install unnecessary @@ -40,7 +40,7 @@ services: - data:/usr/src/paperless/data - media:/usr/src/paperless/media # This should be set to the same value as the consume directory - # in the webserver service above. + # in the webserver service above. - ./consume:/consume # Likewise, you can add a local path to mount a directory for # exporting. This is not strictly needed for paperless to From 0d8351e85bfb497a39c7e524a111b4d9c0e84d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sat, 26 Jan 2019 14:33:51 +0100 Subject: [PATCH 08/14] Install psycopg2 in the Docker container --- Pipfile | 1 + Pipfile.lock | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/Pipfile b/Pipfile index 47b4c8663..7cf7db7e6 100644 --- a/Pipfile +++ b/Pipfile @@ -34,6 +34,7 @@ pytest-django = "*" pytest-sugar = "*" pytest-env = "*" pytest-xdist = "*" +psycopg2 = "*" [dev-packages] ipython = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5cbeecc69..395688203 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -365,6 +365,41 @@ ], "version": "==0.8.0" }, + "psycopg2": { + "hashes": [ + "sha256:10e391687b171878181e71736d0effe3772314a339d9ae30995ec8171a0c834e", + "sha256:1283f9d45e458c2dcb15ba89367923563f90ef636fe78ee22df75183484a0237", + "sha256:1a9c32e4d140bea225f9821d993b2e53c913e717ea97b851246aa9b300095d8f", + "sha256:1be6f2438d2b71fec7b07c3c0949dd321b04349c382907ea76b36120edec8300", + "sha256:20ca6f29e118b8dd7133e8708b3fba2881e70a4e0841f874ed23985b7201a076", + "sha256:227c115b3c1f65d61385e51ac690b91b584640aefb45bffacd4bd33d02ed7221", + "sha256:27959abe64ca1fc6d8cd11a71a1f421d8287831a3262bd4cacd43bbf43cc3c82", + "sha256:2b2daf1fe30a58300542aea679fd87d1e1c2afd36e7644837b7954fa2dbacb92", + "sha256:36e51a51f295fdf67bcf05e7b1877011a6b39e6622b0013fe31c5025241873a3", + "sha256:3992b9b914f2eb77dc07e8045d2ca979e491612808bc5c7cd68f307469acf9f6", + "sha256:39a11de2335ad45ececed43ab851d36a4c52843d756471b940804f301792781e", + "sha256:3c2afe9ef0d1649005e3ccf93c1aaccd6f8ee379530e763d3b3b77f406b7c0ae", + "sha256:3fb18e0e52807fe3a300dc1b5421aa492d5e759550918f597d61863419482535", + "sha256:55eab94de96ee9702f23283e9c8b03cfdb0001e2b14d5d2e1bd5ff8114b96b9f", + "sha256:7e95c0ab7e7e6e452586f35d4d8966b1e924c8dd2c23977e3ea4968770ff1d26", + "sha256:7f47514dbddf604f196fcfe5da955537f04691bef8124aff5632316a78d992b7", + "sha256:8345370356bb4bddf93acbcfd0357163dd6b09471937adcfb38a2fbb49bdce53", + "sha256:8bc6ecb220c0b88d3742042013129c817c44459795c97e9ce1bca70a3f37a53b", + "sha256:8df623f248be15d1725faf5f333791678775047f12f17a90d29b5d22573f5cdc", + "sha256:9645f1305e4268cc0fc88c823cd6c91de27c003e183c233a6a230e5e963039ee", + "sha256:a68719ed5be8373dd72c9e45d55f7a202285e05a2e392eaa8872a67ea47d7d20", + "sha256:aca0edf062ec09e954fdf0cc93d3a872362701210983a1442549e703aedec25d", + "sha256:b0dd2114d93d8f424bb8ae76e0dc540f104b70ca9163172c05e7700b1459d4c9", + "sha256:b2c09359d6802279efb9efb3f91a9c94567151baee95175f9b637ea628f35244", + "sha256:ca7bc37b1efb7cc25271bf10f398462ed975d95259af1406d38fcb268466e34f", + "sha256:e64235d9013ebf6319cb9654e08f5066112c34d8c4cc41186254ab9c3d6d5b9b", + "sha256:ec9be679c0065667503851141c31fa699e1cc69ded3ba8e5d3673dd5a6eb1370", + "sha256:eca00d0f91fcb44d88b12f1fd16ad138e38fa07debb79587e2b7ff1fe80d72b9", + "sha256:f256e807b8b2b45b6af60d7f2bb5194aab2f4acc861241c4d8ef942a55f5030d", + "sha256:fce7612a3bd6a7ba95799f88285653bf130bd7ca066b52674d5f850108b2aec0" + ], + "version": "==0.8.0" + }, "py": { "hashes": [ "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", From 711c0925e1eeb137bafb594659f437075b9a8a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sat, 26 Jan 2019 22:10:13 +0100 Subject: [PATCH 09/14] Add and configure DjangoQL --- Pipfile | 1 + Pipfile.lock | 14 ++++++++++++++ requirements.txt | 1 + src/documents/admin.py | 3 ++- src/paperless/settings.py | 2 +- 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 47b4c8663..ccb577812 100644 --- a/Pipfile +++ b/Pipfile @@ -34,6 +34,7 @@ pytest-django = "*" pytest-sugar = "*" pytest-env = "*" pytest-xdist = "*" +djangoql = "*" [dev-packages] ipython = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5cbeecc69..c3492b434 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -155,6 +155,13 @@ "index": "pypi", "version": "==2.0.0" }, + "djangoql": { + "hashes": [ + "sha256:7c488ec4e3362e5389ba3c1169d1ff9a27c4222601f32c6dbf6130ce04330d76" + ], + "index": "pypi", + "version": "==0.12.3" + }, "djangorestframework": { "hashes": [ "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136", @@ -365,6 +372,13 @@ ], "version": "==0.8.0" }, + "ply": { + "hashes": [ + "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", + "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce" + ], + "version": "==3.11" + }, "py": { "hashes": [ "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", diff --git a/requirements.txt b/requirements.txt index d30cbf078..360ad310f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ django==2.0.9 djangorestframework==3.9.0 docopt==0.6.2 docutils==0.14 +djangoql==0.12.3 execnet==1.5.0 factory-boy==2.11.1 faker==0.9.2 diff --git a/src/documents/admin.py b/src/documents/admin.py index ff0365a53..a7c9e828e 100644 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -11,6 +11,7 @@ from django.urls import reverse from django.utils.html import format_html, format_html_join from django.utils.http import urlquote from django.utils.safestring import mark_safe +from djangoql.admin import DjangoQLSearchMixin from documents.actions import ( add_tag_to_selected, @@ -165,7 +166,7 @@ class TagAdmin(CommonAdmin): document_count.admin_order_field = "document_count" -class DocumentAdmin(CommonAdmin): +class DocumentAdmin(DjangoQLSearchMixin, CommonAdmin): class Media: css = { diff --git a/src/paperless/settings.py b/src/paperless/settings.py index cb8851f62..cd12ad2f0 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -83,7 +83,7 @@ INSTALLED_APPS = [ "rest_framework", "crispy_forms", "django_filters", - + "djangoql", ] if os.getenv("PAPERLESS_INSTALLED_APPS"): From 5ecdb37f56cb8c47bf089fc37d7bfc78185a9915 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 12:52:15 +0000 Subject: [PATCH 10/14] pep8 --- src/documents/serialisers.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index b497a1dd0..0075d4de7 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -7,7 +7,14 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Correspondent - fields = ("id", "slug", "name", "match", "matching_algorithm", "is_insensitive") + fields = ( + "id", + "slug", + "name", + "match", + "matching_algorithm", + "is_insensitive" + ) class TagSerializer(serializers.HyperlinkedModelSerializer): @@ -15,7 +22,14 @@ class TagSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Tag fields = ( - "id", "slug", "name", "colour", "match", "matching_algorithm", "is_insensitive") + "id", + "slug", + "name", + "colour", + "match", + "matching_algorithm", + "is_insensitive" + ) class CorrespondentField(serializers.HyperlinkedRelatedField): From 310bc62ef2d7d00ae67b59ad42125cb7d2da7679 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:03:55 +0000 Subject: [PATCH 11/14] Move note about DEBUG up into 3. --- docs/setup.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/setup.rst b/docs/setup.rst index b3550b3e9..6781bb8c0 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -77,16 +77,19 @@ Standard (Bare Metal) encrypt/decrypt the original documents. Don't worry about defining this if you don't want to use encryption (the default). + Note also that if you're using the ``runserver`` as mentioned below, you + should make sure that PAPERLESS_DEBUG="true" or is just commented out as + this is the default. + 4. Initialise the SQLite database with ``./manage.py migrate``. 5. Create a user for your Paperless instance with ``./manage.py createsuperuser``. Follow the prompts to create your user. 6. Start the webserver with ``./manage.py runserver :``. - If no specifc IP or port are given, the default is ``127.0.0.1:8000`` - also known as http://localhost:8000/. + If no specific IP or port is given, the default is ``127.0.0.1:8000`` also + known as http://localhost:8000/. You should now be able to visit your (empty) installation at `Paperless webserver`_ or whatever you chose before. You can login with the user/pass you created in #5. - If using runserver within /etc/paperless.conf make sure PAPERLESS_DEBUG="true" 7. In a separate window, change to the ``src`` directory in this repo again, but this time, you should start the consumer script with From 868257065341f086f3a96048279a55a9af137b5f Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:05:56 +0000 Subject: [PATCH 12/14] Align example conf with real-world defaults --- paperless.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paperless.conf.example b/paperless.conf.example index a86523176..aed2eee71 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -61,7 +61,7 @@ PAPERLESS_EMAIL_SECRET="" # Controls whether django's debug mode is enabled. Disable this on production # systems. Debug mode is enabled by default. -PAPERLESS_DEBUG="false" +#PAPERLESS_DEBUG="true" # Paperless can be instructed to attempt to encrypt your PDF files with GPG From aa7610084d34a2ba53c89aaad9c2087f900d6fc5 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:46:10 +0000 Subject: [PATCH 13/14] Fix missing links --- docs/changelog.rst | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8157dd625..b5ffece99 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,23 @@ Changelog ######### +2.7.0 +===== + +* `syntonym`_ submitted a pull request to catch IMAP connection errors `#475`_. +* `Stéphane Brunner`_ added ``psycopg2`` to the Pipfile `#489`_. He also fixed + a syntax error in ``docker-compose.yml.example`` `#488`_ and added [DjangoQL](https://github.com/ivelum/djangoql), + which allows a litany of handy search functionality `#492`_. +* `CkuT`_ and `JOKer`_ hacked out a simple, but super-helpful optimisation to + how the thumbnails are served up, improving performance considerably `#481`_. +* `tsia`_ added a few fields to the tags REST API. `#483`_. +* `Brian Cribbs`_ improved the documentation to help people using Paperless + over NFS `#484`_. +* `Brendan M. Sleight`_ updated the documentation to include a note for setting the + ``DEBUG`` value. The ``paperless.conf.example`` file was also updated to + mirror the project defaults. + + 2.6.1 ===== @@ -10,7 +27,7 @@ Changelog that using the push API will work for users of the Docker install. Thanks to `Colin Frei`_ for fixing this in `#466`_. * `khrise`_ submitted a pull request to include the ``added`` property to the - REST API `#471`. + REST API `#471`_. 2.6.0 @@ -609,6 +626,10 @@ bulk of the work on this big change. .. _Sblop: https://github.com/Sblop .. _Colin Frei: https://github.com/colinfrei .. _khrise: https://github.com/khrise +.. _syntonym: https://github.com/syntonym +.. _JOKer: https://github.com/JOKer +.. _Brian Cribbs: https://github.com/cribbstechnolog +.. _Brendan M. Sleight: https://github.com/bmsleight .. _#20: https://github.com/danielquinn/paperless/issues/20 .. _#44: https://github.com/danielquinn/paperless/issues/44 @@ -710,6 +731,13 @@ bulk of the work on this big change. .. _#442: https://github.com/danielquinn/paperless/pull/442 .. _#466: https://github.com/danielquinn/paperless/pull/466 .. _#471: https://github.com/danielquinn/paperless/pull/471 +.. _#475: https://github.com/danielquinn/paperless/pull/475 +.. _#481: https://github.com/danielquinn/paperless/pull/481 +.. _#483: https://github.com/danielquinn/paperless/pull/483 +.. _#484: https://github.com/danielquinn/paperless/pull/484 +.. _#488: https://github.com/danielquinn/paperless/pull/488 +.. _#489: https://github.com/danielquinn/paperless/pull/489 +.. _#492: https://github.com/danielquinn/paperless/pull/492 .. _pipenv: https://docs.pipenv.org/ .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/ From 5bcd83357509fb5754e714a918af12c1a04c9e95 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:48:05 +0000 Subject: [PATCH 14/14] Ignore sqlite3-journal files too --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c5cfb7cc9..957d47743 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ media/overrides.js # Sqlite database db.sqlite3 +db.sqlite3-journal # PyCharm .idea