From 5c1edf78cef27808e9c70158c6f432a2557bfcbf Mon Sep 17 00:00:00 2001 From: syntonym Date: Fri, 11 Jan 2019 19:22:51 +0100 Subject: [PATCH 01/14] Catches OSError on IMAP connection error When something goes wrong with the imaplib.IMAP4_SSL connection (like the host is temporarely down or the DNS does not resolve) it generates an OSError which is currently not catched and handled. Now OSErrors are translated to MailFetcherErrors which get logged and the IMAP connection is retried in the next IMAP check. Fixes #474 --- src/documents/mail.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/documents/mail.py b/src/documents/mail.py index afa1b4362..d54b387b7 100644 --- a/src/documents/mail.py +++ b/src/documents/mail.py @@ -216,7 +216,11 @@ class MailFetcher(Loggable): return r def _connect(self): - self._connection = imaplib.IMAP4_SSL(self._host, self._port) + try: + self._connection = imaplib.IMAP4_SSL(self._host, self._port) + except OSError as e: + msg = "Problem connecting to {}: {}".format(self._host, e.strerror) + raise MailFetcherError(msg) def _login(self): From 29db177ce2f6a012f219eb14a050b4c756709ce5 Mon Sep 17 00:00:00 2001 From: Christian Wiwie Date: Fri, 18 Jan 2019 20:33:35 +0100 Subject: [PATCH 02/14] Update requirements.rst --- docs/requirements.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/requirements.rst b/docs/requirements.rst index b6cbad213..2bfc5cdec 100644 --- a/docs/requirements.rst +++ b/docs/requirements.rst @@ -12,6 +12,7 @@ should work) that has the following software installed: * `Imagemagick`_ version 6.7.5 or higher * `unpaper`_ * `libpoppler-cpp-dev`_ PDF rendering library +* `optipng`_ .. _Python3: https://python.org/ .. _GNU Privacy Guard: https://gnupg.org @@ -19,6 +20,7 @@ should work) that has the following software installed: .. _Imagemagick: http://imagemagick.org/ .. _unpaper: https://www.flameeyes.eu/projects/unpaper .. _libpoppler-cpp-dev: https://poppler.freedesktop.org/ +.. _optipng: http://optipng.sourceforge.net/ Notably, you should confirm how you access your Python3 installation. Many Linux distributions will install Python3 in parallel to Python2, using the From 79868930f1530676ff116491328344aaab0cf4b1 Mon Sep 17 00:00:00 2001 From: CkuT Date: Mon, 21 Jan 2019 20:04:57 +0100 Subject: [PATCH 03/14] Add Cache-Control header for thumbnails This drastically optimizes admin interface loading by telling the browser to cache thumbnails. The max-age recommendation is 1 year according to rfc2616 Closes #411 --- src/documents/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/documents/views.py b/src/documents/views.py index 186b733a5..f72ca22b1 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -2,6 +2,7 @@ from django.http import HttpResponse, HttpResponseBadRequest from django.views.generic import DetailView, FormView, TemplateView from django_filters.rest_framework import DjangoFilterBackend from django.conf import settings +from django.utils import cache from paperless.db import GnuPG from paperless.mixins import SessionOrBasicAuthMixin @@ -56,10 +57,12 @@ class FetchView(SessionOrBasicAuthMixin, DetailView): } if self.kwargs["kind"] == "thumb": - return HttpResponse( + response = HttpResponse( self._get_raw_data(self.object.thumbnail_file), content_type=content_types[Document.TYPE_PNG] ) + cache.patch_cache_control(response, max_age=31536000, private=True) + return response response = HttpResponse( self._get_raw_data(self.object.source_file), From 43e71cfcaa6516dbcaf6422897463d361f66208c Mon Sep 17 00:00:00 2001 From: tsia Date: Tue, 22 Jan 2019 20:51:20 +0100 Subject: [PATCH 04/14] added fields to the correspondent and tag REST API --- src/documents/serialisers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index bd00fe466..b497a1dd0 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -7,7 +7,7 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Correspondent - fields = ("id", "slug", "name") + fields = ("id", "slug", "name", "match", "matching_algorithm", "is_insensitive") class TagSerializer(serializers.HyperlinkedModelSerializer): @@ -15,7 +15,7 @@ class TagSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Tag fields = ( - "id", "slug", "name", "colour", "match", "matching_algorithm") + "id", "slug", "name", "colour", "match", "matching_algorithm", "is_insensitive") class CorrespondentField(serializers.HyperlinkedRelatedField): From 7c7a81409641c2c8725f3e0c26cbd91cc05662d1 Mon Sep 17 00:00:00 2001 From: Brian Cribbs Date: Tue, 22 Jan 2019 15:18:14 -0500 Subject: [PATCH 05/14] adding information about NFS mounts and inotify --- docs/setup.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/setup.rst b/docs/setup.rst index e86fe9efe..07dd555de 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -143,6 +143,15 @@ Docker Method instructions in comments in the file. The only change that is a hard requirement is to specify where the consumption directory should mount.[#dockercomposeyml]_ + + .. caution:: + + If you are using NFS mounts for the consume directory you also need to + change the command to turn off inotify as it doesn't work with NFS + + `command: ["document_consumer", "--no-inotify"]` + + 5. Modify ``docker-compose.env`` and adapt the following environment variables: ``PAPERLESS_PASSPHRASE`` From 5e674f17afcc2d1219e235a54c1d6e74144c688d Mon Sep 17 00:00:00 2001 From: bmsleight Date: Sat, 26 Jan 2019 13:15:45 +0000 Subject: [PATCH 06/14] Add note runserver PAPERLESS_DEBUG='true' --- docs/setup.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/setup.rst b/docs/setup.rst index e86fe9efe..75bc5eb78 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -86,6 +86,7 @@ Standard (Bare Metal) You should now be able to visit your (empty) installation at `Paperless webserver`_ or whatever you chose before. You can login with the user/pass you created in #5. + If using runserver within /etc/paperless.conf make sure PAPERLESS_DEBUG="true" 7. In a separate window, change to the ``src`` directory in this repo again, but this time, you should start the consumer script with From 0509d5a3d2d4ec8a4876bbdce0f72f6d375dec44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sat, 26 Jan 2019 14:22:21 +0100 Subject: [PATCH 07/14] Tabs are not allowed in Yaml files --- docker-compose.yml.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml.example b/docker-compose.yml.example index fb905832e..312467b91 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml.example @@ -17,9 +17,9 @@ services: volumes: - data:/usr/src/paperless/data - media:/usr/src/paperless/media - # You have to adapt the local path you want the consumption + # You have to adapt the local path you want the consumption # directory to mount to by modifying the part before the ':'. - - ./consume:/consume + - ./consume:/consume env_file: docker-compose.env # The reason the line is here is so that the webserver that doesn't do # any text recognition and doesn't have to install unnecessary @@ -40,7 +40,7 @@ services: - data:/usr/src/paperless/data - media:/usr/src/paperless/media # This should be set to the same value as the consume directory - # in the webserver service above. + # in the webserver service above. - ./consume:/consume # Likewise, you can add a local path to mount a directory for # exporting. This is not strictly needed for paperless to From 25e953bbf0dbd25e7134661fe896bcdcae3d5f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sat, 26 Jan 2019 14:33:51 +0100 Subject: [PATCH 08/14] Install psycopg2 in the Docker container --- Pipfile | 1 + Pipfile.lock | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/Pipfile b/Pipfile index 47b4c8663..7cf7db7e6 100644 --- a/Pipfile +++ b/Pipfile @@ -34,6 +34,7 @@ pytest-django = "*" pytest-sugar = "*" pytest-env = "*" pytest-xdist = "*" +psycopg2 = "*" [dev-packages] ipython = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5cbeecc69..395688203 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -365,6 +365,41 @@ ], "version": "==0.8.0" }, + "psycopg2": { + "hashes": [ + "sha256:10e391687b171878181e71736d0effe3772314a339d9ae30995ec8171a0c834e", + "sha256:1283f9d45e458c2dcb15ba89367923563f90ef636fe78ee22df75183484a0237", + "sha256:1a9c32e4d140bea225f9821d993b2e53c913e717ea97b851246aa9b300095d8f", + "sha256:1be6f2438d2b71fec7b07c3c0949dd321b04349c382907ea76b36120edec8300", + "sha256:20ca6f29e118b8dd7133e8708b3fba2881e70a4e0841f874ed23985b7201a076", + "sha256:227c115b3c1f65d61385e51ac690b91b584640aefb45bffacd4bd33d02ed7221", + "sha256:27959abe64ca1fc6d8cd11a71a1f421d8287831a3262bd4cacd43bbf43cc3c82", + "sha256:2b2daf1fe30a58300542aea679fd87d1e1c2afd36e7644837b7954fa2dbacb92", + "sha256:36e51a51f295fdf67bcf05e7b1877011a6b39e6622b0013fe31c5025241873a3", + "sha256:3992b9b914f2eb77dc07e8045d2ca979e491612808bc5c7cd68f307469acf9f6", + "sha256:39a11de2335ad45ececed43ab851d36a4c52843d756471b940804f301792781e", + "sha256:3c2afe9ef0d1649005e3ccf93c1aaccd6f8ee379530e763d3b3b77f406b7c0ae", + "sha256:3fb18e0e52807fe3a300dc1b5421aa492d5e759550918f597d61863419482535", + "sha256:55eab94de96ee9702f23283e9c8b03cfdb0001e2b14d5d2e1bd5ff8114b96b9f", + "sha256:7e95c0ab7e7e6e452586f35d4d8966b1e924c8dd2c23977e3ea4968770ff1d26", + "sha256:7f47514dbddf604f196fcfe5da955537f04691bef8124aff5632316a78d992b7", + "sha256:8345370356bb4bddf93acbcfd0357163dd6b09471937adcfb38a2fbb49bdce53", + "sha256:8bc6ecb220c0b88d3742042013129c817c44459795c97e9ce1bca70a3f37a53b", + "sha256:8df623f248be15d1725faf5f333791678775047f12f17a90d29b5d22573f5cdc", + "sha256:9645f1305e4268cc0fc88c823cd6c91de27c003e183c233a6a230e5e963039ee", + "sha256:a68719ed5be8373dd72c9e45d55f7a202285e05a2e392eaa8872a67ea47d7d20", + "sha256:aca0edf062ec09e954fdf0cc93d3a872362701210983a1442549e703aedec25d", + "sha256:b0dd2114d93d8f424bb8ae76e0dc540f104b70ca9163172c05e7700b1459d4c9", + "sha256:b2c09359d6802279efb9efb3f91a9c94567151baee95175f9b637ea628f35244", + "sha256:ca7bc37b1efb7cc25271bf10f398462ed975d95259af1406d38fcb268466e34f", + "sha256:e64235d9013ebf6319cb9654e08f5066112c34d8c4cc41186254ab9c3d6d5b9b", + "sha256:ec9be679c0065667503851141c31fa699e1cc69ded3ba8e5d3673dd5a6eb1370", + "sha256:eca00d0f91fcb44d88b12f1fd16ad138e38fa07debb79587e2b7ff1fe80d72b9", + "sha256:f256e807b8b2b45b6af60d7f2bb5194aab2f4acc861241c4d8ef942a55f5030d", + "sha256:fce7612a3bd6a7ba95799f88285653bf130bd7ca066b52674d5f850108b2aec0" + ], + "version": "==0.8.0" + }, "py": { "hashes": [ "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", From 4efb153e866c62cbac056471de62b0545a083891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Sat, 26 Jan 2019 22:10:13 +0100 Subject: [PATCH 09/14] Add and configure DjangoQL --- Pipfile | 1 + Pipfile.lock | 14 ++++++++++++++ requirements.txt | 1 + src/documents/admin.py | 3 ++- src/paperless/settings.py | 2 +- 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 47b4c8663..ccb577812 100644 --- a/Pipfile +++ b/Pipfile @@ -34,6 +34,7 @@ pytest-django = "*" pytest-sugar = "*" pytest-env = "*" pytest-xdist = "*" +djangoql = "*" [dev-packages] ipython = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5cbeecc69..c3492b434 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -155,6 +155,13 @@ "index": "pypi", "version": "==2.0.0" }, + "djangoql": { + "hashes": [ + "sha256:7c488ec4e3362e5389ba3c1169d1ff9a27c4222601f32c6dbf6130ce04330d76" + ], + "index": "pypi", + "version": "==0.12.3" + }, "djangorestframework": { "hashes": [ "sha256:607865b0bb1598b153793892101d881466bd5a991de12bd6229abb18b1c86136", @@ -365,6 +372,13 @@ ], "version": "==0.8.0" }, + "ply": { + "hashes": [ + "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3", + "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce" + ], + "version": "==3.11" + }, "py": { "hashes": [ "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", diff --git a/requirements.txt b/requirements.txt index d30cbf078..360ad310f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ django==2.0.9 djangorestframework==3.9.0 docopt==0.6.2 docutils==0.14 +djangoql==0.12.3 execnet==1.5.0 factory-boy==2.11.1 faker==0.9.2 diff --git a/src/documents/admin.py b/src/documents/admin.py index ff0365a53..a7c9e828e 100644 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -11,6 +11,7 @@ from django.urls import reverse from django.utils.html import format_html, format_html_join from django.utils.http import urlquote from django.utils.safestring import mark_safe +from djangoql.admin import DjangoQLSearchMixin from documents.actions import ( add_tag_to_selected, @@ -165,7 +166,7 @@ class TagAdmin(CommonAdmin): document_count.admin_order_field = "document_count" -class DocumentAdmin(CommonAdmin): +class DocumentAdmin(DjangoQLSearchMixin, CommonAdmin): class Media: css = { diff --git a/src/paperless/settings.py b/src/paperless/settings.py index cb8851f62..cd12ad2f0 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -83,7 +83,7 @@ INSTALLED_APPS = [ "rest_framework", "crispy_forms", "django_filters", - + "djangoql", ] if os.getenv("PAPERLESS_INSTALLED_APPS"): From 97f1e4ab16fc035242655051fc7d1de94dbb34f3 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 12:52:15 +0000 Subject: [PATCH 10/14] pep8 --- src/documents/serialisers.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index b497a1dd0..0075d4de7 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -7,7 +7,14 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Correspondent - fields = ("id", "slug", "name", "match", "matching_algorithm", "is_insensitive") + fields = ( + "id", + "slug", + "name", + "match", + "matching_algorithm", + "is_insensitive" + ) class TagSerializer(serializers.HyperlinkedModelSerializer): @@ -15,7 +22,14 @@ class TagSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Tag fields = ( - "id", "slug", "name", "colour", "match", "matching_algorithm", "is_insensitive") + "id", + "slug", + "name", + "colour", + "match", + "matching_algorithm", + "is_insensitive" + ) class CorrespondentField(serializers.HyperlinkedRelatedField): From a58a7ce0f71d7273b3cb6d64947a2a0508601cfd Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:03:55 +0000 Subject: [PATCH 11/14] Move note about DEBUG up into 3. --- docs/setup.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/setup.rst b/docs/setup.rst index b3550b3e9..6781bb8c0 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -77,16 +77,19 @@ Standard (Bare Metal) encrypt/decrypt the original documents. Don't worry about defining this if you don't want to use encryption (the default). + Note also that if you're using the ``runserver`` as mentioned below, you + should make sure that PAPERLESS_DEBUG="true" or is just commented out as + this is the default. + 4. Initialise the SQLite database with ``./manage.py migrate``. 5. Create a user for your Paperless instance with ``./manage.py createsuperuser``. Follow the prompts to create your user. 6. Start the webserver with ``./manage.py runserver :``. - If no specifc IP or port are given, the default is ``127.0.0.1:8000`` - also known as http://localhost:8000/. + If no specific IP or port is given, the default is ``127.0.0.1:8000`` also + known as http://localhost:8000/. You should now be able to visit your (empty) installation at `Paperless webserver`_ or whatever you chose before. You can login with the user/pass you created in #5. - If using runserver within /etc/paperless.conf make sure PAPERLESS_DEBUG="true" 7. In a separate window, change to the ``src`` directory in this repo again, but this time, you should start the consumer script with From 3c78105fd733b1a1adb072dc4316a1bf08cc2421 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:05:56 +0000 Subject: [PATCH 12/14] Align example conf with real-world defaults --- paperless.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paperless.conf.example b/paperless.conf.example index a86523176..aed2eee71 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -61,7 +61,7 @@ PAPERLESS_EMAIL_SECRET="" # Controls whether django's debug mode is enabled. Disable this on production # systems. Debug mode is enabled by default. -PAPERLESS_DEBUG="false" +#PAPERLESS_DEBUG="true" # Paperless can be instructed to attempt to encrypt your PDF files with GPG From 9f56bf9992d31a0bd5b98f15adab54d9288aa57d Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:46:10 +0000 Subject: [PATCH 13/14] Fix missing links --- docs/changelog.rst | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8157dd625..b5ffece99 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,23 @@ Changelog ######### +2.7.0 +===== + +* `syntonym`_ submitted a pull request to catch IMAP connection errors `#475`_. +* `Stéphane Brunner`_ added ``psycopg2`` to the Pipfile `#489`_. He also fixed + a syntax error in ``docker-compose.yml.example`` `#488`_ and added [DjangoQL](https://github.com/ivelum/djangoql), + which allows a litany of handy search functionality `#492`_. +* `CkuT`_ and `JOKer`_ hacked out a simple, but super-helpful optimisation to + how the thumbnails are served up, improving performance considerably `#481`_. +* `tsia`_ added a few fields to the tags REST API. `#483`_. +* `Brian Cribbs`_ improved the documentation to help people using Paperless + over NFS `#484`_. +* `Brendan M. Sleight`_ updated the documentation to include a note for setting the + ``DEBUG`` value. The ``paperless.conf.example`` file was also updated to + mirror the project defaults. + + 2.6.1 ===== @@ -10,7 +27,7 @@ Changelog that using the push API will work for users of the Docker install. Thanks to `Colin Frei`_ for fixing this in `#466`_. * `khrise`_ submitted a pull request to include the ``added`` property to the - REST API `#471`. + REST API `#471`_. 2.6.0 @@ -609,6 +626,10 @@ bulk of the work on this big change. .. _Sblop: https://github.com/Sblop .. _Colin Frei: https://github.com/colinfrei .. _khrise: https://github.com/khrise +.. _syntonym: https://github.com/syntonym +.. _JOKer: https://github.com/JOKer +.. _Brian Cribbs: https://github.com/cribbstechnolog +.. _Brendan M. Sleight: https://github.com/bmsleight .. _#20: https://github.com/danielquinn/paperless/issues/20 .. _#44: https://github.com/danielquinn/paperless/issues/44 @@ -710,6 +731,13 @@ bulk of the work on this big change. .. _#442: https://github.com/danielquinn/paperless/pull/442 .. _#466: https://github.com/danielquinn/paperless/pull/466 .. _#471: https://github.com/danielquinn/paperless/pull/471 +.. _#475: https://github.com/danielquinn/paperless/pull/475 +.. _#481: https://github.com/danielquinn/paperless/pull/481 +.. _#483: https://github.com/danielquinn/paperless/pull/483 +.. _#484: https://github.com/danielquinn/paperless/pull/484 +.. _#488: https://github.com/danielquinn/paperless/pull/488 +.. _#489: https://github.com/danielquinn/paperless/pull/489 +.. _#492: https://github.com/danielquinn/paperless/pull/492 .. _pipenv: https://docs.pipenv.org/ .. _a new home on Docker Hub: https://hub.docker.com/r/danielquinn/paperless/ From 94c2950afeb5025cd6d19edffdd6ed89ec73f7b0 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sun, 27 Jan 2019 13:48:05 +0000 Subject: [PATCH 14/14] Ignore sqlite3-journal files too --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c5cfb7cc9..957d47743 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ media/overrides.js # Sqlite database db.sqlite3 +db.sqlite3-journal # PyCharm .idea