From 572e40ca2713bc5d79fc3ee5fb852b0c7b14d6db Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Sat, 7 Nov 2020 11:30:45 +0100
Subject: [PATCH 01/54] backend that supports asgi and status update sockets
 with channels

---
 Pipfile                            |   2 +
 Pipfile.lock                       | 582 +++++++++++++++++++++++++----
 src/documents/consumer.py          |  44 ++-
 src/documents/parsers.py           |   3 +-
 src/paperless/asgi.py              |  37 ++
 src/paperless/settings.py          |  12 +
 src/paperless_tesseract/parsers.py |  25 +-
 7 files changed, 613 insertions(+), 92 deletions(-)
 create mode 100644 src/paperless/asgi.py

diff --git a/Pipfile b/Pipfile
index e8f862578..d526ae252 100644
--- a/Pipfile
+++ b/Pipfile
@@ -26,6 +26,8 @@ fuzzywuzzy = "*"
 python-Levenshtein = "*"
 django-extensions = ""
 watchdog = "*"
+channels = "~=3.0"
+channels-redis = "*"
 
 [dev-packages]
 coveralls = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 8b3bf705a..642e38214 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "2c1558fe7df0aee1ee20b095c2102f802470bf4a4ae09a7749ac487f8bfab8b6"
+            "sha256": "192d7419b844e6bb81fed793e7766b2ba15f2a016af1a33fc73cf09e12de5fb7"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -14,13 +14,151 @@
         ]
     },
     "default": {
+        "aioredis": {
+            "hashes": [
+                "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a",
+                "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"
+            ],
+            "version": "==1.3.1"
+        },
         "asgiref": {
             "hashes": [
-                "sha256:7e51911ee147dd685c3c8b805c0ad0cb58d360987b56953878f8c06d2d1c6f1a",
-                "sha256:9fc6fb5d39b8af147ba40765234fa822b39818b12cc80b35ad9b0cef3a476aed"
+                "sha256:a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e",
+                "sha256:cd88907ecaec59d78e4ac00ea665b03e571cb37e3a0e37b3702af1a9e86c365a"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.2.10"
+            "version": "==3.3.0"
+        },
+        "async-timeout": {
+            "hashes": [
+                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
+                "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
+            ],
+            "markers": "python_full_version >= '3.5.3'",
+            "version": "==3.0.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.3.0"
+        },
+        "autobahn": {
+            "hashes": [
+                "sha256:24ce276d313e84d68241c3aef30d484f352b90a40168981b3640312c821df77b",
+                "sha256:86bbce30cdd407137c57670993a8f9bfdfe3f8e994b889181d85e844d5aa8dfb"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==20.7.1"
+        },
+        "automat": {
+            "hashes": [
+                "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33",
+                "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"
+            ],
+            "version": "==20.2.0"
+        },
+        "cffi": {
+            "hashes": [
+                "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d",
+                "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b",
+                "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4",
+                "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f",
+                "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3",
+                "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579",
+                "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537",
+                "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e",
+                "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05",
+                "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171",
+                "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca",
+                "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522",
+                "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c",
+                "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc",
+                "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d",
+                "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808",
+                "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828",
+                "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869",
+                "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d",
+                "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9",
+                "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0",
+                "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc",
+                "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15",
+                "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c",
+                "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a",
+                "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3",
+                "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1",
+                "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768",
+                "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d",
+                "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b",
+                "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e",
+                "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d",
+                "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730",
+                "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394",
+                "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1",
+                "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"
+            ],
+            "version": "==1.14.3"
+        },
+        "channels": {
+            "hashes": [
+                "sha256:5cdd9c6b9ee663cdf1bbb00de7cdab885a3c418f9d32a29f04b09498828020f6",
+                "sha256:b02e150b48704ec3607d4168402ac5c26138dd183fcdb7f2aeb965e6e19fd558"
+            ],
+            "index": "pypi",
+            "version": "==3.0.1"
+        },
+        "channels-redis": {
+            "hashes": [
+                "sha256:18d63f6462a58011740dc8eeb57ea4b31ec220eb551cb71b27de9c6779a549de",
+                "sha256:2fb31a63b05373f6402da2e6a91a22b9e66eb8b56626c6bfc93e156c734c5ae6"
+            ],
+            "index": "pypi",
+            "version": "==3.2.0"
+        },
+        "constantly": {
+            "hashes": [
+                "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35",
+                "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"
+            ],
+            "version": "==15.1.0"
+        },
+        "cryptography": {
+            "hashes": [
+                "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538",
+                "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f",
+                "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77",
+                "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b",
+                "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33",
+                "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e",
+                "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb",
+                "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e",
+                "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7",
+                "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297",
+                "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d",
+                "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7",
+                "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b",
+                "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7",
+                "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4",
+                "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8",
+                "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b",
+                "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851",
+                "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13",
+                "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b",
+                "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3",
+                "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==3.2.1"
+        },
+        "daphne": {
+            "hashes": [
+                "sha256:60856f7efa0b1e1b969efa074e8698bd09de4713ecc06e6a4d19d04c66c4a3bd",
+                "sha256:b43e70d74ff832a634ff6c92badd208824e4530e08b340116517e5aad0aca774"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.0.0"
         },
         "dateparser": {
             "hashes": [
@@ -32,11 +170,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:a2127ad0150ec6966655bedf15dbbff9697cc86d61653db2da1afa506c0b04cc",
-                "sha256:c93c28ccf1d094cbd00d860e83128a39e45d2c571d3b54361713aaaf9a94cac4"
+                "sha256:14a4b7cd77297fba516fc0d92444cc2e2e388aa9de32d7a68d4a83d58f5a4927",
+                "sha256:14b87775ffedab2ef6299b73343d1b4b41e5d4e2aa58c6581f114dbec01e3f8f"
             ],
             "index": "pypi",
-            "version": "==3.1.2"
+            "version": "==3.1.3"
         },
         "django-cors-headers": {
             "hashes": [
@@ -65,11 +203,10 @@
         },
         "djangorestframework": {
             "hashes": [
-                "sha256:5c5071fcbad6dce16f566d492015c829ddb0df42965d488b878594aabc3aed21",
-                "sha256:d54452aedebb4b650254ca092f9f4f5df947cb1de6ab245d817b08b4f4156249"
+                "sha256:0209bafcb7b5010fdfec784034f059d512256424de2a0f084cb82b096d6dd6a7"
             ],
             "index": "pypi",
-            "version": "==3.12.1"
+            "version": "==3.12.2"
         },
         "filemagic": {
             "hashes": [
@@ -94,6 +231,80 @@
             "index": "pypi",
             "version": "==20.0.4"
         },
+        "hiredis": {
+            "hashes": [
+                "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
+                "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0",
+                "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0",
+                "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01",
+                "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a",
+                "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b",
+                "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6",
+                "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73",
+                "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee",
+                "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55",
+                "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12",
+                "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b",
+                "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323",
+                "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c",
+                "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655",
+                "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5",
+                "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75",
+                "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb",
+                "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23",
+                "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1",
+                "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f",
+                "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872",
+                "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058",
+                "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454",
+                "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882",
+                "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2",
+                "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132",
+                "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6",
+                "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c",
+                "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363",
+                "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3",
+                "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4",
+                "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919",
+                "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349",
+                "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae",
+                "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da",
+                "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f",
+                "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed",
+                "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628",
+                "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64",
+                "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86",
+                "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf",
+                "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c",
+                "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded",
+                "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390",
+                "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.1.0"
+        },
+        "hyperlink": {
+            "hashes": [
+                "sha256:47fcc7cd339c6cb2444463ec3277bdcfe142c8b1daf2160bdd52248deec815af",
+                "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"
+            ],
+            "version": "==20.0.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.10"
+        },
+        "incremental": {
+            "hashes": [
+                "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
+                "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"
+            ],
+            "version": "==17.5.0"
+        },
         "joblib": {
             "hashes": [
                 "sha256:698c311779f347cf6b7e6b8a39bb682277b8ee4aba8cf9507bc0cf4cd4737b72",
@@ -110,45 +321,68 @@
             "index": "pypi",
             "version": "==1.0.8"
         },
+        "msgpack": {
+            "hashes": [
+                "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408",
+                "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8",
+                "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84",
+                "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d",
+                "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a",
+                "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322",
+                "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2",
+                "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e",
+                "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97",
+                "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0",
+                "sha256:aa5c057eab4f40ec47ea6f5a9825846be2ff6bf34102c560bad5cad5a677c5be",
+                "sha256:b3758dfd3423e358bbb18a7cccd1c74228dffa7a697e5be6cb9535de625c0dbf",
+                "sha256:c901e8058dd6653307906c5f157f26ed09eb94a850dddd989621098d347926ab",
+                "sha256:cec8bf10981ed70998d98431cd814db0ecf3384e6b113366e7f36af71a0fca08",
+                "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e",
+                "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272",
+                "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1",
+                "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140"
+            ],
+            "version": "==1.0.0"
+        },
         "numpy": {
             "hashes": [
-                "sha256:0ee77786eebbfa37f2141fd106b549d37c89207a0d01d8852fde1c82e9bfc0e7",
-                "sha256:199bebc296bd8a5fc31c16f256ac873dd4d5b4928dfd50e6c4995570fc71a8f3",
-                "sha256:1a307bdd3dd444b1d0daa356b5f4c7de2e24d63bdc33ea13ff718b8ec4c6a268",
-                "sha256:1ea7e859f16e72ab81ef20aae69216cfea870676347510da9244805ff9670170",
-                "sha256:271139653e8b7a046d11a78c0d33bafbddd5c443a5b9119618d0652a4eb3a09f",
-                "sha256:35bf5316af8dc7c7db1ad45bec603e5fb28671beb98ebd1d65e8059efcfd3b72",
-                "sha256:463792a249a81b9eb2b63676347f996d3f0082c2666fd0604f4180d2e5445996",
-                "sha256:50d3513469acf5b2c0406e822d3f314d7ac5788c2b438c24e5dd54d5a81ef522",
-                "sha256:50f68ebc439821b826823a8da6caa79cd080dee2a6d5ab9f1163465a060495ed",
-                "sha256:51e8d2ae7c7e985c7bebf218e56f72fa93c900ad0c8a7d9fbbbf362f45710f69",
-                "sha256:522053b731e11329dd52d258ddf7de5288cae7418b55e4b7d32f0b7e31787e9d",
-                "sha256:5ea4401ada0d3988c263df85feb33818dc995abc85b8125f6ccb762009e7bc68",
-                "sha256:604d2e5a31482a3ad2c88206efd43d6fcf666ada1f3188fd779b4917e49b7a98",
-                "sha256:6ff88bcf1872b79002569c63fe26cd2cda614e573c553c4d5b814fb5eb3d2822",
-                "sha256:7197ee0a25629ed782c7bd01871ee40702ffeef35bc48004bc2fdcc71e29ba9d",
-                "sha256:741d95eb2b505bb7a99fbf4be05fa69f466e240c2b4f2d3ddead4f1b5f82a5a5",
-                "sha256:83af653bb92d1e248ccf5fdb05ccc934c14b936bcfe9b917dc180d3f00250ac6",
-                "sha256:8802d23e4895e0c65e418abe67cdf518aa5cbb976d97f42fd591f921d6dffad0",
-                "sha256:8edc4d687a74d0a5f8b9b26532e860f4f85f56c400b3a98899fc44acb5e27add",
-                "sha256:942d2cdcb362739908c26ce8dd88db6e139d3fa829dd7452dd9ff02cba6b58b2",
-                "sha256:9a0669787ba8c9d3bb5de5d9429208882fb47764aa79123af25c5edc4f5966b9",
-                "sha256:9d08d84bb4128abb9fbd9f073e5c69f70e5dab991a9c42e5b4081ea5b01b5db0",
-                "sha256:9f7f56b5e85b08774939622b7d45a5d00ff511466522c44fc0756ac7692c00f2",
-                "sha256:a2daea1cba83210c620e359de2861316f49cc7aea8e9a6979d6cb2ddab6dda8c",
-                "sha256:b9074d062d30c2779d8af587924f178a539edde5285d961d2dfbecbac9c4c931",
-                "sha256:c4aa79993f5d856765819a3651117520e41ac3f89c3fc1cb6dee11aa562df6da",
-                "sha256:d78294f1c20f366cde8a75167f822538a7252b6e8b9d6dbfb3bdab34e7c1929e",
-                "sha256:dfdc8b53aa9838b9d44ed785431ca47aa3efaa51d0d5dd9c412ab5247151a7c4",
-                "sha256:dffed17848e8b968d8d3692604e61881aa6ef1f8074c99e81647ac84f6038535",
-                "sha256:e080087148fd70469aade2abfeadee194357defd759f9b59b349c6192aba994c",
-                "sha256:e983cbabe10a8989333684c98fdc5dd2f28b236216981e0c26ed359aaa676772",
-                "sha256:ea6171d2d8d648dee717457d0f75db49ad8c2f13100680e284d7becf3dc311a6",
-                "sha256:eefc13863bf01583a85e8c1121a901cc7cb8f059b960c4eba30901e2e6aba95f",
-                "sha256:efd656893171bbf1331beca4ec9f2e74358fc732a2084f664fd149cc4b3441d2"
+                "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db",
+                "sha256:09c12096d843b90eafd01ea1b3307e78ddd47a55855ad402b157b6c4862197ce",
+                "sha256:13d166f77d6dc02c0a73c1101dd87fdf01339febec1030bd810dcd53fff3b0f1",
+                "sha256:141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512",
+                "sha256:16c1b388cc31a9baa06d91a19366fb99ddbe1c7b205293ed072211ee5bac1ed2",
+                "sha256:18bed2bcb39e3f758296584337966e68d2d5ba6aab7e038688ad53c8f889f757",
+                "sha256:1aeef46a13e51931c0b1cf8ae1168b4a55ecd282e6688fdb0a948cc5a1d5afb9",
+                "sha256:27d3f3b9e3406579a8af3a9f262f5339005dd25e0ecf3cf1559ff8a49ed5cbf2",
+                "sha256:2a2740aa9733d2e5b2dfb33639d98a64c3b0f24765fed86b0fd2aec07f6a0a08",
+                "sha256:4377e10b874e653fe96985c05feed2225c912e328c8a26541f7fc600fb9c637b",
+                "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb",
+                "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc",
+                "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac",
+                "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83",
+                "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36",
+                "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387",
+                "sha256:8cac8790a6b1ddf88640a9267ee67b1aee7a57dfa2d2dd33999d080bc8ee3a0f",
+                "sha256:8ece138c3a16db8c1ad38f52eb32be6086cc72f403150a79336eb2045723a1ad",
+                "sha256:9eeb7d1d04b117ac0d38719915ae169aa6b61fca227b0b7d198d43728f0c879c",
+                "sha256:a09f98011236a419ee3f49cedc9ef27d7a1651df07810ae430a6b06576e0b414",
+                "sha256:a5d897c14513590a85774180be713f692df6fa8ecf6483e561a6d47309566f37",
+                "sha256:ad6f2ff5b1989a4899bf89800a671d71b1612e5ff40866d1f4d8bcf48d4e5764",
+                "sha256:c42c4b73121caf0ed6cd795512c9c09c52a7287b04d105d112068c1736d7c753",
+                "sha256:cb1017eec5257e9ac6209ac172058c430e834d5d2bc21961dceeb79d111e5909",
+                "sha256:d6c7bb82883680e168b55b49c70af29b84b84abb161cbac2800e8fcb6f2109b6",
+                "sha256:e452dc66e08a4ce642a961f134814258a082832c78c90351b75c41ad16f79f63",
+                "sha256:e5b6ed0f0b42317050c88022349d994fe72bfe35f5908617512cd8c8ef9da2a9",
+                "sha256:e9b30d4bd69498fc0c3fe9db5f62fffbb06b8eb9321f92cc970f2969be5e3949",
+                "sha256:ec149b90019852266fec2341ce1db513b843e496d5a8e8cdb5ced1923a92faab",
+                "sha256:edb01671b3caae1ca00881686003d16c2209e07b7ef8b7639f1867852b948f7c",
+                "sha256:f0d3929fe88ee1c155129ecd82f981b8856c5d97bcb0d5f23e9b4242e79d1de3",
+                "sha256:f29454410db6ef8126c83bd3c968d143304633d45dc57b51252afbd79d700893",
+                "sha256:fe45becb4c2f72a0907c1d0246ea6449fe7a9e2293bb0e11c4e9a32bb0930a15",
+                "sha256:fedbd128668ead37f33917820b704784aff695e0019309ad446a6d0b065b57e4"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==1.19.3"
+            "version": "==1.19.4"
         },
         "pathtools": {
             "hashes": [
@@ -202,9 +436,11 @@
                 "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c",
                 "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67",
                 "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0",
+                "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6",
                 "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db",
                 "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94",
                 "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52",
+                "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056",
                 "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b",
                 "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd",
                 "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550",
@@ -236,6 +472,58 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "pyasn1": {
+            "hashes": [
+                "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359",
+                "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576",
+                "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf",
+                "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7",
+                "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d",
+                "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00",
+                "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8",
+                "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86",
+                "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12",
+                "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776",
+                "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba",
+                "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2",
+                "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"
+            ],
+            "version": "==0.4.8"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8",
+                "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199",
+                "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811",
+                "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed",
+                "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4",
+                "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e",
+                "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74",
+                "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb",
+                "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45",
+                "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd",
+                "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0",
+                "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d",
+                "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"
+            ],
+            "version": "==0.2.8"
+        },
+        "pycparser": {
+            "hashes": [
+                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
+                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.20"
+        },
+        "pyhamcrest": {
+            "hashes": [
+                "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316",
+                "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.2"
+        },
         "pyocr": {
             "hashes": [
                 "sha256:fa15adc7e1cf0d345a2990495fe125a947c6e09a60ddba0256a1c14b2e603179"
@@ -243,6 +531,13 @@
             "index": "pypi",
             "version": "==0.7.2"
         },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504",
+                "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507"
+            ],
+            "version": "==19.1.0"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
@@ -276,10 +571,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed",
-                "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"
+                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
+                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
             ],
-            "version": "==2020.1"
+            "version": "==2020.4"
         },
         "regex": {
             "hashes": [
@@ -287,26 +582,42 @@
                 "sha256:06b52815d4ad38d6524666e0d50fe9173533c9cc145a5779b89733284e6f688f",
                 "sha256:11116d424734fe356d8777f89d625f0df783251ada95d6261b4c36ad27a394bb",
                 "sha256:119e0355dbdd4cf593b17f2fc5dbd4aec2b8899d0057e4957ba92f941f704bf5",
+                "sha256:127a9e0c0d91af572fbb9e56d00a504dbd4c65e574ddda3d45b55722462210de",
                 "sha256:1ec66700a10e3c75f1f92cbde36cca0d3aaee4c73dfa26699495a3a30b09093c",
+                "sha256:227a8d2e5282c2b8346e7f68aa759e0331a0b4a890b55a5cfbb28bd0261b84c0",
+                "sha256:2564def9ce0710d510b1fc7e5178ce2d20f75571f788b5197b3c8134c366f50c",
+                "sha256:297116e79074ec2a2f885d22db00ce6e88b15f75162c5e8b38f66ea734e73c64",
                 "sha256:2dc522e25e57e88b4980d2bdd334825dbf6fa55f28a922fc3bfa60cc09e5ef53",
                 "sha256:3a5f08039eee9ea195a89e180c5762bfb55258bfb9abb61a20d3abee3b37fd12",
+                "sha256:3dfca201fa6b326239e1bccb00b915e058707028809b8ecc0cf6819ad233a740",
                 "sha256:49461446b783945597c4076aea3f49aee4b4ce922bd241e4fcf62a3e7c61794c",
                 "sha256:4afa350f162551cf402bfa3cd8302165c8e03e689c897d185f16a167328cc6dd",
                 "sha256:4b5a9bcb56cc146c3932c648603b24514447eafa6ce9295234767bf92f69b504",
+                "sha256:52e83a5f28acd621ba8e71c2b816f6541af7144b69cc5859d17da76c436a5427",
                 "sha256:625116aca6c4b57c56ea3d70369cacc4d62fead4930f8329d242e4fe7a58ce4b",
                 "sha256:654c1635f2313d0843028487db2191530bca45af61ca85d0b16555c399625b0e",
                 "sha256:8092a5a06ad9a7a247f2a76ace121183dc4e1a84c259cf9c2ce3bbb69fac3582",
                 "sha256:832339223b9ce56b7b15168e691ae654d345ac1635eeb367ade9ecfe0e66bee0",
                 "sha256:8ca9dca965bd86ea3631b975d63b0693566d3cc347e55786d5514988b6f5b84c",
+                "sha256:96f99219dddb33e235a37283306834700b63170d7bb2a1ee17e41c6d589c8eb9",
+                "sha256:9b6305295b6591e45f069d3553c54d50cc47629eb5c218aac99e0f7fafbf90a1",
                 "sha256:a62162be05edf64f819925ea88d09d18b09bebf20971b363ce0c24e8b4aa14c0",
+                "sha256:aacc8623ffe7999a97935eeabbd24b1ae701d08ea8f874a6ff050e93c3e658cf",
+                "sha256:b45bab9f224de276b7bc916f6306b86283f6aa8afe7ed4133423efb42015a898",
                 "sha256:b88fa3b8a3469f22b4f13d045d9bd3eda797aa4e406fde0a2644bc92bbdd4bdd",
+                "sha256:b8a686a6c98872007aa41fdbb2e86dc03b287d951ff4a7f1da77fb7f14113e4d",
+                "sha256:bd904c0dec29bbd0769887a816657491721d5f545c29e30fd9d7a1a275dc80ab",
+                "sha256:bf4f896c42c63d1f22039ad57de2644c72587756c0cfb3cc3b7530cfe228277f",
                 "sha256:c13d311a4c4a8d671f5860317eb5f09591fbe8259676b86a85769423b544451e",
                 "sha256:c2c6c56ee97485a127555c9595c069201b5161de9d05495fbe2132b5ac104786",
+                "sha256:c32c91a0f1ac779cbd73e62430de3d3502bbc45ffe5bb6c376015acfa848144b",
                 "sha256:c3466a84fce42c2016113101018a9981804097bacbab029c2d5b4fcb224b89de",
+                "sha256:c454ad88e56e80e44f824ef8366bb7e4c3def12999151fd5c0ea76a18fe9aa3e",
                 "sha256:c8a2b7ccff330ae4c460aff36626f911f918555660cc28163417cb84ffb25789",
                 "sha256:cb905f3d2e290a8b8f1579d3984f2cfa7c3a29cc7cba608540ceeed18513f520",
                 "sha256:cfcf28ed4ce9ced47b9b9670a4f0d3d3c0e4d4779ad4dadb1ad468b097f808aa",
                 "sha256:dd3e6547ecf842a29cf25123fbf8d2461c53c8d37aa20d87ecee130c89b7079b",
+                "sha256:de7fd57765398d141949946c84f3590a68cf5887dac3fc52388df0639b01eda4",
                 "sha256:ea37320877d56a7f0a1e6a625d892cf963aa7f570013499f5b8d5ab8402b5625",
                 "sha256:f1fce1e4929157b2afeb4bb7069204d4370bab9f4fc03ca1fbec8bd601f8c87d",
                 "sha256:f43109822df2d3faac7aad79613f5f02e4eab0fc8ad7932d2e70e2a83bd49c26"
@@ -337,28 +648,41 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:07b083128beae040f1129bd8a82b01804f5e716a7fd2962c1053fa683433e4ab",
-                "sha256:0edd67e8a00903aaf7a29c968555a2e27c5a69fea9d1dcfffda80614281a884f",
-                "sha256:12fdcbfa56cac926a0a9364a30cbf4ad03c2c7b59f75b14234656a5e4fd52bf3",
-                "sha256:1fee28b6641ecbff6e80fe7788e50f50c5576157d278fa40f36c851940eb0aff",
-                "sha256:33e6a7439f43f37d4c1135bc95bcd490ffeac6ef4b374892c7005ce2c729cf4a",
-                "sha256:5163200ab14fd2b83aba8f0c4ddcc1fa982a43192867264ab0f4c8065fd10d17",
-                "sha256:66ec29348444ed6e8a14c9adc2de65e74a8fc526dc2c770741725464488ede1f",
-                "sha256:8cc5c39ed287a8b52a5509cd6680af078a40b0e010e2657eca01ffbfec929468",
-                "sha256:a1a13858b10d41beb0413c4378462b43eafef88a1948d286cb357eadc0aec024",
-                "sha256:a3db1fe7c6cb29ca02b14c9141151ebafd11e06ffb6da8ecd330eee5c8283a8a",
-                "sha256:aebb69bcdec209d874fc4b0c7ac36f509d50418a431c1422465fa34c2c0143ea",
-                "sha256:b9751b39c52a3fa59312bd2e1f40144ee26b51404db5d2f0d5259c511ff6f614",
-                "sha256:bc0e63daf43bf052aefbbd6c5424bc03f629d115ece828e87303a0bcc04a37e4",
-                "sha256:d5e3cc60868f396b78fc881d2c76460febccfe90f6d2f082b9952265c79a8788",
-                "sha256:ddae76784574cc4c172f3d5edd7308be16078dd3b977e8746860c76c195fa707",
-                "sha256:e2602f79c85924e4486f684aa9bbab74afff90606100db88d0785a0088be7edb",
-                "sha256:e527c9221b6494bcd06a17f9f16874406b32121385f9ab353b8a9545be458f0b",
-                "sha256:f574558f1b774864516f3c3fe072ebc90a29186f49b720f60ed339294b7f32ac",
-                "sha256:ffcbd331f1ffa82e22f1d408e93c37463c9a83088243158635baec61983aaacf"
+                "sha256:168c45c0c32e23f613db7c9e4e780bc61982d71dcd406ead746c7c7c2f2004ce",
+                "sha256:213bc59191da2f479984ad4ec39406bf949a99aba70e9237b916ce7547b6ef42",
+                "sha256:25b241034215247481f53355e05f9e25462682b13bd9191359075682adcd9554",
+                "sha256:2c872de0c69ed20fb1a9b9cf6f77298b04a26f0b8720a5457be08be254366c6e",
+                "sha256:3397c129b479846d7eaa18f999369a24322d008fac0782e7828fa567358c36ce",
+                "sha256:368c0f69f93186309e1b4beb8e26d51dd6f5010b79264c0f1e9ca00cd92ea8c9",
+                "sha256:3d5db5d815370c28d938cf9b0809dade4acf7aba57eaf7ef733bfedc9b2474c4",
+                "sha256:4598cf03136067000855d6b44d7a1f4f46994164bcd450fb2c3d481afc25dd06",
+                "sha256:4a453d5e5689de62e5d38edf40af3f17560bfd63c9c5bd228c18c1f99afa155b",
+                "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47",
+                "sha256:634568a3018bc16a83cda28d4f7aed0d803dd5618facb36e977e53b2df868443",
+                "sha256:65923bc3809524e46fb7eb4d6346552cbb6a1ffc41be748535aa502a2e3d3389",
+                "sha256:6b0ceb23560f46dd236a8ad4378fc40bad1783e997604ba845e131d6c680963e",
+                "sha256:8c8d6ca19c8497344b810b0b0344f8375af5f6bb9c98bd42e33f747417ab3f57",
+                "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62",
+                "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d",
+                "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437",
+                "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2",
+                "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54",
+                "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474",
+                "sha256:e360cb2299028d0b0d0f65a5c5e51fc16a335f1603aa2357c25766c8dab56938",
+                "sha256:e98d49a5717369d8241d6cf33ecb0ca72deee392414118198a8e5b4c35c56340",
+                "sha256:ed572470af2438b526ea574ff8f05e7f39b44ac37f712105e57fc4d53a6fb660",
+                "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012",
+                "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==1.5.3"
+            "version": "==1.5.4"
+        },
+        "service-identity": {
+            "hashes": [
+                "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
+                "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"
+            ],
+            "version": "==18.1.0"
         },
         "six": {
             "hashes": [
@@ -384,6 +708,46 @@
             "markers": "python_version >= '3.5'",
             "version": "==2.1.0"
         },
+        "twisted": {
+            "extras": [
+                "tls"
+            ],
+            "hashes": [
+                "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f",
+                "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042",
+                "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c",
+                "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292",
+                "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22",
+                "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec",
+                "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478",
+                "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2",
+                "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29",
+                "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114",
+                "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797",
+                "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa",
+                "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15",
+                "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd",
+                "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274",
+                "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad",
+                "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7",
+                "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a",
+                "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10",
+                "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780",
+                "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504",
+                "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467",
+                "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==20.3.0"
+        },
+        "txaio": {
+            "hashes": [
+                "sha256:17938f2bca4a9cabce61346758e482ca4e600160cbc28e861493eac74a19539d",
+                "sha256:38a469daf93c37e5527cb062653d6393ae11663147c42fab7ddc3f6d00d434ae"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==20.4.1"
+        },
         "tzlocal": {
             "hashes": [
                 "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44",
@@ -414,6 +778,64 @@
             ],
             "index": "pypi",
             "version": "==2.7.4"
+        },
+        "zope.interface": {
+            "hashes": [
+                "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
+                "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d",
+                "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123",
+                "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232",
+                "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549",
+                "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102",
+                "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5",
+                "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45",
+                "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00",
+                "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc",
+                "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7",
+                "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104",
+                "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034",
+                "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3",
+                "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3",
+                "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4",
+                "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86",
+                "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96",
+                "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546",
+                "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb",
+                "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3",
+                "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b",
+                "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b",
+                "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec",
+                "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae",
+                "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e",
+                "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386",
+                "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2",
+                "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a",
+                "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d",
+                "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a",
+                "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24",
+                "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d",
+                "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b",
+                "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50",
+                "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523",
+                "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a",
+                "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095",
+                "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a",
+                "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520",
+                "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65",
+                "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11",
+                "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c",
+                "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7",
+                "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332",
+                "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e",
+                "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c",
+                "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7",
+                "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20",
+                "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc",
+                "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd",
+                "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==5.2.0"
         }
     },
     "develop": {
@@ -441,11 +863,11 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594",
-                "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.2.0"
+            "version": "==20.3.0"
         },
         "babel": {
             "hashes": [
@@ -556,11 +978,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:30afa8f564350770373f299d2d267bff42aaba699a7ae0a3b6f378b2a8170569",
-                "sha256:a7a36c3c657f06bd1e3e3821b9480f2a92017d8a26e150e464ab6b97743cbc92"
+                "sha256:6afc461ab3f779c9c16e299fc731d775e39ea7e8e063b3053ee359ae198a15ca",
+                "sha256:ce1c38823eb0f927567cde5bf2e7c8ca565c7a70316139342050ce2ca74b4026"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==4.14.0"
+            "version": "==4.14.2"
         },
         "filelock": {
             "hashes": [
@@ -751,10 +1173,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed",
-                "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"
+                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
+                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
             ],
-            "version": "==2020.1"
+            "version": "==2020.4"
         },
         "requests": {
             "hashes": [
@@ -781,11 +1203,11 @@
         },
         "sphinx": {
             "hashes": [
-                "sha256:321d6d9b16fa381a5306e5a0b76cd48ffbc588e6340059a729c6fdd66087e0e8",
-                "sha256:ce6fd7ff5b215af39e2fcd44d4a321f6694b4530b6f2b2109b64d120773faea0"
+                "sha256:1c21e7c5481a31b531e6cbf59c3292852ccde175b504b00ce2ff0b8f4adc3649",
+                "sha256:3abdb2c57a65afaaa4f8573cbabd5465078eb6fd282c1e4f87f006875a7ec0c7"
             ],
             "index": "pypi",
-            "version": "==3.2.1"
+            "version": "==3.3.0"
         },
         "sphinxcontrib-applehelp": {
             "hashes": [
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index f61d11136..3a73fc0cd 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -5,6 +5,8 @@ import os
 import re
 import uuid
 
+from asgiref.sync import async_to_sync
+from channels.layers import get_channel_layer
 from django.conf import settings
 from django.db import transaction
 from django.utils import timezone
@@ -33,6 +35,17 @@ class Consumer:
       5. Delete the document and image(s)
     """
 
+    def _send_progress(self, filename, current_progress, max_progress, status, message, document_id=None):
+        payload = {
+            'filename': os.path.basename(filename),
+            'current_progress': current_progress,
+            'max_progress': max_progress,
+            'status': status,
+            'message': message,
+            'document_id': document_id
+        }
+        async_to_sync(self.channel_layer.group_send)("status_updates", {'type': 'status_update', 'data': payload})
+
     def __init__(self, consume=settings.CONSUMPTION_DIR,
                  scratch=settings.SCRATCH_DIR):
 
@@ -44,6 +57,8 @@ class Consumer:
 
         self.classifier = DocumentClassifier()
 
+        self.channel_layer = get_channel_layer()
+
         os.makedirs(self.scratch, exist_ok=True)
 
         self.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
@@ -60,7 +75,6 @@ class Consumer:
             raise ConsumerError(
                 "Consumption directory {} does not exist".format(self.consume))
 
-
     def log(self, level, message):
         getattr(self.logger, level)(message, extra={
             "group": self.logging_group
@@ -88,6 +102,7 @@ class Consumer:
 
         self.log("info", "Consuming {}".format(doc))
 
+
         parser_class = get_parser_class(doc)
         if not parser_class:
             self.log(
@@ -96,6 +111,7 @@ class Consumer:
         else:
             self.log("info", "Parser: {}".format(parser_class.__name__))
 
+        self._send_progress(file, 0, 100, 'WORKING', 'Consumption started')
 
         document_consumption_started.send(
             sender=self.__class__,
@@ -103,20 +119,37 @@ class Consumer:
             logging_group=self.logging_group
         )
 
-        document_parser = parser_class(doc, self.logging_group)
+        def progress_callback(current_progress, max_progress, message):
+            # recalculate progress to be within 20 and 80
+            p = int((current_progress / max_progress) * 60 + 20)
+            self._send_progress(file, p, 100, "WORKING", message)
+
+        document_parser = parser_class(doc, self.logging_group, progress_callback)
 
         try:
             self.log("info", "Generating thumbnail for {}...".format(doc))
+            self._send_progress(file, 10, 100, 'WORKING',
+                                'Generating thumbnail...')
             thumbnail = document_parser.get_optimised_thumbnail()
+            self._send_progress(file, 20, 100, 'WORKING',
+                                'Getting text from document...')
+            text = document_parser.get_text()
+            self._send_progress(file, 80, 100, 'WORKING',
+                                'Getting date from document...')
             date = document_parser.get_date()
+            self._send_progress(file, 85, 100, 'WORKING',
+                                'Storing the document...')
             document = self._store(
-                document_parser.get_text(),
+                text,
                 doc,
                 thumbnail,
                 date
             )
         except ParseError as e:
             self.log("fatal", "PARSE FAILURE for {}: {}".format(doc, e))
+            self._send_progress(file, 100, 100, 'FAILED',
+                                "Failed: {}".format(e))
+
             document_parser.cleanup()
             return False
         else:
@@ -136,12 +169,17 @@ class Consumer:
             except (FileNotFoundError, IncompatibleClassifierVersionError) as e:
                 logging.getLogger(__name__).warning("Cannot classify documents: {}.".format(e))
 
+            self._send_progress(file, 90, 100, 'WORKING',
+                                'Performing post-consumption tasks...')
+
             document_consumption_finished.send(
                 sender=self.__class__,
                 document=document,
                 logging_group=self.logging_group,
                 classifier=classifier
             )
+            self._send_progress(file, 100, 100, 'SUCCESS',
+                                'Finished.', document.id)
             return True
 
     def _store(self, text, doc, thumbnail, date):
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index 0cbd13987..7976f4739 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -106,11 +106,12 @@ class DocumentParser:
     `paperless_tesseract.parsers` for inspiration.
     """
 
-    def __init__(self, path, logging_group):
+    def __init__(self, path, logging_group, progress_callback):
         self.document_path = path
         self.tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR)
         self.logger = logging.getLogger(__name__)
         self.logging_group = logging_group
+        self.progress_callback = progress_callback
 
     def get_thumbnail(self):
         """
diff --git a/src/paperless/asgi.py b/src/paperless/asgi.py
new file mode 100644
index 000000000..9c3d17b1b
--- /dev/null
+++ b/src/paperless/asgi.py
@@ -0,0 +1,37 @@
+import json
+import os
+
+from asgiref.sync import async_to_sync
+from channels.auth import AuthMiddlewareStack
+from channels.generic.websocket import WebsocketConsumer
+from channels.routing import ProtocolTypeRouter, URLRouter
+from django.core.asgi import get_asgi_application
+from django.urls import re_path
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'paperless.settings')
+
+
+class StatusConsumer(WebsocketConsumer):
+    def connect(self):
+        self.accept()
+        async_to_sync(self.channel_layer.group_add)('status_updates', self.channel_name)
+
+    def disconnect(self, close_code):
+        async_to_sync(self.channel_layer.group_discard)('status_updates', self.channel_name)
+
+    def status_update(self, event):
+        self.send(json.dumps(event['data']))
+
+
+websocket_urlpatterns = [
+    re_path(r'ws/status/$', StatusConsumer.as_asgi()),
+]
+
+application = ProtocolTypeRouter({
+    "http": get_asgi_application(),
+    "websocket": AuthMiddlewareStack(
+        URLRouter(
+            websocket_urlpatterns
+        )
+    ),
+})
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index bb71e4764..c9db3e4b1 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -69,6 +69,8 @@ INSTALLED_APPS = [
     "rest_framework.authtoken",
     "django_filters",
 
+    "channels",
+
 ]
 
 REST_FRAMEWORK = {
@@ -98,6 +100,7 @@ LOGIN_URL = "admin:login"
 FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME")
 
 WSGI_APPLICATION = 'paperless.wsgi.application'
+ASGI_APPLICATION = "paperless.asgi.application"
 
 STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", "/static/")
 
@@ -299,3 +302,12 @@ FILENAME_DATE_ORDER = os.getenv("PAPERLESS_FILENAME_DATE_ORDER")
 FILENAME_PARSE_TRANSFORMS = []
 for t in json.loads(os.getenv("PAPERLESS_FILENAME_PARSE_TRANSFORMS", "[]")):
     FILENAME_PARSE_TRANSFORMS.append((re.compile(t["pattern"]), t["repl"]))
+
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels_redis.core.RedisChannelLayer",
+        "CONFIG": {
+            "hosts": [("127.0.0.1", 6379)],
+        },
+    },
+}
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index befc9bcd7..535ee501c 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -27,8 +27,8 @@ class RasterisedDocumentParser(DocumentParser):
     image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.)
     """
 
-    def __init__(self, path, logging_group):
-        super().__init__(path, logging_group)
+    def __init__(self, path, logging_group, progress_callback):
+        super().__init__(path, logging_group, progress_callback)
         self._text = None
 
     def get_thumbnail(self):
@@ -91,6 +91,7 @@ class RasterisedDocumentParser(DocumentParser):
             self._text = get_text_from_pdf(self.document_path)
             return self._text
 
+        self.progress_callback(0,1,"Making greyscale images.")
         images = self._get_greyscale()
 
         if not images:
@@ -100,8 +101,10 @@ class RasterisedDocumentParser(DocumentParser):
 
             sample_page_index = int(len(images) / 2)
             self.log("info", "Attempting language detection on page {} of {}...".format(sample_page_index+1, len(images)))
+            self.progress_callback(0.4, 1, "Language Detection.")
             sample_page_text = self._ocr([images[sample_page_index]], settings.OCR_LANGUAGE)[0]
             guessed_language = self._guess_language(sample_page_text)
+            self.progress_callback(0.6, 1, "OCR all the pages.")
 
             if not guessed_language or guessed_language not in ISO639:
                 self.log("warning", "Language detection failed.")
@@ -117,7 +120,7 @@ class RasterisedDocumentParser(DocumentParser):
 
             else:
                 self.log("info", "Detected language: {}".format(guessed_language))
-                ocr_pages = self._ocr(images, ISO639[guessed_language])
+                ocr_pages = self._ocr(images, ISO639[guessed_language], report_progress=True)
 
             self.log("info", "OCR completed.")
             self._text = strip_excess_whitespace(" ".join(ocr_pages))
@@ -151,6 +154,8 @@ class RasterisedDocumentParser(DocumentParser):
 
         self.log("info", "Running unpaper on {} pages...".format(len(pnms)))
 
+        self.progress_callback(0.2,1, "Running unpaper on {} pages...".format(len(pnms)))
+
         # Run unpaper in parallel on converted images
         with Pool(processes=settings.OCR_THREADS) as pool:
             pnms = pool.map(run_unpaper, pnms)
@@ -165,11 +170,16 @@ class RasterisedDocumentParser(DocumentParser):
             self.log('debug', "Language detection failed with: {}".format(e))
             return None
 
-    def _ocr(self, imgs, lang):
+    def _ocr(self, imgs, lang, report_progress=False):
         self.log("info", "Performing OCR on {} page(s) with language {}".format(len(imgs), lang))
+        r = []
         with Pool(processes=settings.OCR_THREADS) as pool:
-            r = pool.map(image_to_string, itertools.product(imgs, [lang]))
-            return r
+            # r = pool.map(image_to_string, itertools.product(imgs, [lang]))
+            for i, page in enumerate(pool.imap(image_to_string, itertools.product(imgs, [lang]))):
+                if report_progress:
+                    self.progress_callback(0.6 + (i / len(imgs)) * 0.4, 1, "OCR'ed {} pages".format(i+1))
+                r += [page]
+        return r
 
     def _complete_ocr_default_language(self, images, sample_page_index, sample_page):
         """
@@ -182,14 +192,13 @@ class RasterisedDocumentParser(DocumentParser):
         del images_copy[sample_page_index]
         if images_copy:
             self.log('info', 'Continuing ocr with default language.')
-            ocr_pages = self._ocr(images_copy, settings.OCR_LANGUAGE)
+            ocr_pages = self._ocr(images_copy, settings.OCR_LANGUAGE, report_progress=True)
             ocr_pages.insert(sample_page_index, sample_page)
             return ocr_pages
         else:
             return [sample_page]
 
 
-
 def strip_excess_whitespace(text):
     collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
     no_leading_whitespace = re.sub(

From 036f11acaae7368b322ce62c31d85e3ebbd35ee9 Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Sat, 7 Nov 2020 12:05:15 +0100
Subject: [PATCH 02/54] better toasts, better dashboard, first implementation
 of consumer status

---
 src-ui/src/app/app.component.ts               | 35 ++++++++-
 src-ui/src/app/app.module.ts                  | 10 ++-
 .../edit-dialog/edit-dialog.component.ts      |  4 +-
 .../common/toasts/toasts.component.html       |  5 +-
 .../dashboard/dashboard.component.html        | 56 ++-------------
 .../dashboard/dashboard.component.ts          | 53 +-------------
 .../consumer-status-widget.component.css      |  0
 .../consumer-status-widget.component.html     | 10 +++
 .../consumer-status-widget.component.spec.ts  | 25 +++++++
 .../consumer-status-widget.component.ts       | 35 +++++++++
 .../file-upload-widget.component.css          |  0
 .../file-upload-widget.component.html         | 11 +++
 .../file-upload-widget.component.spec.ts      | 25 +++++++
 .../file-upload-widget.component.ts           | 44 ++++++++++++
 .../saved-view-widget.component.css           |  0
 .../saved-view-widget.component.html          | 16 +++++
 .../saved-view-widget.component.spec.ts       | 25 +++++++
 .../saved-view-widget.component.ts            | 41 +++++++++++
 .../statistics-widget.component.css           |  0
 .../statistics-widget.component.html          |  3 +
 .../statistics-widget.component.spec.ts       | 25 +++++++
 .../statistics-widget.component.ts            | 32 +++++++++
 .../app/components/login/login.component.ts   |  4 +-
 src-ui/src/app/services/auth.interceptor.ts   |  2 +-
 .../services/consumer-status.service.spec.ts  | 16 +++++
 .../app/services/consumer-status.service.ts   | 71 +++++++++++++++++++
 src-ui/src/app/services/toast.service.ts      | 33 ++++-----
 src-ui/src/environments/environment.ts        |  3 +-
 28 files changed, 450 insertions(+), 134 deletions(-)
 create mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.css
 create mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
 create mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts
 create mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
 create mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.css
 create mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html
 create mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts
 create mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts
 create mode 100644 src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.css
 create mode 100644 src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html
 create mode 100644 src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.spec.ts
 create mode 100644 src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts
 create mode 100644 src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.css
 create mode 100644 src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html
 create mode 100644 src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.spec.ts
 create mode 100644 src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts
 create mode 100644 src-ui/src/app/services/consumer-status.service.spec.ts
 create mode 100644 src-ui/src/app/services/consumer-status.service.ts

diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts
index a6cd8bebe..5809968e8 100644
--- a/src-ui/src/app/app.component.ts
+++ b/src-ui/src/app/app.component.ts
@@ -1,14 +1,43 @@
-import { Component } from '@angular/core';
+import { Component, OnDestroy, OnInit } from '@angular/core';
+import { Router } from '@angular/router';
+import { Subscription } from 'rxjs';
+import { ConsumerStatusService } from './services/consumer-status.service';
+import { Toast, ToastService } from './services/toast.service';
 
 @Component({
   selector: 'app-root',
   templateUrl: './app.component.html',
   styleUrls: ['./app.component.css']
 })
-export class AppComponent {
+export class AppComponent implements OnInit, OnDestroy {
+
+  successSubscription: Subscription;
+  failedSubscription: Subscription;
   
-  constructor () {
+  constructor ( private consumerStatusService: ConsumerStatusService, private toastService: ToastService, private router: Router ) {
   }
 
+  ngOnDestroy(): void {
+    this.consumerStatusService.disconnect()
+    this.successSubscription.unsubscribe()
+    this.failedSubscription.unsubscribe()
+  }
+
+  ngOnInit(): void {
+    this.consumerStatusService.connect()
+
+    this.successSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
+      this.toastService.showToast({title: "Document added", content: `Document ${status.filename} was added to paperless.`, actionName: "Open document", action: () => {
+        this.router.navigate(['documents', status.document_id])
+      }})
+    })
+
+    this.failedSubscription = this.consumerStatusService.onDocumentConsumptionFailed().subscribe(status => {
+      this.toastService.showError(`Could not consume ${status.filename}: ${status.message}`)
+    })
+
+  }
+
+  
 
 }
diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts
index e10bdbd0c..5cc59d567 100644
--- a/src-ui/src/app/app.module.ts
+++ b/src-ui/src/app/app.module.ts
@@ -40,6 +40,10 @@ import { SaveViewConfigDialogComponent } from './components/document-list/save-v
 import { InfiniteScrollModule } from 'ngx-infinite-scroll';
 import { DateTimeComponent } from './components/common/input/date-time/date-time.component';
 import { TagsComponent } from './components/common/input/tags/tags.component';
+import { SavedViewWidgetComponent } from './components/dashboard/widgets/saved-view-widget/saved-view-widget.component';
+import { ConsumerStatusWidgetComponent } from './components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component';
+import { StatisticsWidgetComponent } from './components/dashboard/widgets/statistics-widget/statistics-widget.component';
+import { FileUploadWidgetComponent } from './components/dashboard/widgets/file-upload-widget/file-upload-widget.component';
 
 @NgModule({
   declarations: [
@@ -73,7 +77,11 @@ import { TagsComponent } from './components/common/input/tags/tags.component';
     CheckComponent,
     SaveViewConfigDialogComponent,
     DateTimeComponent,
-    TagsComponent
+    TagsComponent,
+    ConsumerStatusWidgetComponent,
+    SavedViewWidgetComponent,
+    StatisticsWidgetComponent,
+    FileUploadWidgetComponent
   ],
   imports: [
     BrowserModule,
diff --git a/src-ui/src/app/components/common/edit-dialog/edit-dialog.component.ts b/src-ui/src/app/components/common/edit-dialog/edit-dialog.component.ts
index ba0d90847..04eb1f250 100644
--- a/src-ui/src/app/components/common/edit-dialog/edit-dialog.component.ts
+++ b/src-ui/src/app/components/common/edit-dialog/edit-dialog.component.ts
@@ -5,7 +5,7 @@ import { Observable } from 'rxjs';
 import { MATCHING_ALGORITHMS } from 'src/app/data/matching-model';
 import { ObjectWithId } from 'src/app/data/object-with-id';
 import { AbstractPaperlessService } from 'src/app/services/rest/abstract-paperless-service';
-import { Toast, ToastService } from 'src/app/services/toast.service';
+import { ToastService } from 'src/app/services/toast.service';
 
 @Directive()
 export abstract class EditDialogComponent<T extends ObjectWithId> implements OnInit {
@@ -66,7 +66,7 @@ export abstract class EditDialogComponent<T extends ObjectWithId> implements OnI
       this.activeModal.close()
       this.success.emit(result)
     }, error => {
-      this.toastService.showToast(Toast.makeError(`Could not save ${this.entityName}: ${error.error.name}`))
+      this.toastService.showError(`Could not save ${this.entityName}: ${error.error.name}`)
     })
   }
 
diff --git a/src-ui/src/app/components/common/toasts/toasts.component.html b/src-ui/src/app/components/common/toasts/toasts.component.html
index 04aa15a67..4e920877e 100644
--- a/src-ui/src/app/components/common/toasts/toasts.component.html
+++ b/src-ui/src/app/components/common/toasts/toasts.component.html
@@ -1,7 +1,8 @@
 <ngb-toast
   *ngFor="let toast of toasts"
-  [header]="toast.title" [autohide]="true" [delay]="toast.delay"
+  [header]="toast.title" [autohide]="true" [delay]="toast.delay || 5000"
   [class]="toast.classname"
   (hide)="toastService.closeToast(toast)">
-  {{toast.content}}
+  <p>{{toast.content}}</p>
+  <p *ngIf="toast.action"><button class="btn btn-sm btn-outline-secondary" (click)="toastService.closeToast(toast); toast.action()">{{toast.actionName}}</button></p>
 </ngb-toast>
\ No newline at end of file
diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html
index 694b431c4..4b732c9e8 100644
--- a/src-ui/src/app/components/dashboard/dashboard.component.html
+++ b/src-ui/src/app/components/dashboard/dashboard.component.html
@@ -6,59 +6,11 @@
 
 <div class='row'>
   <div class="col-lg">
-    <ng-container *ngFor="let v of savedDashboardViews">
-      <h4>{{v.viewConfig.title}}</h4>
-
-      <table class="table table-sm table-hover table-borderless">
-        <thead>
-        <tr>
-          <th>Date created</th>
-          <th scope="col">Document</th>
-        </tr>
-        </thead>
-        <tbody>
-          <tr *ngFor="let doc of v.documents" routerLink="/documents/{{doc.id}}">
-            <td>{{doc.created | date}}</td>
-            <td>{{doc.title}}<app-tag [tag]="t" *ngFor="let t of doc.tags" class="ml-1"></app-tag>
-          </tr>
-        </tbody>
-      </table>
-
-    </ng-container>
-    <ng-container *ngIf="savedDashboardViews.length == 0">
-      <h4>Saved views</h4>
-      <p>This space is reserved to display your saved views. Go to your documents and save a view to have it displayed here!</p>
-    </ng-container>
-
+    <app-saved-view-widget [viewConfig]="conf" *ngFor="let conf of savedViewConfigService.getDashboardConfigs()"></app-saved-view-widget>
   </div>
   <div class="col-lg">
-    <h4>Statistics</h4>
-    <p>Documents in inbox: {{statistics.documents_inbox}}</p>
-    <p>Total documents: {{statistics.documents_total}}</p>
-    <h4>Upload new Document</h4>
-    <form>
-      <ngx-file-drop
-        dropZoneLabel="Drop documents here"
-        (onFileDrop)="dropped($event)"
-        (onFileOver)="fileOver($event)"
-        (onFileLeave)="fileLeave($event)"
-        dropZoneClassName="bg-light mt-4 card">
-
-      </ngx-file-drop>
-    </form>
-    <h5 class="mt-3">Document conumser status</h5>
-    <p>This is what it might look like in the future.</p>
-    <div class="card bg-light mb-2">
-      <div class="card-body">
-        <p class="card-text"><strong>Filename.pdf:</strong> Running tesseract on page 4/8...</p>
-        <p><ngb-progressbar type="info" [value]="50"></ngb-progressbar></p>
-      </div>
-    </div>
-    <div class="card bg-light mb-2">
-      <div class="card-body">
-        <p class="card-text"><strong>Filename2.pdf:</strong> Completed.</p>
-        <p><ngb-progressbar type="success" [value]="100"></ngb-progressbar></p>
-      </div>
-    </div>
+    <app-statistics-widget></app-statistics-widget>
+    <app-file-upload-widget></app-file-upload-widget>
+    <app-consumer-status-widget></app-consumer-status-widget>
   </div>
 </div>
diff --git a/src-ui/src/app/components/dashboard/dashboard.component.ts b/src-ui/src/app/components/dashboard/dashboard.component.ts
index f8d5fb0ae..a9c72e496 100644
--- a/src-ui/src/app/components/dashboard/dashboard.component.ts
+++ b/src-ui/src/app/components/dashboard/dashboard.component.ts
@@ -4,14 +4,9 @@ import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop';
 import { Observable } from 'rxjs';
 import { DocumentService } from 'src/app/services/rest/document.service';
 import { SavedViewConfigService } from 'src/app/services/saved-view-config.service';
-import { Toast, ToastService } from 'src/app/services/toast.service';
+import { ToastService } from 'src/app/services/toast.service';
 import { environment } from 'src/environments/environment';
 
-export interface Statistics {
-  documents_total?: number
-  documents_inbox?: number
-}
-
 @Component({
   selector: 'app-dashboard',
   templateUrl: './dashboard.component.html',
@@ -19,53 +14,9 @@ export interface Statistics {
 })
 export class DashboardComponent implements OnInit {
 
-  constructor(private documentService: DocumentService, private toastService: ToastService,
-    public savedViewConfigService: SavedViewConfigService, private http: HttpClient) { }
-
-
-  savedDashboardViews = []
-  statistics: Statistics = {}
+  constructor(public savedViewConfigService: SavedViewConfigService) { }
 
   ngOnInit(): void {
-    this.savedViewConfigService.getDashboardConfigs().forEach(config => {
-      this.documentService.list(1,10,config.sortField,config.sortDirection,config.filterRules).subscribe(result => {
-        this.savedDashboardViews.push({viewConfig: config, documents: result.results})
-      })
-    })
-    this.getStatistics().subscribe(statistics => {
-      this.statistics = statistics
-    })
   }
 
-  getStatistics(): Observable<Statistics> {
-    return this.http.get(`${environment.apiBaseUrl}statistics/`)
-  }
-
-
-  public fileOver(event){
-    console.log(event);
-  }
- 
-  public fileLeave(event){
-    console.log(event);
-  }
- 
-  public dropped(files: NgxFileDropEntry[]) {
-    for (const droppedFile of files) {
-      if (droppedFile.fileEntry.isFile) {
-        const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
-        console.log(fileEntry)
-        fileEntry.file((file: File) => {
-          console.log(file)
-          const formData = new FormData()
-          formData.append('document', file, file.name)
-          this.documentService.uploadDocument(formData).subscribe(result => {
-            this.toastService.showToast(Toast.make("Information", "The document has been uploaded and will be processed by the consumer shortly."))
-          }, error => {
-            this.toastService.showToast(Toast.makeError("An error has occured while uploading the document. Sorry!"))
-          })
-        });
-      }
-    }
-  }
 }
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.css b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.css
new file mode 100644
index 000000000..e69de29bb
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
new file mode 100644
index 000000000..9f6aa3b87
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
@@ -0,0 +1,10 @@
+<h4 class="mt-3">Document consumer status</h4>
+
+<div class="p-2 mb-1 border bg-light" *ngFor="let s of getStatus()">
+  <div class="mb-1"><strong>{{s.filename}}:</strong> {{s.message}}</div>
+  <ngb-progressbar [type]="getType(s.status)" [value]="s.current_progress" [max]="s.max_progress" class="mb-2"></ngb-progressbar>
+  <div *ngIf="isFinished(s)">
+    <button *ngIf="s.document_id" class="btn btn-sm btn-outline-secondary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
+    <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(s)">Dismiss</button>
+  </div>
+</div>
\ No newline at end of file
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts
new file mode 100644
index 000000000..02aa088f5
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts
@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ConsumerStatusWidgetComponent } from './consumer-status-widget.component';
+
+describe('ConsumerStatusWidgetComponent', () => {
+  let component: ConsumerStatusWidgetComponent;
+  let fixture: ComponentFixture<ConsumerStatusWidgetComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ ConsumerStatusWidgetComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(ConsumerStatusWidgetComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
new file mode 100644
index 000000000..0c4e35682
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
@@ -0,0 +1,35 @@
+import { Component, OnInit } from '@angular/core';
+import { ConsumerStatusService, FileStatus } from 'src/app/services/consumer-status.service';
+
+@Component({
+  selector: 'app-consumer-status-widget',
+  templateUrl: './consumer-status-widget.component.html',
+  styleUrls: ['./consumer-status-widget.component.css']
+})
+export class ConsumerStatusWidgetComponent implements OnInit {
+
+  constructor(private consumerStatusService: ConsumerStatusService) { }
+
+  ngOnInit(): void {
+  }
+
+  getStatus() {
+    return this.consumerStatusService.consumerStatus
+  }
+
+  isFinished(status: FileStatus) {
+    return status.status == "FAILED" || status.status == "SUCCESS"
+  }
+
+  getType(status) {
+    switch (status) {
+      case "WORKING": return "primary"
+      case "FAILED": return "danger"
+      case "SUCCESS": return "success"
+    }
+  }
+
+  dismiss(status: FileStatus) {
+    this.consumerStatusService.dismiss(status)
+  }
+}
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.css b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.css
new file mode 100644
index 000000000..e69de29bb
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html
new file mode 100644
index 000000000..0c5ea634a
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html
@@ -0,0 +1,11 @@
+<h4>Upload new Document</h4>
+<form>
+  <ngx-file-drop
+    dropZoneLabel="Drop documents here"
+    (onFileDrop)="dropped($event)"
+    (onFileOver)="fileOver($event)"
+    (onFileLeave)="fileLeave($event)"
+    dropZoneClassName="bg-light mt-4 card">
+
+  </ngx-file-drop>
+</form>
\ No newline at end of file
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts
new file mode 100644
index 000000000..847f5288b
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts
@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { FileUploadWidgetComponent } from './file-upload-widget.component';
+
+describe('FileUploadWidgetComponent', () => {
+  let component: FileUploadWidgetComponent;
+  let fixture: ComponentFixture<FileUploadWidgetComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ FileUploadWidgetComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(FileUploadWidgetComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts
new file mode 100644
index 000000000..5d4bac936
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts
@@ -0,0 +1,44 @@
+import { Component, OnInit } from '@angular/core';
+import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop';
+import { DocumentService } from 'src/app/services/rest/document.service';
+import { ToastService } from 'src/app/services/toast.service';
+
+@Component({
+  selector: 'app-file-upload-widget',
+  templateUrl: './file-upload-widget.component.html',
+  styleUrls: ['./file-upload-widget.component.css']
+})
+export class FileUploadWidgetComponent implements OnInit {
+
+  constructor(private documentService: DocumentService, private toastService: ToastService) { }
+
+  ngOnInit(): void {
+  }
+
+  public fileOver(event){
+    console.log(event);
+  }
+ 
+  public fileLeave(event){
+    console.log(event);
+  }
+ 
+  public dropped(files: NgxFileDropEntry[]) {
+    for (const droppedFile of files) {
+      if (droppedFile.fileEntry.isFile) {
+        const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
+        console.log(fileEntry)
+        fileEntry.file((file: File) => {
+          console.log(file)
+          const formData = new FormData()
+          formData.append('document', file, file.name)
+          this.documentService.uploadDocument(formData).subscribe(result => {
+            this.toastService.showInfo("The document has been uploaded and will be processed by the consumer shortly.")
+          }, error => {
+            this.toastService.showError("An error has occured while uploading the document. Sorry!")
+          })
+        });
+      }
+    }
+  }
+}
diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.css b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.css
new file mode 100644
index 000000000..e69de29bb
diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html
new file mode 100644
index 000000000..110464641
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html
@@ -0,0 +1,16 @@
+<h4>{{viewConfig.title}}</h4>
+
+<table class="table table-sm table-hover table-borderless">
+  <thead>
+  <tr>
+    <th>Date created</th>
+    <th scope="col">Document</th>
+  </tr>
+  </thead>
+  <tbody>
+    <tr *ngFor="let doc of documents" routerLink="/documents/{{doc.id}}">
+      <td>{{doc.created | date}}</td>
+      <td>{{doc.title}}<app-tag [tag]="t" *ngFor="let t of doc.tags" class="ml-1"></app-tag>
+    </tr>
+  </tbody>
+</table>
\ No newline at end of file
diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.spec.ts b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.spec.ts
new file mode 100644
index 000000000..f0095b618
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.spec.ts
@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { SavedViewWidgetComponent } from './saved-view-widget.component';
+
+describe('SavedViewWidgetComponent', () => {
+  let component: SavedViewWidgetComponent;
+  let fixture: ComponentFixture<SavedViewWidgetComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ SavedViewWidgetComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(SavedViewWidgetComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts
new file mode 100644
index 000000000..eb2d53aee
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts
@@ -0,0 +1,41 @@
+import { Component, Input, OnDestroy, OnInit } from '@angular/core';
+import { Subscription } from 'rxjs';
+import { PaperlessDocument } from 'src/app/data/paperless-document';
+import { SavedViewConfig } from 'src/app/data/saved-view-config';
+import { ConsumerStatusService } from 'src/app/services/consumer-status.service';
+import { DocumentService } from 'src/app/services/rest/document.service';
+
+@Component({
+  selector: 'app-saved-view-widget',
+  templateUrl: './saved-view-widget.component.html',
+  styleUrls: ['./saved-view-widget.component.css']
+})
+export class SavedViewWidgetComponent implements OnInit, OnDestroy {
+
+  constructor(private documentService: DocumentService, private consumerStatusService: ConsumerStatusService) { }
+
+  @Input()
+  viewConfig: SavedViewConfig
+
+  documents: PaperlessDocument[]
+
+  subscription: Subscription
+
+  ngOnInit(): void {
+    this.reload()
+    this.subscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
+      this.reload()
+    })
+  }
+
+  ngOnDestroy(): void {
+    this.subscription.unsubscribe()
+  }
+
+  reload() {
+    this.documentService.list(1,10,this.viewConfig.sortField,this.viewConfig.sortDirection,this.viewConfig.filterRules).subscribe(result => {
+      this.documents = result.results
+    })
+  }
+
+}
diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.css b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.css
new file mode 100644
index 000000000..e69de29bb
diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html
new file mode 100644
index 000000000..2f89a2b34
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html
@@ -0,0 +1,3 @@
+<h4>Statistics</h4>
+<p>Documents in inbox: {{statistics.documents_inbox}}</p>
+<p>Total documents: {{statistics.documents_total}}</p>
\ No newline at end of file
diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.spec.ts b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.spec.ts
new file mode 100644
index 000000000..e8e44ca54
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.spec.ts
@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { StatisticsWidgetComponent } from './statistics-widget.component';
+
+describe('StatisticsWidgetComponent', () => {
+  let component: StatisticsWidgetComponent;
+  let fixture: ComponentFixture<StatisticsWidgetComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ StatisticsWidgetComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(StatisticsWidgetComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts
new file mode 100644
index 000000000..4efb03895
--- /dev/null
+++ b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts
@@ -0,0 +1,32 @@
+import { HttpClient } from '@angular/common/http';
+import { Component, OnInit } from '@angular/core';
+import { Observable } from 'rxjs';
+import { environment } from 'src/environments/environment';
+
+export interface Statistics {
+  documents_total?: number
+  documents_inbox?: number
+}
+
+@Component({
+  selector: 'app-statistics-widget',
+  templateUrl: './statistics-widget.component.html',
+  styleUrls: ['./statistics-widget.component.css']
+})
+export class StatisticsWidgetComponent implements OnInit {
+
+  constructor(private http: HttpClient) { }
+
+  statistics: Statistics = {}
+
+  ngOnInit(): void {
+    this.getStatistics().subscribe(statistics => {
+      this.statistics = statistics
+    })
+  }
+
+  getStatistics(): Observable<Statistics> {
+    return this.http.get(`${environment.apiBaseUrl}statistics/`)
+  }
+
+}
diff --git a/src-ui/src/app/components/login/login.component.ts b/src-ui/src/app/components/login/login.component.ts
index e74dcfb7f..a241543c7 100644
--- a/src-ui/src/app/components/login/login.component.ts
+++ b/src-ui/src/app/components/login/login.component.ts
@@ -2,7 +2,7 @@ import { Component, OnInit } from '@angular/core';
 import { FormControl, FormGroup } from '@angular/forms';
 import { Router } from '@angular/router';
 import { AuthService } from 'src/app/services/auth.service';
-import { Toast, ToastService } from 'src/app/services/toast.service';
+import { ToastService } from 'src/app/services/toast.service';
 
 @Component({
   selector: 'app-login',
@@ -26,7 +26,7 @@ export class LoginComponent implements OnInit {
     this.auth.login(this.loginForm.value.username, this.loginForm.value.password, this.loginForm.value.rememberMe).subscribe(result => {
       this.router.navigate([''])
     }, (error) => {
-      this.toastService.showToast(Toast.makeError("Unable to log in with provided credentials."))
+      this.toastService.showError("Unable to log in with provided credentials.")
     }
     )
   }
diff --git a/src-ui/src/app/services/auth.interceptor.ts b/src-ui/src/app/services/auth.interceptor.ts
index 704b558ac..37d9e7906 100644
--- a/src-ui/src/app/services/auth.interceptor.ts
+++ b/src-ui/src/app/services/auth.interceptor.ts
@@ -28,7 +28,7 @@ export class AuthInterceptor implements HttpInterceptor {
       catchError((error: HttpErrorResponse) => {
         if (error.status == 401 && this.authService.isAuthenticated()) {
           this.authService.logout()
-          this.toastService.showToast(Toast.makeError("Your session has expired. Please log in again."))
+          this.toastService.showError("Your session has expired. Please log in again.")
         }
         return throwError(error)
       })
diff --git a/src-ui/src/app/services/consumer-status.service.spec.ts b/src-ui/src/app/services/consumer-status.service.spec.ts
new file mode 100644
index 000000000..d19f455e2
--- /dev/null
+++ b/src-ui/src/app/services/consumer-status.service.spec.ts
@@ -0,0 +1,16 @@
+import { TestBed } from '@angular/core/testing';
+
+import { ConsumerStatusService } from './consumer-status.service';
+
+describe('ConsumerStatusService', () => {
+  let service: ConsumerStatusService;
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({});
+    service = TestBed.inject(ConsumerStatusService);
+  });
+
+  it('should be created', () => {
+    expect(service).toBeTruthy();
+  });
+});
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
new file mode 100644
index 000000000..070420b0f
--- /dev/null
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -0,0 +1,71 @@
+import { Injectable } from '@angular/core';
+import { Subject } from 'rxjs';
+
+export interface FileStatus {
+  filename?: string
+  current_progress?: number
+  max_progress?: number
+  status?: string
+  message?: string
+  document_id?: number
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class ConsumerStatusService {
+
+  constructor() { }
+
+  private statusWebSocked: WebSocket
+
+  consumerStatus: FileStatus[] = []
+  private documentConsumptionFinishedSubject = new Subject<FileStatus>()
+  private documentConsumptionFailedSubject = new Subject<FileStatus>()
+
+  connect() {
+    this.disconnect()
+    this.statusWebSocked = new WebSocket("ws://localhost:8000/ws/status/");
+    this.statusWebSocked.onmessage = (ev) => {
+      let statusUpdate: FileStatus = JSON.parse(ev['data'])
+
+      let index = this.consumerStatus.findIndex(fs => fs.filename == statusUpdate.filename)
+      if (index > -1) {
+        this.consumerStatus[index] = statusUpdate
+      } else {
+        this.consumerStatus.push(statusUpdate)
+      }
+
+      if (statusUpdate.status == "SUCCESS") {
+        this.documentConsumptionFinishedSubject.next(statusUpdate)
+      }
+      if (statusUpdate.status == "FAILED") {
+        this.documentConsumptionFailedSubject.next(statusUpdate)
+      }
+    }
+  }
+
+  disconnect() {
+    if (this.statusWebSocked) {
+      this.statusWebSocked.close()
+      this.statusWebSocked = null
+    }
+  }
+
+  dismiss(status: FileStatus) {
+    let index = this.consumerStatus.findIndex(s => s.filename == status.filename)
+
+    if (index > -1) {
+      this.consumerStatus.splice(index, 1)
+    }
+  }
+
+  onDocumentConsumptionFinished() {
+    return this.documentConsumptionFinishedSubject
+  }
+
+  onDocumentConsumptionFailed() {
+    return this.documentConsumptionFailedSubject
+  }
+
+}
diff --git a/src-ui/src/app/services/toast.service.ts b/src-ui/src/app/services/toast.service.ts
index a3ce060a9..d5781139e 100644
--- a/src-ui/src/app/services/toast.service.ts
+++ b/src-ui/src/app/services/toast.service.ts
@@ -1,30 +1,17 @@
 import { Injectable } from '@angular/core';
 import { Subject, zip } from 'rxjs';
 
-export class Toast {
-
-  static make(title: string, content: string, classname?: string, delay?: number): Toast {
-    let t = new Toast()
-    t.title = title
-    t.content = content
-    t.classname = classname
-    if (delay) {
-      t.delay = delay
-    }
-    return t
-  }
-
-  static makeError(content: string) {
-    return Toast.make("Error", content, null, 10000)
-  }
+export interface Toast {
 
   title: string
 
-  classname: string
-
   content: string
 
-  delay: number = 5000
+  delay?: number
+
+  action?: any
+
+  actionName?: string
 
 }
 
@@ -44,6 +31,14 @@ export class ToastService {
     this.toastsSubject.next(this.toasts)
   }
 
+  showInfo(message: string) {
+    this.showToast({title: "Information", content: message, delay: 5000})
+  }
+
+  showError(message: string) {
+    this.showToast({title: "Error", content: message, delay: 10000})
+  }
+
   closeToast(toast: Toast) {
     let index = this.toasts.findIndex(t => t == toast)
     if (index > -1) {
diff --git a/src-ui/src/environments/environment.ts b/src-ui/src/environments/environment.ts
index a0877d69f..c945a0364 100644
--- a/src-ui/src/environments/environment.ts
+++ b/src-ui/src/environments/environment.ts
@@ -4,7 +4,8 @@
 
 export const environment = {
   production: false,
-  apiBaseUrl: "http://localhost:8000/api/"
+  apiBaseUrl: "http://localhost:8000/api/",
+  wsBaseUrl: "ws://localhost:8000/ws/"
 };
 
 /*

From b0465e65c3ac85b58e3cb6c3daa2f755ee386c98 Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Sat, 7 Nov 2020 12:10:53 +0100
Subject: [PATCH 03/54] nicer status

---
 .../consumer-status-widget.component.html                   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
index 9f6aa3b87..d6559e184 100644
--- a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
@@ -1,10 +1,10 @@
 <h4 class="mt-3">Document consumer status</h4>
 
-<div class="p-2 mb-1 border bg-light" *ngFor="let s of getStatus()">
+<div class="mb-2 border-bottom" *ngFor="let s of getStatus()">
   <div class="mb-1"><strong>{{s.filename}}:</strong> {{s.message}}</div>
   <ngb-progressbar [type]="getType(s.status)" [value]="s.current_progress" [max]="s.max_progress" class="mb-2"></ngb-progressbar>
-  <div *ngIf="isFinished(s)">
-    <button *ngIf="s.document_id" class="btn btn-sm btn-outline-secondary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
+  <div *ngIf="isFinished(s)" class="mb-2">
+    <button *ngIf="s.document_id" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
     <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(s)">Dismiss</button>
   </div>
 </div>
\ No newline at end of file

From ae8a048ea6a570e9a15ba67d954297582b4ddeca Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Sat, 7 Nov 2020 12:47:17 +0100
Subject: [PATCH 04/54] fixed up the docker

---
 Pipfile                    |  1 +
 Pipfile.lock               |  4 ++--
 scripts/supervisord.conf   |  4 ++--
 src/paperless/asgi.py      | 30 ++++++++----------------------
 src/paperless/consumers.py | 16 ++++++++++++++++
 src/paperless/settings.py  |  2 +-
 src/paperless/urls.py      |  8 +++++++-
 7 files changed, 37 insertions(+), 28 deletions(-)
 create mode 100644 src/paperless/consumers.py

diff --git a/Pipfile b/Pipfile
index d526ae252..09de36334 100644
--- a/Pipfile
+++ b/Pipfile
@@ -28,6 +28,7 @@ django-extensions = ""
 watchdog = "*"
 channels = "~=3.0"
 channels-redis = "*"
+daphne = "~=3.0"
 
 [dev-packages]
 coveralls = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 642e38214..21b4becad 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "192d7419b844e6bb81fed793e7766b2ba15f2a016af1a33fc73cf09e12de5fb7"
+            "sha256": "66530e76de7948d8123529eff0b150926aa46a410da8b31e12b6d468e5996e7a"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -157,7 +157,7 @@
                 "sha256:60856f7efa0b1e1b969efa074e8698bd09de4713ecc06e6a4d19d04c66c4a3bd",
                 "sha256:b43e70d74ff832a634ff6c92badd208824e4530e08b340116517e5aad0aca774"
             ],
-            "markers": "python_version >= '3.6'",
+            "index": "pypi",
             "version": "==3.0.0"
         },
         "dateparser": {
diff --git a/scripts/supervisord.conf b/scripts/supervisord.conf
index d3ff288de..cb6fd1650 100644
--- a/scripts/supervisord.conf
+++ b/scripts/supervisord.conf
@@ -6,8 +6,8 @@ logfile_maxbytes=50MB        ; max main logfile bytes b4 rotation; default 50MB
 logfile_backups=10           ; # of main logfile backups; 0 means none, default 10
 loglevel=info                ; log level; default info; others: debug,warn,trace
 
-[program:gunicorn]
-command=gunicorn -c /usr/src/paperless/gunicorn.conf.py -b 0.0.0.0:8000 paperless.wsgi
+[program:daphne]
+command=daphne -b 0.0.0.0 -p 8000 paperless.asgi:application
 user=paperless
 
 stdout_logfile=/dev/stdout
diff --git a/src/paperless/asgi.py b/src/paperless/asgi.py
index 9c3d17b1b..45565c68a 100644
--- a/src/paperless/asgi.py
+++ b/src/paperless/asgi.py
@@ -1,31 +1,17 @@
-import json
 import os
 
-from asgiref.sync import async_to_sync
-from channels.auth import AuthMiddlewareStack
-from channels.generic.websocket import WebsocketConsumer
-from channels.routing import ProtocolTypeRouter, URLRouter
 from django.core.asgi import get_asgi_application
-from django.urls import re_path
+# Fetch Django ASGI application early to ensure AppRegistry is populated
+# before importing consumers and AuthMiddlewareStack that may import ORM
+# models.
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'paperless.settings')
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
+django_asgi_app = get_asgi_application()
 
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
 
-class StatusConsumer(WebsocketConsumer):
-    def connect(self):
-        self.accept()
-        async_to_sync(self.channel_layer.group_add)('status_updates', self.channel_name)
-
-    def disconnect(self, close_code):
-        async_to_sync(self.channel_layer.group_discard)('status_updates', self.channel_name)
-
-    def status_update(self, event):
-        self.send(json.dumps(event['data']))
-
-
-websocket_urlpatterns = [
-    re_path(r'ws/status/$', StatusConsumer.as_asgi()),
-]
+from paperless.urls import websocket_urlpatterns
 
 application = ProtocolTypeRouter({
     "http": get_asgi_application(),
diff --git a/src/paperless/consumers.py b/src/paperless/consumers.py
new file mode 100644
index 000000000..fbb7b72d0
--- /dev/null
+++ b/src/paperless/consumers.py
@@ -0,0 +1,16 @@
+import json
+
+from asgiref.sync import async_to_sync
+from channels.generic.websocket import WebsocketConsumer
+
+
+class StatusConsumer(WebsocketConsumer):
+    def connect(self):
+        self.accept()
+        async_to_sync(self.channel_layer.group_add)('status_updates', self.channel_name)
+
+    def disconnect(self, close_code):
+        async_to_sync(self.channel_layer.group_discard)('status_updates', self.channel_name)
+
+    def status_update(self, event):
+        self.send(json.dumps(event['data']))
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index c9db3e4b1..7ef132bff 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -307,7 +307,7 @@ CHANNEL_LAYERS = {
     "default": {
         "BACKEND": "channels_redis.core.RedisChannelLayer",
         "CONFIG": {
-            "hosts": [("127.0.0.1", 6379)],
+            "hosts": [("broker", 6379)],
         },
     },
 }
diff --git a/src/paperless/urls.py b/src/paperless/urls.py
index 43ba5eb49..4416bc8d5 100755
--- a/src/paperless/urls.py
+++ b/src/paperless/urls.py
@@ -1,11 +1,12 @@
 from django.conf.urls import include, url
 from django.contrib import admin
-from django.urls import path
+from django.urls import path, re_path
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import RedirectView
 from rest_framework.authtoken import views
 from rest_framework.routers import DefaultRouter
 
+from paperless.consumers import StatusConsumer
 from paperless.views import FaviconView
 from documents.views import (
     CorrespondentViewSet,
@@ -65,6 +66,11 @@ urlpatterns = [
 
 ]
 
+
+websocket_urlpatterns = [
+    re_path(r'ws/status/$', StatusConsumer.as_asgi()),
+]
+
 # Text in each page's <h1> (and above login form).
 admin.site.site_header = 'Paperless'
 # Text at the end of each page's <title>.

From 37bd4a7d0edab657488143bcc8427b5d13f1699a Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Sat, 7 Nov 2020 12:56:26 +0100
Subject: [PATCH 05/54] added broker to compose file

---
 docker-compose.yml.example | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docker-compose.yml.example b/docker-compose.yml.example
index 1130e26a3..3d8ed4719 100644
--- a/docker-compose.yml.example
+++ b/docker-compose.yml.example
@@ -1,5 +1,10 @@
 version: "3.4"
 services:
+  broker:
+    image: redis:latest
+    ports:
+      - 6379:6379
+
   db:
     image: postgres:13
     #restart: always

From 9e81c82452db80ced437228f62e6713f925adb9f Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Tue, 10 Nov 2020 01:26:27 +0100
Subject: [PATCH 06/54] Pipfile update

---
 Pipfile.lock | 75 ++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 64 insertions(+), 11 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 21b4becad..932dc380e 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "66530e76de7948d8123529eff0b150926aa46a410da8b31e12b6d468e5996e7a"
+            "sha256": "65b67f06b1ad7d6541a1f84552f1c24a0353ed03aa4c382f8638c9d112b41eda"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -21,13 +21,21 @@
             ],
             "version": "==1.3.1"
         },
+        "arrow": {
+            "hashes": [
+                "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5",
+                "sha256:ff08d10cda1d36c68657d6ad20d74fbea493d980f8b2d45344e00d6ed2bf6ed4"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.17.0"
+        },
         "asgiref": {
             "hashes": [
-                "sha256:a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e",
-                "sha256:cd88907ecaec59d78e4ac00ea665b03e571cb37e3a0e37b3702af1a9e86c365a"
+                "sha256:5ee950735509d04eb673bd7f7120f8fa1c9e2df495394992c73234d526907e17",
+                "sha256:7162a3cb30ab0609f1a4c95938fd73e8604f63bdba516a7f7d64b83ff09478f0"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.3.0"
+            "version": "==3.3.1"
         },
         "async-timeout": {
             "hashes": [
@@ -60,6 +68,13 @@
             ],
             "version": "==20.2.0"
         },
+        "blessed": {
+            "hashes": [
+                "sha256:7d4914079a6e8e14fbe080dcaf14dee596a088057cdc598561080e3266123b48",
+                "sha256:81125aa5b84cb9dfc09ff451886f64b4b923b75c5eaf51fde9d1c48a135eb797"
+            ],
+            "version": "==1.17.11"
+        },
         "cffi": {
             "hashes": [
                 "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d",
@@ -103,11 +118,11 @@
         },
         "channels": {
             "hashes": [
-                "sha256:5cdd9c6b9ee663cdf1bbb00de7cdab885a3c418f9d32a29f04b09498828020f6",
-                "sha256:b02e150b48704ec3607d4168402ac5c26138dd183fcdb7f2aeb965e6e19fd558"
+                "sha256:74db79c9eca616be69d38013b22083ab5d3f9ccda1ab5e69096b1bb7da2d9b18",
+                "sha256:f50a6e79757a64c1e45e95e144a2ac5f1e99ee44a0718ab182c501f5e5abd268"
             ],
             "index": "pypi",
-            "version": "==3.0.1"
+            "version": "==3.0.2"
         },
         "channels-redis": {
             "hashes": [
@@ -190,7 +205,6 @@
                 "sha256:dc663652ac9460fd06580a973576820430c6d428720e874ae46b041fa63e0efa"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.5'",
             "version": "==3.0.9"
         },
         "django-filter": {
@@ -201,6 +215,22 @@
             "index": "pypi",
             "version": "==2.4.0"
         },
+        "django-picklefield": {
+            "hashes": [
+                "sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287",
+                "sha256:3c702a54fde2d322fe5b2f39b8f78d9f655b8f77944ab26f703be6c0ed335a35"
+            ],
+            "markers": "python_version >= '3'",
+            "version": "==3.0.1"
+        },
+        "django-q": {
+            "hashes": [
+                "sha256:523d54dcf1b66152c1b658f914f00ed3b518a3432a9decd4898738ca8dbbe10f",
+                "sha256:7e5c5c021a15cff6807044a3aa48f5757789ccfef839d71c575f5512931a3e33"
+            ],
+            "index": "pypi",
+            "version": "==1.3.4"
+        },
         "djangorestframework": {
             "hashes": [
                 "sha256:0209bafcb7b5010fdfec784034f059d512256424de2a0f084cb82b096d6dd6a7"
@@ -390,6 +420,14 @@
             ],
             "version": "==0.1.2"
         },
+        "pathvalidate": {
+            "hashes": [
+                "sha256:1697c8ea71ff4c48e7aa0eda72fe4581404be8f41e51a17363ef682dd6824d35",
+                "sha256:32d30dbacb711c16bb188b12ce7e9a46b41785f50a12f64500f747480a4b6ee3"
+            ],
+            "index": "pypi",
+            "version": "==2.3.0"
+        },
         "pdftotext": {
             "hashes": [
                 "sha256:98aeb8b07a4127e1a30223bd933ef080bbd29aa88f801717ca6c5618380b8aa6"
@@ -576,6 +614,14 @@
             ],
             "version": "==2020.4"
         },
+        "redis": {
+            "hashes": [
+                "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2",
+                "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24"
+            ],
+            "index": "pypi",
+            "version": "==3.5.3"
+        },
         "regex": {
             "hashes": [
                 "sha256:03855ee22980c3e4863dc84c42d6d2901133362db5daf4c36b710dd895d78f0a",
@@ -762,6 +808,13 @@
             "index": "pypi",
             "version": "==0.10.3"
         },
+        "wcwidth": {
+            "hashes": [
+                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
+                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
+            ],
+            "version": "==0.2.5"
+        },
         "whitenoise": {
             "hashes": [
                 "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
@@ -879,10 +932,10 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3",
-                "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"
+                "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd",
+                "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4"
             ],
-            "version": "==2020.6.20"
+            "version": "==2020.11.8"
         },
         "chardet": {
             "hashes": [

From cb2340539ded2e6044e5386b79198d4539d0383a Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Thu, 19 Nov 2020 22:14:11 +0100
Subject: [PATCH 07/54] updated pipenv

---
 Pipfile.lock | 86 +++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 69 insertions(+), 17 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 932dc380e..ebe2c4095 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "65b67f06b1ad7d6541a1f84552f1c24a0353ed03aa4c382f8638c9d112b41eda"
+            "sha256": "ef3638ed4905e0809823dc1cfefd8e5ee415cd9d33ec3f23e483fb60b87d6fe6"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -10,6 +10,11 @@
                 "name": "pypi",
                 "url": "https://pypi.python.org/simple",
                 "verify_ssl": true
+            },
+            {
+                "name": "piwheels",
+                "url": "https://www.piwheels.org/simple",
+                "verify_ssl": true
             }
         ]
     },
@@ -55,6 +60,7 @@
         },
         "autobahn": {
             "hashes": [
+                "sha256:1eafbbe363a7924fd21bb0b94ece9f3ac2a9aa9c2046e8a85e044f94e8ba2028",
                 "sha256:24ce276d313e84d68241c3aef30d484f352b90a40168981b3640312c821df77b",
                 "sha256:86bbce30cdd407137c57670993a8f9bfdfe3f8e994b889181d85e844d5aa8dfb"
             ],
@@ -64,7 +70,8 @@
         "automat": {
             "hashes": [
                 "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33",
-                "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"
+                "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111",
+                "sha256:d6d976cf8da698fc85fa7def46e2544493f78cb7ee72d2f4acd1a5c759a3060e"
             ],
             "version": "==20.2.0"
         },
@@ -82,6 +89,7 @@
                 "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4",
                 "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f",
                 "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3",
+                "sha256:178a2db1589cb9b0b5b28a74ee0c9d4438bd96f8c6c0ac85662ff3c98f7f8d20",
                 "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579",
                 "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537",
                 "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e",
@@ -108,6 +116,7 @@
                 "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d",
                 "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b",
                 "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e",
+                "sha256:df90c0c9e383e8c3bdced39f113ecc36fa9c623dd04dd1b5199e9edc53389a95",
                 "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d",
                 "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730",
                 "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394",
@@ -143,12 +152,14 @@
             "hashes": [
                 "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538",
                 "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f",
+                "sha256:257dab4f368fae15f378ea9a4d2799bf3696668062de0e9fa0ebb7a738a6917d",
                 "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77",
                 "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b",
                 "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33",
                 "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e",
                 "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb",
                 "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e",
+                "sha256:59f7d4cfea9ef12eb9b14b83d79b432162a0a24a91ddc15c2c9bf76a68d96f2b",
                 "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7",
                 "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297",
                 "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d",
@@ -291,6 +302,7 @@
                 "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2",
                 "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132",
                 "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6",
+                "sha256:9f4e67f87e072de981570eaf7cb41444bbac7e92b05c8651dbab6eb1fb8d5a14",
                 "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c",
                 "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363",
                 "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3",
@@ -298,6 +310,7 @@
                 "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919",
                 "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349",
                 "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae",
+                "sha256:b39989b49e8aca9d224324d2650029eda410a4faf43f6afb0eb4f9acb7be6097",
                 "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da",
                 "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f",
                 "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed",
@@ -315,6 +328,7 @@
         },
         "hyperlink": {
             "hashes": [
+                "sha256:402c1b5fa066ea368f3118fc5a6f8505440b4d1a4ef12a844ca39332a4a29944",
                 "sha256:47fcc7cd339c6cb2444463ec3277bdcfe142c8b1daf2160bdd52248deec815af",
                 "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"
             ],
@@ -322,12 +336,21 @@
         },
         "idna": {
             "hashes": [
+                "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226",
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
+        "imap-tools": {
+            "hashes": [
+                "sha256:070929b8ec429c0aad94588a37a2962eed656a119ab61dcf91489f20fe983f5d",
+                "sha256:6232cd43748741496446871e889eb137351fc7a7e7f4c7888cd8c0fa28e20cda"
+            ],
+            "index": "pypi",
+            "version": "==0.31.0"
+        },
         "incremental": {
             "hashes": [
                 "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
@@ -354,12 +377,14 @@
         "msgpack": {
             "hashes": [
                 "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408",
+                "sha256:0e7b5a69ec5645b0a85baaa354c29acd89eb879aaa89e7f4b37ed4d9c5abafe0",
                 "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8",
                 "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84",
                 "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d",
                 "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a",
                 "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322",
                 "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2",
+                "sha256:71604047feea609ad65f5b837ec89a4de084d55a80f8af7331745a075c3dbd23",
                 "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e",
                 "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97",
                 "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0",
@@ -370,7 +395,8 @@
                 "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e",
                 "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272",
                 "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1",
-                "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140"
+                "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140",
+                "sha256:f7c80ff32171193f18a127ea357118b920020cc0acb0730016bbda02b892a2d2"
             ],
             "version": "==1.0.0"
         },
@@ -389,6 +415,7 @@
                 "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb",
                 "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc",
                 "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac",
+                "sha256:5ddd1dfa2be066595c1993165b4cae84b9866b12339d0c903db7f21a094324a3",
                 "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83",
                 "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36",
                 "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387",
@@ -416,7 +443,8 @@
         },
         "pathtools": {
             "hashes": [
-                "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0"
+                "sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0",
+                "sha256:d77d982475e87f32b82157a43b09f0a5ef3e66c1d8f3c7eb8d2580e783cd8202"
             ],
             "version": "==0.1.2"
         },
@@ -711,6 +739,7 @@
                 "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62",
                 "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d",
                 "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437",
+                "sha256:b5e9d3e4474644915809d6aa1416ff20430a3ed9ae723a5d295da5ddb24985e2",
                 "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2",
                 "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54",
                 "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474",
@@ -759,9 +788,11 @@
                 "tls"
             ],
             "hashes": [
+                "sha256:0150dae5adc962d15e00054cc6926f1e64763fb8dd26e1632593ac06e592104b",
                 "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f",
                 "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042",
                 "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c",
+                "sha256:15e52271f08f62e2230ff093e0278aa01c9dac057c4557cadadd2429eed86a3e",
                 "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292",
                 "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22",
                 "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec",
@@ -836,6 +867,7 @@
             "hashes": [
                 "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
                 "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d",
+                "sha256:09fc3922f235703c0b76f8234867685eee68a24a49fffa2220975f6142db45f1",
                 "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123",
                 "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232",
                 "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549",
@@ -870,6 +902,7 @@
                 "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b",
                 "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50",
                 "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523",
+                "sha256:974f5957e66a7524ea81df7b2686a456bfaf0408dbb7353ddfbedb594eadfef6",
                 "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a",
                 "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095",
                 "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a",
@@ -924,11 +957,11 @@
         },
         "babel": {
             "hashes": [
-                "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38",
-                "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4"
+                "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5",
+                "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==2.8.0"
+            "version": "==2.9.0"
         },
         "certifi": {
             "hashes": [
@@ -954,6 +987,7 @@
                 "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f",
                 "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7",
                 "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c",
+                "sha256:3188a7dfd96f734a7498f37cde6598b1e9c084f1ca68bc1aa04e88db31168ab6",
                 "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5",
                 "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7",
                 "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729",
@@ -979,7 +1013,8 @@
                 "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237",
                 "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7",
                 "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636",
-                "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8"
+                "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8",
+                "sha256:ef221855191457fffeb909d5787d1807800ab4d0111f089e6c93ee68f577634d"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
             "version": "==5.3"
@@ -1001,6 +1036,7 @@
         },
         "docopt": {
             "hashes": [
+                "sha256:15fde8252aa9f2804171014d50d069ffbf42c7a50b7d74bcbb82bfd5700fcfc2",
                 "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"
             ],
             "version": "==0.6.2"
@@ -1031,11 +1067,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:6afc461ab3f779c9c16e299fc731d775e39ea7e8e063b3053ee359ae198a15ca",
-                "sha256:ce1c38823eb0f927567cde5bf2e7c8ca565c7a70316139342050ce2ca74b4026"
+                "sha256:3f5d379e4b5ce92a8afe3c2ce59d7c43886370dd3bf9495a936b91888debfc81",
+                "sha256:8c0e8a06acef4b9312902e2ce18becabe62badd3a6632180bd0680c6ee111473"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==4.14.2"
+            "version": "==4.17.0"
         },
         "filelock": {
             "hashes": [
@@ -1046,6 +1082,7 @@
         },
         "idna": {
             "hashes": [
+                "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226",
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
@@ -1063,12 +1100,14 @@
         "iniconfig": {
             "hashes": [
                 "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:8647b85c03813b8680f4ae9c9db2fd7293f8591ea536a10d73d90f6eb4b10aac",
                 "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
             ],
             "version": "==1.1.1"
         },
         "jinja2": {
             "hashes": [
+                "sha256:3f172970d5670703bd3812e8ca6459a9a7e069fa8e51b40195f83c81db191ec4",
                 "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0",
                 "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"
             ],
@@ -1082,8 +1121,10 @@
                 "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",
                 "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",
                 "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",
+                "sha256:19536834abffb3fa155017053c607cb835b2ecc6a3a2554a88043d991dffb736",
                 "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",
                 "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",
+                "sha256:3d61f15e39611aacd91b7e71d903787da86d9e80896e683c0103fced9add7834",
                 "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",
                 "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",
                 "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
@@ -1093,6 +1134,7 @@
                 "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",
                 "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",
                 "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",
+                "sha256:7952deddf24b85c88dab48f6ec366ac6e39d2761b5280f2f9594911e03fcd064",
                 "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",
                 "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",
                 "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",
@@ -1195,6 +1237,7 @@
         },
         "pytest-forked": {
             "hashes": [
+                "sha256:2d1bfc93ab65a28324eb0a63503bfb500c2da6916efede7a24b43a04970fe63c",
                 "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca",
                 "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"
             ],
@@ -1233,11 +1276,11 @@
         },
         "requests": {
             "hashes": [
-                "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b",
-                "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"
+                "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
+                "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==2.24.0"
+            "version": "==2.25.0"
         },
         "six": {
             "hashes": [
@@ -1262,6 +1305,14 @@
             "index": "pypi",
             "version": "==3.3.0"
         },
+        "sphinx-rtd-theme": {
+            "hashes": [
+                "sha256:22c795ba2832a169ca301cd0a083f7a434e09c538c70beb42782c073651b707d",
+                "sha256:373413d0f82425aaa28fb288009bf0d0964711d347763af2f1b65cafcb028c82"
+            ],
+            "index": "pypi",
+            "version": "==0.5.0"
+        },
         "sphinxcontrib-applehelp": {
             "hashes": [
                 "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a",
@@ -1312,6 +1363,7 @@
         },
         "termcolor": {
             "hashes": [
+                "sha256:19b1225d03bfb56571484caaa8521d8ec6e2473ae1640c9f48a48dda49417706",
                 "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"
             ],
             "version": "==1.1.0"
@@ -1341,11 +1393,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2",
-                "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e"
+                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
+                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.25.11"
+            "version": "==1.26.2"
         },
         "virtualenv": {
             "hashes": [

From 391020a2b064bff95fa9116599a36fb11129a214 Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Fri, 20 Nov 2020 10:58:17 +0100
Subject: [PATCH 08/54] small changes

---
 src/documents/consumer.py          |  6 +++---
 src/paperless/settings.py          | 18 +++++++++---------
 src/paperless_tesseract/parsers.py |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 973fff925..8edbb00a3 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -223,9 +223,9 @@ class Consumer(LoggingMixin):
                 self.log("debug", "Deleting file {}".format(self.path))
                 os.unlink(self.path)
         except Exception as e:
-            raise ConsumerError(e)
-            self._send_progress(file, 100, 100, 'FAILED',
+            self._send_progress(self.filename, 100, 100, 'FAILED',
                                 "Failed: {}".format(e))
+            raise ConsumerError(e)
         finally:
             document_parser.cleanup()
 
@@ -234,7 +234,7 @@ class Consumer(LoggingMixin):
             "Document {} consumption finished".format(document)
         )
 
-        self._send_progress(file, 100, 100, 'SUCCESS',
+        self._send_progress(self.filename, 100, 100, 'SUCCESS',
                             'Finished.', document.id)
 
         return document
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index cb115739b..37b046b2a 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -143,6 +143,15 @@ TEMPLATES = [
     },
 ]
 
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels_redis.core.RedisChannelLayer",
+        "CONFIG": {
+            "hosts": ["redis://localhost:6379"],
+        },
+    },
+}
+
 ###############################################################################
 # Security                                                                    #
 ###############################################################################
@@ -376,12 +385,3 @@ for t in json.loads(os.getenv("PAPERLESS_FILENAME_PARSE_TRANSFORMS", "[]")):
 # TODO: this should not have a prefix.
 # Specify the filename format for out files
 PAPERLESS_FILENAME_FORMAT = os.getenv("PAPERLESS_FILENAME_FORMAT")
-
-CHANNEL_LAYERS = {
-    "default": {
-        "BACKEND": "channels_redis.core.RedisChannelLayer",
-        "CONFIG": {
-            "hosts": [("broker", 6379)],
-        },
-    },
-}
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index c3715c83a..11a6c8b37 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -90,7 +90,7 @@ class RasterisedDocumentParser(DocumentParser):
             self._text = get_text_from_pdf(self.document_path)
             return self._text
 
-        self.progress_callback(0,1,"Making greyscale images.")
+        self.progress_callback(0, 1, "Making greyscale images.")
         images = self._get_greyscale()
 
         if not images:

From 6cf0b851b7011d86afeced9deb777fdbf374cfc4 Mon Sep 17 00:00:00 2001
From: Jonas Winkler <jonas.winkler@jpwinkler.de>
Date: Sun, 22 Nov 2020 23:12:24 +0100
Subject: [PATCH 09/54] post-merge changes

---
 Pipfile.lock                                  | 430 +++++++++++++++++-
 src-ui/src/app/app.module.ts                  |   2 +
 .../consumer-status-widget.component.html     |  18 +-
 ... => consumer-status-widget.component.scss} |   0
 .../consumer-status-widget.component.ts       |   2 +-
 .../file-upload-widget.component.css          |   0
 .../file-upload-widget.component.html         |  11 -
 .../file-upload-widget.component.spec.ts      |  25 -
 .../file-upload-widget.component.ts           |  44 --
 .../upload-file-widget.component.ts           |   4 +-
 10 files changed, 437 insertions(+), 99 deletions(-)
 rename src-ui/src/app/components/dashboard/widgets/consumer-status-widget/{consumer-status-widget.component.css => consumer-status-widget.component.scss} (100%)
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.css
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts

diff --git a/Pipfile.lock b/Pipfile.lock
index 6ecca3c34..2dd198e14 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "ae2643b9cf0cf5741ae149fb6bc0c480de41329ce48e773eb4b5d760bc5e2244"
+            "sha256": "83cb61d0f0de0ad70aa02e8424deb743331c3578a67ee17ed06394506fdb2c14"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -19,6 +19,13 @@
         ]
     },
     "default": {
+        "aioredis": {
+            "hashes": [
+                "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a",
+                "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"
+            ],
+            "version": "==1.3.1"
+        },
         "arrow": {
             "hashes": [
                 "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5",
@@ -35,6 +42,39 @@
             "markers": "python_version >= '3.5'",
             "version": "==3.3.1"
         },
+        "async-timeout": {
+            "hashes": [
+                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
+                "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
+            ],
+            "markers": "python_full_version >= '3.5.3'",
+            "version": "==3.0.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.3.0"
+        },
+        "autobahn": {
+            "hashes": [
+                "sha256:1eafbbe363a7924fd21bb0b94ece9f3ac2a9aa9c2046e8a85e044f94e8ba2028",
+                "sha256:24ce276d313e84d68241c3aef30d484f352b90a40168981b3640312c821df77b",
+                "sha256:86bbce30cdd407137c57670993a8f9bfdfe3f8e994b889181d85e844d5aa8dfb"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==20.7.1"
+        },
+        "automat": {
+            "hashes": [
+                "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33",
+                "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111",
+                "sha256:d6d976cf8da698fc85fa7def46e2544493f78cb7ee72d2f4acd1a5c759a3060e"
+            ],
+            "version": "==20.2.0"
+        },
         "blessed": {
             "hashes": [
                 "sha256:7d4914079a6e8e14fbe080dcaf14dee596a088057cdc598561080e3266123b48",
@@ -42,6 +82,110 @@
             ],
             "version": "==1.17.11"
         },
+        "cffi": {
+            "hashes": [
+                "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d",
+                "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b",
+                "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4",
+                "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f",
+                "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3",
+                "sha256:178a2db1589cb9b0b5b28a74ee0c9d4438bd96f8c6c0ac85662ff3c98f7f8d20",
+                "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579",
+                "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537",
+                "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e",
+                "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05",
+                "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171",
+                "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca",
+                "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522",
+                "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c",
+                "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc",
+                "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d",
+                "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808",
+                "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828",
+                "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869",
+                "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d",
+                "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9",
+                "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0",
+                "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc",
+                "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15",
+                "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c",
+                "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a",
+                "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3",
+                "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1",
+                "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768",
+                "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d",
+                "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b",
+                "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e",
+                "sha256:df90c0c9e383e8c3bdced39f113ecc36fa9c623dd04dd1b5199e9edc53389a95",
+                "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d",
+                "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730",
+                "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394",
+                "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1",
+                "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"
+            ],
+            "version": "==1.14.3"
+        },
+        "channels": {
+            "hashes": [
+                "sha256:74db79c9eca616be69d38013b22083ab5d3f9ccda1ab5e69096b1bb7da2d9b18",
+                "sha256:f50a6e79757a64c1e45e95e144a2ac5f1e99ee44a0718ab182c501f5e5abd268"
+            ],
+            "index": "pypi",
+            "version": "==3.0.2"
+        },
+        "channels-redis": {
+            "hashes": [
+                "sha256:18d63f6462a58011740dc8eeb57ea4b31ec220eb551cb71b27de9c6779a549de",
+                "sha256:2fb31a63b05373f6402da2e6a91a22b9e66eb8b56626c6bfc93e156c734c5ae6"
+            ],
+            "index": "pypi",
+            "version": "==3.2.0"
+        },
+        "constantly": {
+            "hashes": [
+                "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35",
+                "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"
+            ],
+            "version": "==15.1.0"
+        },
+        "cryptography": {
+            "hashes": [
+                "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538",
+                "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f",
+                "sha256:257dab4f368fae15f378ea9a4d2799bf3696668062de0e9fa0ebb7a738a6917d",
+                "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77",
+                "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b",
+                "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33",
+                "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e",
+                "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb",
+                "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e",
+                "sha256:59f7d4cfea9ef12eb9b14b83d79b432162a0a24a91ddc15c2c9bf76a68d96f2b",
+                "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7",
+                "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297",
+                "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d",
+                "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7",
+                "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b",
+                "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7",
+                "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4",
+                "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8",
+                "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b",
+                "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851",
+                "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13",
+                "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b",
+                "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3",
+                "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==3.2.1"
+        },
+        "daphne": {
+            "hashes": [
+                "sha256:0052c9887600c57054a5867d4b0240159fa009faa3bcf6a1627271d9cdcb005a",
+                "sha256:c22b692707f514de9013651ecb687f2abe4f35cf6fe292ece634e9f1737bc7e3"
+            ],
+            "index": "pypi",
+            "version": "==3.0.1"
+        },
         "dateparser": {
             "hashes": [
                 "sha256:7552c994f893b5cb8fcf103b4cd2ff7f57aab9bfd2619fdf0cf571c0740fd90b",
@@ -121,6 +265,77 @@
             "index": "pypi",
             "version": "==20.0.4"
         },
+        "hiredis": {
+            "hashes": [
+                "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
+                "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0",
+                "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0",
+                "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01",
+                "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a",
+                "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b",
+                "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6",
+                "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73",
+                "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee",
+                "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55",
+                "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12",
+                "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b",
+                "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323",
+                "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c",
+                "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655",
+                "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5",
+                "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75",
+                "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb",
+                "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23",
+                "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1",
+                "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f",
+                "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872",
+                "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058",
+                "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454",
+                "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882",
+                "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2",
+                "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132",
+                "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6",
+                "sha256:9f4e67f87e072de981570eaf7cb41444bbac7e92b05c8651dbab6eb1fb8d5a14",
+                "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c",
+                "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363",
+                "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3",
+                "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4",
+                "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919",
+                "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349",
+                "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae",
+                "sha256:b39989b49e8aca9d224324d2650029eda410a4faf43f6afb0eb4f9acb7be6097",
+                "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da",
+                "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f",
+                "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed",
+                "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628",
+                "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64",
+                "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86",
+                "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf",
+                "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c",
+                "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded",
+                "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390",
+                "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.1.0"
+        },
+        "hyperlink": {
+            "hashes": [
+                "sha256:402c1b5fa066ea368f3118fc5a6f8505440b4d1a4ef12a844ca39332a4a29944",
+                "sha256:47fcc7cd339c6cb2444463ec3277bdcfe142c8b1daf2160bdd52248deec815af",
+                "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"
+            ],
+            "version": "==20.0.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226",
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.10"
+        },
         "imap-tools": {
             "hashes": [
                 "sha256:96e9a4ff6483462635737730a1df28e739faa71967b12a84f4363fb386542246",
@@ -129,6 +344,13 @@
             "index": "pypi",
             "version": "==0.32.0"
         },
+        "incremental": {
+            "hashes": [
+                "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
+                "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"
+            ],
+            "version": "==17.5.0"
+        },
         "joblib": {
             "hashes": [
                 "sha256:698c311779f347cf6b7e6b8a39bb682277b8ee4aba8cf9507bc0cf4cd4737b72",
@@ -146,6 +368,32 @@
             "index": "pypi",
             "version": "==1.0.8"
         },
+        "msgpack": {
+            "hashes": [
+                "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408",
+                "sha256:0e7b5a69ec5645b0a85baaa354c29acd89eb879aaa89e7f4b37ed4d9c5abafe0",
+                "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8",
+                "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84",
+                "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d",
+                "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a",
+                "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322",
+                "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2",
+                "sha256:71604047feea609ad65f5b837ec89a4de084d55a80f8af7331745a075c3dbd23",
+                "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e",
+                "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97",
+                "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0",
+                "sha256:aa5c057eab4f40ec47ea6f5a9825846be2ff6bf34102c560bad5cad5a677c5be",
+                "sha256:b3758dfd3423e358bbb18a7cccd1c74228dffa7a697e5be6cb9535de625c0dbf",
+                "sha256:c901e8058dd6653307906c5f157f26ed09eb94a850dddd989621098d347926ab",
+                "sha256:cec8bf10981ed70998d98431cd814db0ecf3384e6b113366e7f36af71a0fca08",
+                "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e",
+                "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272",
+                "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1",
+                "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140",
+                "sha256:f7c80ff32171193f18a127ea357118b920020cc0acb0730016bbda02b892a2d2"
+            ],
+            "version": "==1.0.0"
+        },
         "numpy": {
             "hashes": [
                 "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db",
@@ -287,6 +535,58 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "pyasn1": {
+            "hashes": [
+                "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359",
+                "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576",
+                "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf",
+                "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7",
+                "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d",
+                "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00",
+                "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8",
+                "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86",
+                "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12",
+                "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776",
+                "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba",
+                "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2",
+                "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"
+            ],
+            "version": "==0.4.8"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8",
+                "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199",
+                "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811",
+                "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed",
+                "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4",
+                "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e",
+                "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74",
+                "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb",
+                "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45",
+                "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd",
+                "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0",
+                "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d",
+                "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"
+            ],
+            "version": "==0.2.8"
+        },
+        "pycparser": {
+            "hashes": [
+                "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
+                "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.20"
+        },
+        "pyhamcrest": {
+            "hashes": [
+                "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316",
+                "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.2"
+        },
         "pyocr": {
             "hashes": [
                 "sha256:fa15adc7e1cf0d345a2990495fe125a947c6e09a60ddba0256a1c14b2e603179",
@@ -295,6 +595,13 @@
             "index": "pypi",
             "version": "==0.7.2"
         },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504",
+                "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507"
+            ],
+            "version": "==19.1.0"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
@@ -456,6 +763,13 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.5.4"
         },
+        "service-identity": {
+            "hashes": [
+                "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
+                "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"
+            ],
+            "version": "==18.1.0"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@@ -480,6 +794,48 @@
             "markers": "python_version >= '3.5'",
             "version": "==2.1.0"
         },
+        "twisted": {
+            "extras": [
+                "tls"
+            ],
+            "hashes": [
+                "sha256:0150dae5adc962d15e00054cc6926f1e64763fb8dd26e1632593ac06e592104b",
+                "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f",
+                "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042",
+                "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c",
+                "sha256:15e52271f08f62e2230ff093e0278aa01c9dac057c4557cadadd2429eed86a3e",
+                "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292",
+                "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22",
+                "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec",
+                "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478",
+                "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2",
+                "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29",
+                "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114",
+                "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797",
+                "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa",
+                "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15",
+                "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd",
+                "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274",
+                "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad",
+                "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7",
+                "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a",
+                "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10",
+                "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780",
+                "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504",
+                "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467",
+                "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==20.3.0"
+        },
+        "txaio": {
+            "hashes": [
+                "sha256:17938f2bca4a9cabce61346758e482ca4e600160cbc28e861493eac74a19539d",
+                "sha256:38a469daf93c37e5527cb062653d6393ae11663147c42fab7ddc3f6d00d434ae"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==20.4.1"
+        },
         "tzlocal": {
             "hashes": [
                 "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44",
@@ -518,6 +874,66 @@
             ],
             "index": "pypi",
             "version": "==2.7.4"
+        },
+        "zope.interface": {
+            "hashes": [
+                "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
+                "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d",
+                "sha256:09fc3922f235703c0b76f8234867685eee68a24a49fffa2220975f6142db45f1",
+                "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123",
+                "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232",
+                "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549",
+                "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102",
+                "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5",
+                "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45",
+                "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00",
+                "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc",
+                "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7",
+                "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104",
+                "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034",
+                "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3",
+                "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3",
+                "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4",
+                "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86",
+                "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96",
+                "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546",
+                "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb",
+                "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3",
+                "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b",
+                "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b",
+                "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec",
+                "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae",
+                "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e",
+                "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386",
+                "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2",
+                "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a",
+                "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d",
+                "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a",
+                "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24",
+                "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d",
+                "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b",
+                "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50",
+                "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523",
+                "sha256:974f5957e66a7524ea81df7b2686a456bfaf0408dbb7353ddfbedb594eadfef6",
+                "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a",
+                "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095",
+                "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a",
+                "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520",
+                "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65",
+                "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11",
+                "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c",
+                "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7",
+                "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332",
+                "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e",
+                "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c",
+                "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7",
+                "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20",
+                "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc",
+                "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd",
+                "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==5.2.0"
         }
     },
     "develop": {
@@ -663,11 +1079,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:3f5d379e4b5ce92a8afe3c2ce59d7c43886370dd3bf9495a936b91888debfc81",
-                "sha256:8c0e8a06acef4b9312902e2ce18becabe62badd3a6632180bd0680c6ee111473"
+                "sha256:5398268e1d751ffdb3ed36b8a790ed98659200599b368eec38a02eed15bce997",
+                "sha256:d4183b8f57316de3be27cd6c3b40e9f9343d27c95c96179f027316c58c2c239e"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==4.17.0"
+            "version": "==4.17.1"
         },
         "filelock": {
             "hashes": [
@@ -999,11 +1415,11 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:b0011228208944ce71052987437d3843e05690b2f23d1c7da4263fde104c97a2",
-                "sha256:b8d6110f493af256a40d65e29846c69340a947669eec8ce784fcf3dd3af28380"
+                "sha256:6af42359fbb33a6c7eab4d3246524b96fd9d8e07e7141b7a65998f96e28b2c57",
+                "sha256:fd4147c5ba3f694e2e4fc3c767407dc2226899623bb9b49c2f15637c2ee335b3"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.1.0"
+            "version": "==20.2.0"
         }
     }
 }
diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts
index 3ccb1c5f1..2a70af813 100644
--- a/src-ui/src/app/app.module.ts
+++ b/src-ui/src/app/app.module.ts
@@ -45,6 +45,7 @@ import { SavedViewWidgetComponent } from './components/dashboard/widgets/saved-v
 import { StatisticsWidgetComponent } from './components/dashboard/widgets/statistics-widget/statistics-widget.component';
 import { UploadFileWidgetComponent } from './components/dashboard/widgets/upload-file-widget/upload-file-widget.component';
 import { WidgetFrameComponent } from './components/dashboard/widgets/widget-frame/widget-frame.component';
+import { ConsumerStatusWidgetComponent } from './components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component';
 
 @NgModule({
   declarations: [
@@ -82,6 +83,7 @@ import { WidgetFrameComponent } from './components/dashboard/widgets/widget-fram
     SavedViewWidgetComponent,
     StatisticsWidgetComponent,
     UploadFileWidgetComponent,
+    ConsumerStatusWidgetComponent,
     WidgetFrameComponent
   ],
   imports: [
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
index d6559e184..ff2117729 100644
--- a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
@@ -1,10 +1,10 @@
-<h4 class="mt-3">Document consumer status</h4>
-
-<div class="mb-2 border-bottom" *ngFor="let s of getStatus()">
-  <div class="mb-1"><strong>{{s.filename}}:</strong> {{s.message}}</div>
-  <ngb-progressbar [type]="getType(s.status)" [value]="s.current_progress" [max]="s.max_progress" class="mb-2"></ngb-progressbar>
-  <div *ngIf="isFinished(s)" class="mb-2">
-    <button *ngIf="s.document_id" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
-    <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(s)">Dismiss</button>
+<app-widget-frame title="Document consumer status">
+  <div class="mb-2 border-bottom" *ngFor="let s of getStatus()">
+    <div class="mb-1"><strong>{{s.filename}}:</strong> {{s.message}}</div>
+    <ngb-progressbar [type]="getType(s.status)" [value]="s.current_progress" [max]="s.max_progress" class="mb-2"></ngb-progressbar>
+    <div *ngIf="isFinished(s)" class="mb-2">
+      <button *ngIf="s.document_id" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
+      <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(s)">Dismiss</button>
+    </div>
   </div>
-</div>
\ No newline at end of file
+</app-widget-frame>
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.css b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.scss
similarity index 100%
rename from src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.css
rename to src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.scss
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
index 0c4e35682..8e44af6d5 100644
--- a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
@@ -4,7 +4,7 @@ import { ConsumerStatusService, FileStatus } from 'src/app/services/consumer-sta
 @Component({
   selector: 'app-consumer-status-widget',
   templateUrl: './consumer-status-widget.component.html',
-  styleUrls: ['./consumer-status-widget.component.css']
+  styleUrls: ['./consumer-status-widget.component.scss']
 })
 export class ConsumerStatusWidgetComponent implements OnInit {
 
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.css b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.css
deleted file mode 100644
index e69de29bb..000000000
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html
deleted file mode 100644
index 0c5ea634a..000000000
--- a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.html
+++ /dev/null
@@ -1,11 +0,0 @@
-<h4>Upload new Document</h4>
-<form>
-  <ngx-file-drop
-    dropZoneLabel="Drop documents here"
-    (onFileDrop)="dropped($event)"
-    (onFileOver)="fileOver($event)"
-    (onFileLeave)="fileLeave($event)"
-    dropZoneClassName="bg-light mt-4 card">
-
-  </ngx-file-drop>
-</form>
\ No newline at end of file
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts
deleted file mode 100644
index 847f5288b..000000000
--- a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.spec.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import { ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { FileUploadWidgetComponent } from './file-upload-widget.component';
-
-describe('FileUploadWidgetComponent', () => {
-  let component: FileUploadWidgetComponent;
-  let fixture: ComponentFixture<FileUploadWidgetComponent>;
-
-  beforeEach(async () => {
-    await TestBed.configureTestingModule({
-      declarations: [ FileUploadWidgetComponent ]
-    })
-    .compileComponents();
-  });
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(FileUploadWidgetComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});
diff --git a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts
deleted file mode 100644
index 5d4bac936..000000000
--- a/src-ui/src/app/components/dashboard/widgets/file-upload-widget/file-upload-widget.component.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-import { Component, OnInit } from '@angular/core';
-import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop';
-import { DocumentService } from 'src/app/services/rest/document.service';
-import { ToastService } from 'src/app/services/toast.service';
-
-@Component({
-  selector: 'app-file-upload-widget',
-  templateUrl: './file-upload-widget.component.html',
-  styleUrls: ['./file-upload-widget.component.css']
-})
-export class FileUploadWidgetComponent implements OnInit {
-
-  constructor(private documentService: DocumentService, private toastService: ToastService) { }
-
-  ngOnInit(): void {
-  }
-
-  public fileOver(event){
-    console.log(event);
-  }
- 
-  public fileLeave(event){
-    console.log(event);
-  }
- 
-  public dropped(files: NgxFileDropEntry[]) {
-    for (const droppedFile of files) {
-      if (droppedFile.fileEntry.isFile) {
-        const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
-        console.log(fileEntry)
-        fileEntry.file((file: File) => {
-          console.log(file)
-          const formData = new FormData()
-          formData.append('document', file, file.name)
-          this.documentService.uploadDocument(formData).subscribe(result => {
-            this.toastService.showInfo("The document has been uploaded and will be processed by the consumer shortly.")
-          }, error => {
-            this.toastService.showError("An error has occured while uploading the document. Sorry!")
-          })
-        });
-      }
-    }
-  }
-}
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index a95d5f4db..cb13b2d74 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -33,9 +33,9 @@ export class UploadFileWidgetComponent implements OnInit {
           const formData = new FormData()
           formData.append('document', file, file.name)
           this.documentService.uploadDocument(formData).subscribe(result => {
-            this.toastService.showToast(Toast.make("Information", "The document has been uploaded and will be processed by the consumer shortly."))
+            this.toastService.showInfo("The document has been uploaded and will be processed by the consumer shortly.")
           }, error => {
-            this.toastService.showToast(Toast.makeError("An error has occured while uploading the document. Sorry!"))
+            this.toastService.showError("An error has occured while uploading the document. Sorry!")
           })
         });
       }

From fcaaf7ce035d56647097ff8f10ab4c84b03ac8d9 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 6 Dec 2020 22:54:11 +0100
Subject: [PATCH 10/54] pipfile fix

---
 Pipfile.lock | 337 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 335 insertions(+), 2 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 6158a70e0..13d1d74ea 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "b10db53eb22d917723aa6107ff0970dc4e2aa886ee03d3ae08a994a856d57986"
+            "sha256": "3faa161608e685d788b8921f80b810b176fd2b4ed9020d3e6322dffecbcb5542"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -21,6 +21,13 @@
         ]
     },
     "default": {
+        "aioredis": {
+            "hashes": [
+                "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a",
+                "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"
+            ],
+            "version": "==1.3.1"
+        },
         "arrow": {
             "hashes": [
                 "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5",
@@ -37,6 +44,39 @@
             "markers": "python_version >= '3.5'",
             "version": "==3.3.1"
         },
+        "async-timeout": {
+            "hashes": [
+                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
+                "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
+            ],
+            "markers": "python_full_version >= '3.5.3'",
+            "version": "==3.0.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.3.0"
+        },
+        "autobahn": {
+            "hashes": [
+                "sha256:1eafbbe363a7924fd21bb0b94ece9f3ac2a9aa9c2046e8a85e044f94e8ba2028",
+                "sha256:24ce276d313e84d68241c3aef30d484f352b90a40168981b3640312c821df77b",
+                "sha256:86bbce30cdd407137c57670993a8f9bfdfe3f8e994b889181d85e844d5aa8dfb"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==20.7.1"
+        },
+        "automat": {
+            "hashes": [
+                "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33",
+                "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111",
+                "sha256:d6d976cf8da698fc85fa7def46e2544493f78cb7ee72d2f4acd1a5c759a3060e"
+            ],
+            "version": "==20.2.0"
+        },
         "blessed": {
             "hashes": [
                 "sha256:0a74a8d3f0366db600d061273df77d44f0db07daade7bb7a4d49c8bc22ed9f74",
@@ -87,6 +127,22 @@
             ],
             "version": "==1.14.4"
         },
+        "channels": {
+            "hashes": [
+                "sha256:74db79c9eca616be69d38013b22083ab5d3f9ccda1ab5e69096b1bb7da2d9b18",
+                "sha256:f50a6e79757a64c1e45e95e144a2ac5f1e99ee44a0718ab182c501f5e5abd268"
+            ],
+            "index": "pypi",
+            "version": "==3.0.2"
+        },
+        "channels-redis": {
+            "hashes": [
+                "sha256:18d63f6462a58011740dc8eeb57ea4b31ec220eb551cb71b27de9c6779a549de",
+                "sha256:2fb31a63b05373f6402da2e6a91a22b9e66eb8b56626c6bfc93e156c734c5ae6"
+            ],
+            "index": "pypi",
+            "version": "==3.2.0"
+        },
         "chardet": {
             "hashes": [
                 "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
@@ -104,6 +160,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==14.0"
         },
+        "constantly": {
+            "hashes": [
+                "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35",
+                "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"
+            ],
+            "version": "==15.1.0"
+        },
         "cryptography": {
             "hashes": [
                 "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538",
@@ -134,6 +197,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==3.2.1"
         },
+        "daphne": {
+            "hashes": [
+                "sha256:0052c9887600c57054a5867d4b0240159fa009faa3bcf6a1627271d9cdcb005a",
+                "sha256:c22b692707f514de9013651ecb687f2abe4f35cf6fe292ece634e9f1737bc7e3"
+            ],
+            "index": "pypi",
+            "version": "==3.0.1"
+        },
         "dateparser": {
             "hashes": [
                 "sha256:7552c994f893b5cb8fcf103b4cd2ff7f57aab9bfd2619fdf0cf571c0740fd90b",
@@ -213,6 +284,60 @@
             "index": "pypi",
             "version": "==20.0.4"
         },
+        "hiredis": {
+            "hashes": [
+                "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
+                "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0",
+                "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0",
+                "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01",
+                "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a",
+                "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b",
+                "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6",
+                "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73",
+                "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee",
+                "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55",
+                "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12",
+                "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b",
+                "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323",
+                "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c",
+                "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655",
+                "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5",
+                "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75",
+                "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb",
+                "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23",
+                "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1",
+                "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f",
+                "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872",
+                "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058",
+                "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454",
+                "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882",
+                "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2",
+                "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132",
+                "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6",
+                "sha256:9f4e67f87e072de981570eaf7cb41444bbac7e92b05c8651dbab6eb1fb8d5a14",
+                "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c",
+                "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363",
+                "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3",
+                "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4",
+                "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919",
+                "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349",
+                "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae",
+                "sha256:b39989b49e8aca9d224324d2650029eda410a4faf43f6afb0eb4f9acb7be6097",
+                "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da",
+                "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f",
+                "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed",
+                "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628",
+                "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64",
+                "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86",
+                "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf",
+                "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c",
+                "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded",
+                "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390",
+                "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.1.0"
+        },
         "humanfriendly": {
             "hashes": [
                 "sha256:175ffa628aa76da2c17369a5da5856084562cc66dfe7f82ae93ca3ef175277a6",
@@ -221,6 +346,22 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==9.0"
         },
+        "hyperlink": {
+            "hashes": [
+                "sha256:402c1b5fa066ea368f3118fc5a6f8505440b4d1a4ef12a844ca39332a4a29944",
+                "sha256:47fcc7cd339c6cb2444463ec3277bdcfe142c8b1daf2160bdd52248deec815af",
+                "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"
+            ],
+            "version": "==20.0.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226",
+                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
+                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
+            ],
+            "version": "==2.10"
+        },
         "imap-tools": {
             "hashes": [
                 "sha256:72bf46dc135b039a5d5b59f4e079242ac15eac02a30038e8cb2dec7b153cab65",
@@ -244,6 +385,13 @@
             "markers": "python_version < '3.8'",
             "version": "==3.1.1"
         },
+        "incremental": {
+            "hashes": [
+                "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
+                "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"
+            ],
+            "version": "==17.5.0"
+        },
         "inotify-simple": {
             "hashes": [
                 "sha256:8440ffe49c4ae81a8df57c1ae1eb4b6bfa7acb830099bfb3e305b383005cc128",
@@ -322,6 +470,32 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.6.2"
         },
+        "msgpack": {
+            "hashes": [
+                "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408",
+                "sha256:0e7b5a69ec5645b0a85baaa354c29acd89eb879aaa89e7f4b37ed4d9c5abafe0",
+                "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8",
+                "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84",
+                "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d",
+                "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a",
+                "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322",
+                "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2",
+                "sha256:71604047feea609ad65f5b837ec89a4de084d55a80f8af7331745a075c3dbd23",
+                "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e",
+                "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97",
+                "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0",
+                "sha256:aa5c057eab4f40ec47ea6f5a9825846be2ff6bf34102c560bad5cad5a677c5be",
+                "sha256:b3758dfd3423e358bbb18a7cccd1c74228dffa7a697e5be6cb9535de625c0dbf",
+                "sha256:c901e8058dd6653307906c5f157f26ed09eb94a850dddd989621098d347926ab",
+                "sha256:cec8bf10981ed70998d98431cd814db0ecf3384e6b113366e7f36af71a0fca08",
+                "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e",
+                "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272",
+                "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1",
+                "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140",
+                "sha256:f7c80ff32171193f18a127ea357118b920020cc0acb0730016bbda02b892a2d2"
+            ],
+            "version": "==1.0.0"
+        },
         "numpy": {
             "hashes": [
                 "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db",
@@ -514,6 +688,42 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "pyasn1": {
+            "hashes": [
+                "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359",
+                "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576",
+                "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf",
+                "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7",
+                "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d",
+                "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00",
+                "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8",
+                "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86",
+                "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12",
+                "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776",
+                "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba",
+                "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2",
+                "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"
+            ],
+            "version": "==0.4.8"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8",
+                "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199",
+                "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811",
+                "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed",
+                "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4",
+                "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e",
+                "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74",
+                "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb",
+                "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45",
+                "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd",
+                "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0",
+                "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d",
+                "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"
+            ],
+            "version": "==0.2.8"
+        },
         "pycparser": {
             "hashes": [
                 "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
@@ -522,6 +732,21 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.20"
         },
+        "pyhamcrest": {
+            "hashes": [
+                "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316",
+                "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.2"
+        },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:898aefbde331ba718570244c3b01dcddb1b31a3b336613436a45e52e27d9a82d",
+                "sha256:92f08eccbd73701cf744e8ffd6989aa7842d48cbe3fea8a7c031c5647f590ac5"
+            ],
+            "version": "==20.0.0"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
@@ -730,6 +955,13 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.5.4"
         },
+        "service-identity": {
+            "hashes": [
+                "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
+                "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"
+            ],
+            "version": "==18.1.0"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@@ -769,6 +1001,48 @@
             "index": "pypi",
             "version": "==4.54.1"
         },
+        "twisted": {
+            "extras": [
+                "tls"
+            ],
+            "hashes": [
+                "sha256:0150dae5adc962d15e00054cc6926f1e64763fb8dd26e1632593ac06e592104b",
+                "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f",
+                "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042",
+                "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c",
+                "sha256:15e52271f08f62e2230ff093e0278aa01c9dac057c4557cadadd2429eed86a3e",
+                "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292",
+                "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22",
+                "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec",
+                "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478",
+                "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2",
+                "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29",
+                "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114",
+                "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797",
+                "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa",
+                "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15",
+                "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd",
+                "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274",
+                "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad",
+                "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7",
+                "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a",
+                "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10",
+                "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780",
+                "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504",
+                "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467",
+                "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==20.3.0"
+        },
+        "txaio": {
+            "hashes": [
+                "sha256:17938f2bca4a9cabce61346758e482ca4e600160cbc28e861493eac74a19539d",
+                "sha256:38a469daf93c37e5527cb062653d6393ae11663147c42fab7ddc3f6d00d434ae"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==20.4.1"
+        },
         "tzlocal": {
             "hashes": [
                 "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44",
@@ -815,6 +1089,66 @@
             ],
             "markers": "python_version >= '3.6'",
             "version": "==3.4.0"
+        },
+        "zope.interface": {
+            "hashes": [
+                "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
+                "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d",
+                "sha256:09fc3922f235703c0b76f8234867685eee68a24a49fffa2220975f6142db45f1",
+                "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123",
+                "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232",
+                "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549",
+                "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102",
+                "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5",
+                "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45",
+                "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00",
+                "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc",
+                "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7",
+                "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104",
+                "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034",
+                "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3",
+                "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3",
+                "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4",
+                "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86",
+                "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96",
+                "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546",
+                "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb",
+                "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3",
+                "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b",
+                "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b",
+                "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec",
+                "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae",
+                "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e",
+                "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386",
+                "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2",
+                "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a",
+                "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d",
+                "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a",
+                "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24",
+                "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d",
+                "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b",
+                "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50",
+                "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523",
+                "sha256:974f5957e66a7524ea81df7b2686a456bfaf0408dbb7353ddfbedb594eadfef6",
+                "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a",
+                "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095",
+                "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a",
+                "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520",
+                "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65",
+                "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11",
+                "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c",
+                "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7",
+                "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332",
+                "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e",
+                "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c",
+                "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7",
+                "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20",
+                "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc",
+                "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd",
+                "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==5.2.0"
         }
     },
     "develop": {
@@ -980,7 +1314,6 @@
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "imagesize": {

From e2456f4b3fb586edcb7794915b258fb4aa118489 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 7 Dec 2020 12:44:23 +0100
Subject: [PATCH 11/54] changes

---
 .../consumer-status-widget.component.html     | 16 +++++-----
 .../saved-view-widget.component.css           |  0
 .../statistics-widget.component.css           |  0
 .../upload-file-widget.component.ts           |  2 +-
 .../document-list/document-list.component.ts  |  2 +-
 src/documents/consumer.py                     | 29 +++++++++----------
 6 files changed, 25 insertions(+), 24 deletions(-)
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.css
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.css

diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
index ff2117729..9ea715043 100644
--- a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
@@ -1,10 +1,12 @@
 <app-widget-frame title="Document consumer status">
-  <div class="mb-2 border-bottom" *ngFor="let s of getStatus()">
-    <div class="mb-1"><strong>{{s.filename}}:</strong> {{s.message}}</div>
-    <ngb-progressbar [type]="getType(s.status)" [value]="s.current_progress" [max]="s.max_progress" class="mb-2"></ngb-progressbar>
-    <div *ngIf="isFinished(s)" class="mb-2">
-      <button *ngIf="s.document_id" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
-      <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(s)">Dismiss</button>
+  <ng-container content>
+    <div class="mb-2 border-bottom" *ngFor="let s of getStatus()">
+      <div class="mb-1"><strong>{{s.filename}}:</strong> {{s.message}}</div>
+      <ngb-progressbar [type]="getType(s.status)" [value]="s.current_progress" [max]="s.max_progress" class="mb-2"></ngb-progressbar>
+      <div *ngIf="isFinished(s)" class="mb-2">
+        <button *ngIf="s.document_id" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
+        <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(s)">Dismiss</button>
+      </div>
     </div>
-  </div>
+  </ng-container>
 </app-widget-frame>
diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.css b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.css
deleted file mode 100644
index e69de29bb..000000000
diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.css b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.css
deleted file mode 100644
index e69de29bb..000000000
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index 97b4ffee8..16a220229 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -29,7 +29,7 @@ export class UploadFileWidgetComponent implements OnInit {
           const formData = new FormData()
           formData.append('document', file, file.name)
           this.documentService.uploadDocument(formData).subscribe(result => {
-            this.toastService.showInfo(The document has been uploaded and will be processed by the consumer shortly.")
+            this.toastService.showInfo("The document has been uploaded and will be processed by the consumer shortly.")
           }, error => {
             switch (error.status) {
               case 400: {
diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts
index fe6c8a894..4dc986d51 100644
--- a/src-ui/src/app/components/document-list/document-list.component.ts
+++ b/src-ui/src/app/components/document-list/document-list.component.ts
@@ -76,7 +76,7 @@ export class DocumentListComponent implements OnInit {
 
   saveViewConfig() {
     this.savedViewConfigService.updateConfig(this.list.savedView)
-    this.toastService.showToast(Toast.make("Information", `View "${this.list.savedView.title}" saved successfully.`))
+    this.toastService.showInfo(`View "${this.list.savedView.title}" saved successfully.`)
   }
 
   saveViewConfigAs() {
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index b57e81d06..30c57e6a0 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -43,6 +43,11 @@ class Consumer(LoggingMixin):
                                                      {'type': 'status_update',
                                                       'data': payload})
 
+    def _fail(self, message):
+        self._send_progress(self.filename, 100, 100, 'FAILED',
+                            message)
+        raise ConsumerError(f"{self.filename}: {message}")
+
     def __init__(self):
         super().__init__()
         self.path = None
@@ -56,8 +61,7 @@ class Consumer(LoggingMixin):
 
     def pre_check_file_exists(self):
         if not os.path.isfile(self.path):
-            raise ConsumerError("Cannot consume {}: It is not a file".format(
-                self.path))
+            self._fail("File not found")
 
     def pre_check_duplicate(self):
         with open(self.path, "rb") as f:
@@ -65,9 +69,7 @@ class Consumer(LoggingMixin):
         if Document.objects.filter(Q(checksum=checksum) | Q(archive_checksum=checksum)).exists():  # NOQA: E501
             if settings.CONSUMER_DELETE_DUPLICATES:
                 os.unlink(self.path)
-            raise ConsumerError(
-                "Not consuming {}: It is a duplicate.".format(self.filename)
-            )
+            self._fail("Document is a duplicate")
 
     def pre_check_directories(self):
         os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
@@ -93,6 +95,9 @@ class Consumer(LoggingMixin):
         self.override_document_type_id = override_document_type_id
         self.override_tag_ids = override_tag_ids
 
+        self._send_progress(self.filename, 0, 100, 'WORKING',
+                            'Received new file.')
+
         # this is for grouping logging entries for this particular file
         # together.
 
@@ -112,7 +117,7 @@ class Consumer(LoggingMixin):
 
         parser_class = get_parser_class_for_mime_type(mime_type)
         if not parser_class:
-            raise ConsumerError(f"No parsers abvailable for {self.filename}")
+            self._fail("No parsers abvailable")
         else:
             self.log("debug",
                      f"Parser: {parser_class.__name__} "
@@ -120,8 +125,6 @@ class Consumer(LoggingMixin):
 
         # Notify all listeners that we're going to do some work.
 
-        self._send_progress(self.filename, 0, 100, 'WORKING', 'Consumption started')
-
         document_consumption_started.send(
             sender=self.__class__,
             filename=self.path,
@@ -130,7 +133,7 @@ class Consumer(LoggingMixin):
 
         def progress_callback(current_progress, max_progress, message):
             # recalculate progress to be within 20 and 80
-            p = int((current_progress / max_progress) * 60 + 20)
+            p = int((current_progress / max_progress) * 50 + 20)
             self._send_progress(self.filename, p, 100, "WORKING", message)
 
         # This doesn't parse the document yet, but gives us a parser.
@@ -167,9 +170,7 @@ class Consumer(LoggingMixin):
             self.log(
                 "error",
                 f"Error while consuming document {self.filename}: {e}")
-            self._send_progress(self.filename, 100, 100, 'FAILED',
-                                "Failed: {}".format(e))
-            raise ConsumerError(e)
+            self._fail(e)
 
         # Prepare the document classifier.
 
@@ -246,9 +247,7 @@ class Consumer(LoggingMixin):
                 f"The following error occured while consuming "
                 f"{self.filename}: {e}"
             )
-            self._send_progress(self.filename, 100, 100, 'FAILED',
-                                "Failed: {}".format(e))
-            raise ConsumerError(e)
+            self._fail(str(e))
         finally:
             document_parser.cleanup()
 

From d153672f0d32358a469b33785c4d283782f34c10 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 4 Jan 2021 23:05:16 +0100
Subject: [PATCH 12/54] some changes

---
 Pipfile.lock                                  | 420 +++++++++++++++---
 src-ui/src/app/app.component.ts               |   2 +-
 .../common/toasts/toasts.component.html       |   4 +-
 src-ui/src/environments/environment.ts        |   2 +-
 src/documents/consumer.py                     |   2 +-
 src/documents/views.py                        |   2 +-
 6 files changed, 356 insertions(+), 76 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 6d9685f9c..068b7cd07 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "993e362c31af6b8094693075f614270a820cf0b557369d66d674e1a107b7bd31"
+            "sha256": "8dbe59054648cdfec443bad9f73c25263e43f18f35e74dc72a8429a0035ecbeb"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -21,6 +21,13 @@
         ]
     },
     "default": {
+        "aioredis": {
+            "hashes": [
+                "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a",
+                "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"
+            ],
+            "version": "==1.3.1"
+        },
         "arrow": {
             "hashes": [
                 "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5",
@@ -37,6 +44,38 @@
             "markers": "python_version >= '3.5'",
             "version": "==3.3.1"
         },
+        "async-timeout": {
+            "hashes": [
+                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
+                "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
+            ],
+            "markers": "python_full_version >= '3.5.3'",
+            "version": "==3.0.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.3.0"
+        },
+        "autobahn": {
+            "hashes": [
+                "sha256:410a93e0e29882c8b5d5ab05d220b07609b886ef5f23c0b8d39153254ffd6895",
+                "sha256:52ee4236ff9a1fcbbd9500439dcf3284284b37f8a6b31ecc8a36e00cf9f95049"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==20.12.3"
+        },
+        "automat": {
+            "hashes": [
+                "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33",
+                "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111",
+                "sha256:d6d976cf8da698fc85fa7def46e2544493f78cb7ee72d2f4acd1a5c759a3060e"
+            ],
+            "version": "==20.2.0"
+        },
         "blessed": {
             "hashes": [
                 "sha256:0a74a8d3f0366db600d061273df77d44f0db07daade7bb7a4d49c8bc22ed9f74",
@@ -94,6 +133,22 @@
             ],
             "version": "==1.14.4"
         },
+        "channels": {
+            "hashes": [
+                "sha256:056b72e51080a517a0f33a0a30003e03833b551d75394d6636c885d4edb8188f",
+                "sha256:3f15bdd2138bb4796e76ea588a0a344b12a7964ea9b2e456f992fddb988a4317"
+            ],
+            "index": "pypi",
+            "version": "==3.0.3"
+        },
+        "channels-redis": {
+            "hashes": [
+                "sha256:18d63f6462a58011740dc8eeb57ea4b31ec220eb551cb71b27de9c6779a549de",
+                "sha256:2fb31a63b05373f6402da2e6a91a22b9e66eb8b56626c6bfc93e156c734c5ae6"
+            ],
+            "index": "pypi",
+            "version": "==3.2.0"
+        },
         "chardet": {
             "hashes": [
                 "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
@@ -110,6 +165,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==15.0"
         },
+        "constantly": {
+            "hashes": [
+                "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35",
+                "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"
+            ],
+            "version": "==15.1.0"
+        },
         "cryptography": {
             "hashes": [
                 "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d",
@@ -131,6 +193,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==3.3.1"
         },
+        "daphne": {
+            "hashes": [
+                "sha256:0052c9887600c57054a5867d4b0240159fa009faa3bcf6a1627271d9cdcb005a",
+                "sha256:c22b692707f514de9013651ecb687f2abe4f35cf6fe292ece634e9f1737bc7e3"
+            ],
+            "index": "pypi",
+            "version": "==3.0.1"
+        },
         "dateparser": {
             "hashes": [
                 "sha256:7552c994f893b5cb8fcf103b4cd2ff7f57aab9bfd2619fdf0cf571c0740fd90b",
@@ -218,6 +288,60 @@
             "index": "pypi",
             "version": "==20.0.4"
         },
+        "hiredis": {
+            "hashes": [
+                "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
+                "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0",
+                "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0",
+                "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01",
+                "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a",
+                "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b",
+                "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6",
+                "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73",
+                "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee",
+                "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55",
+                "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12",
+                "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b",
+                "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323",
+                "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c",
+                "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655",
+                "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5",
+                "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75",
+                "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb",
+                "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23",
+                "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1",
+                "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f",
+                "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872",
+                "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058",
+                "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454",
+                "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882",
+                "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2",
+                "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132",
+                "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6",
+                "sha256:9f4e67f87e072de981570eaf7cb41444bbac7e92b05c8651dbab6eb1fb8d5a14",
+                "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c",
+                "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363",
+                "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3",
+                "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4",
+                "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919",
+                "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349",
+                "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae",
+                "sha256:b39989b49e8aca9d224324d2650029eda410a4faf43f6afb0eb4f9acb7be6097",
+                "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da",
+                "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f",
+                "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed",
+                "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628",
+                "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64",
+                "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86",
+                "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf",
+                "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c",
+                "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded",
+                "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390",
+                "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.1.0"
+        },
         "humanfriendly": {
             "hashes": [
                 "sha256:066562956639ab21ff2676d1fda0b5987e985c534fc76700a19bd54bcb81121d",
@@ -226,6 +350,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==9.1"
         },
+        "hyperlink": {
+            "hashes": [
+                "sha256:402c1b5fa066ea368f3118fc5a6f8505440b4d1a4ef12a844ca39332a4a29944",
+                "sha256:47fcc7cd339c6cb2444463ec3277bdcfe142c8b1daf2160bdd52248deec815af",
+                "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63"
+            ],
+            "version": "==20.0.1"
+        },
         "idna": {
             "hashes": [
                 "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226",
@@ -250,13 +382,12 @@
             ],
             "version": "==0.4.0"
         },
-        "importlib-metadata": {
+        "incremental": {
             "hashes": [
-                "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed",
-                "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"
+                "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
+                "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"
             ],
-            "markers": "python_version < '3.8'",
-            "version": "==3.3.0"
+            "version": "==17.5.0"
         },
         "inotify-simple": {
             "hashes": [
@@ -336,6 +467,41 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.6.2"
         },
+        "msgpack": {
+            "hashes": [
+                "sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9",
+                "sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841",
+                "sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439",
+                "sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694",
+                "sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a",
+                "sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f",
+                "sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e",
+                "sha256:7307e86f7ce75b49e65b55660b10b258e9e7b5e0f80d31d7a86a278d8204d1b4",
+                "sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1",
+                "sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c",
+                "sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b",
+                "sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759",
+                "sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326",
+                "sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc",
+                "sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192",
+                "sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83",
+                "sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06",
+                "sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e",
+                "sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9",
+                "sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33",
+                "sha256:c82dc0ba34d620fb94d12a7725e9362958bb1be3938688a061f53ed86bee005a",
+                "sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54",
+                "sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f",
+                "sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887",
+                "sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009",
+                "sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2",
+                "sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c",
+                "sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87",
+                "sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984",
+                "sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6"
+            ],
+            "version": "==1.0.2"
+        },
         "numpy": {
             "hashes": [
                 "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db",
@@ -521,6 +687,42 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "pyasn1": {
+            "hashes": [
+                "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359",
+                "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576",
+                "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf",
+                "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7",
+                "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d",
+                "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00",
+                "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8",
+                "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86",
+                "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12",
+                "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776",
+                "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba",
+                "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2",
+                "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"
+            ],
+            "version": "==0.4.8"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8",
+                "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199",
+                "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811",
+                "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed",
+                "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4",
+                "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e",
+                "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74",
+                "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb",
+                "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45",
+                "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd",
+                "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0",
+                "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d",
+                "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"
+            ],
+            "version": "==0.2.8"
+        },
         "pycparser": {
             "hashes": [
                 "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
@@ -529,6 +731,22 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.20"
         },
+        "pyhamcrest": {
+            "hashes": [
+                "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316",
+                "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.2"
+        },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51",
+                "sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==20.0.1"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
@@ -715,35 +933,36 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:168c45c0c32e23f613db7c9e4e780bc61982d71dcd406ead746c7c7c2f2004ce",
-                "sha256:213bc59191da2f479984ad4ec39406bf949a99aba70e9237b916ce7547b6ef42",
-                "sha256:25b241034215247481f53355e05f9e25462682b13bd9191359075682adcd9554",
-                "sha256:2c872de0c69ed20fb1a9b9cf6f77298b04a26f0b8720a5457be08be254366c6e",
-                "sha256:3397c129b479846d7eaa18f999369a24322d008fac0782e7828fa567358c36ce",
-                "sha256:368c0f69f93186309e1b4beb8e26d51dd6f5010b79264c0f1e9ca00cd92ea8c9",
-                "sha256:3d5db5d815370c28d938cf9b0809dade4acf7aba57eaf7ef733bfedc9b2474c4",
-                "sha256:4598cf03136067000855d6b44d7a1f4f46994164bcd450fb2c3d481afc25dd06",
-                "sha256:4a453d5e5689de62e5d38edf40af3f17560bfd63c9c5bd228c18c1f99afa155b",
-                "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47",
-                "sha256:634568a3018bc16a83cda28d4f7aed0d803dd5618facb36e977e53b2df868443",
-                "sha256:65923bc3809524e46fb7eb4d6346552cbb6a1ffc41be748535aa502a2e3d3389",
-                "sha256:6b0ceb23560f46dd236a8ad4378fc40bad1783e997604ba845e131d6c680963e",
-                "sha256:8c8d6ca19c8497344b810b0b0344f8375af5f6bb9c98bd42e33f747417ab3f57",
-                "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62",
-                "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d",
-                "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437",
-                "sha256:b5e9d3e4474644915809d6aa1416ff20430a3ed9ae723a5d295da5ddb24985e2",
-                "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2",
-                "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54",
-                "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474",
-                "sha256:e360cb2299028d0b0d0f65a5c5e51fc16a335f1603aa2357c25766c8dab56938",
-                "sha256:e98d49a5717369d8241d6cf33ecb0ca72deee392414118198a8e5b4c35c56340",
-                "sha256:ed572470af2438b526ea574ff8f05e7f39b44ac37f712105e57fc4d53a6fb660",
-                "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012",
-                "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c"
+                "sha256:155225621df90fcd151e25d51c50217e412de717475999ebb76e17e310176981",
+                "sha256:1bc5b446600c4ff7ab36bade47180673141322f0febaa555f1c433fe04f2a0e3",
+                "sha256:2f1c2ebca6fd867160e70102200b1bd07b3b2d31a3e6af3c58d688c15d0d07b7",
+                "sha256:313785c4dab65060f9648112d025f6d2fec69a8a889c714328882d678a95f053",
+                "sha256:31ab217b5c27ab429d07428a76002b33662f98986095bbce5d55e0788f7e8b15",
+                "sha256:3d4303e3e21d07d9557b26a1707bb9fc065510ee8501c9bf22a0157249a82fd0",
+                "sha256:4f1d9cc977ac6a4a63c124045c1e8bf67ec37098f67c699887a93736961a00ae",
+                "sha256:58731bbe0103e96b89b2f41516699db9b63066e4317e31b8402891571f6d358f",
+                "sha256:8629135ee00cc2182ac8be8e75643b9f02235942443732c2ed69ab48edcb6614",
+                "sha256:876badc33eec20709d4e042a09834f5953ebdac4088d45a4f3a1f18b56885718",
+                "sha256:8840a9adb4ede3751f49761653d3ebf664f25195fdd42ada394ffea8903dd51d",
+                "sha256:aef3a2dbc436bbe8f6e0b635f0b5fe5ed024b522eee4637dbbe0b974129ca734",
+                "sha256:b8af26839ae343655f3ca377a5d5e5466f1d3b3ac7432a43449154fe958ae0e0",
+                "sha256:c0911f3180de343643f369dc5cfedad6ba9f939c2d516bddea4a6871eb000722",
+                "sha256:cb6dc9f82dfd95f6b9032a8d7ea70efeeb15d5b5fd6ed4e8537bb3c673580566",
+                "sha256:cdbc47628184a0ebeb5c08f1892614e1bd4a51f6e0d609c6eed253823a960f5b",
+                "sha256:d902d3a5ad7f28874c0a82db95246d24ca07ad932741df668595fe00a4819870",
+                "sha256:eab389aba0ad8b5e6b5abdc3337ade46823df75f80a8edd4c67833567577cb3d",
+                "sha256:eb7928275f3560d47e5538e15e9f32b3d64cd30ea8f85f3e82987425476f53f6",
+                "sha256:f68d5761a2d2376e2b194c8e9192bbf7c51306ca176f1a0889990a52ef0d551f"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.5.4"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.6.0"
+        },
+        "service-identity": {
+            "hashes": [
+                "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
+                "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"
+            ],
+            "version": "==18.1.0"
         },
         "six": {
             "hashes": [
@@ -792,14 +1011,47 @@
             "index": "pypi",
             "version": "==4.55.1"
         },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
-                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
-                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+        "twisted": {
+            "extras": [
+                "tls"
             ],
-            "markers": "python_version < '3.8'",
-            "version": "==3.7.4.3"
+            "hashes": [
+                "sha256:0150dae5adc962d15e00054cc6926f1e64763fb8dd26e1632593ac06e592104b",
+                "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f",
+                "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042",
+                "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c",
+                "sha256:15e52271f08f62e2230ff093e0278aa01c9dac057c4557cadadd2429eed86a3e",
+                "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292",
+                "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22",
+                "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec",
+                "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478",
+                "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2",
+                "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29",
+                "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114",
+                "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797",
+                "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa",
+                "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15",
+                "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd",
+                "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274",
+                "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad",
+                "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7",
+                "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a",
+                "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10",
+                "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780",
+                "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504",
+                "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467",
+                "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==20.3.0"
+        },
+        "txaio": {
+            "hashes": [
+                "sha256:1488d31d564a116538cc1265ac3f7979fb6223bb5a9e9f1479436ee2c17d8549",
+                "sha256:a8676d6c68aea1f0e2548c4afdb8e6253873af3bc2659bb5bcd9f39dff7ff90f"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==20.12.1"
         },
         "tzlocal": {
             "hashes": [
@@ -863,13 +1115,65 @@
             "index": "pypi",
             "version": "==2.7.4"
         },
-        "zipp": {
+        "zope.interface": {
             "hashes": [
-                "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
-                "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
+                "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
+                "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d",
+                "sha256:09fc3922f235703c0b76f8234867685eee68a24a49fffa2220975f6142db45f1",
+                "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123",
+                "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232",
+                "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549",
+                "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102",
+                "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5",
+                "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45",
+                "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00",
+                "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc",
+                "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7",
+                "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104",
+                "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034",
+                "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3",
+                "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3",
+                "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4",
+                "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86",
+                "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96",
+                "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546",
+                "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb",
+                "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3",
+                "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b",
+                "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b",
+                "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec",
+                "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae",
+                "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e",
+                "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386",
+                "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2",
+                "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a",
+                "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d",
+                "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a",
+                "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24",
+                "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d",
+                "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b",
+                "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50",
+                "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523",
+                "sha256:974f5957e66a7524ea81df7b2686a456bfaf0408dbb7353ddfbedb594eadfef6",
+                "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a",
+                "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095",
+                "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a",
+                "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520",
+                "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65",
+                "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11",
+                "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c",
+                "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7",
+                "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332",
+                "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e",
+                "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c",
+                "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7",
+                "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20",
+                "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc",
+                "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd",
+                "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.0"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==5.2.0"
         }
     },
     "develop": {
@@ -1062,22 +1366,6 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.2.0"
         },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed",
-                "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==3.3.0"
-        },
-        "importlib-resources": {
-            "hashes": [
-                "sha256:0a948d0c8c3f9344de62997e3f73444dbba233b1eaf24352933c2d264b9e4182",
-                "sha256:6b45007a479c4ec21165ae3ffbe37faf35404e2041fac6ae1da684f38530ca73"
-            ],
-            "markers": "python_version < '3.7'",
-            "version": "==4.1.1"
-        },
         "iniconfig": {
             "hashes": [
                 "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
@@ -1389,14 +1677,6 @@
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.2.2"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
-                "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.0"
         }
     }
 }
diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts
index 55c23b4de..8ca276dd4 100644
--- a/src-ui/src/app/app.component.ts
+++ b/src-ui/src/app/app.component.ts
@@ -31,7 +31,7 @@ export class AppComponent implements OnInit, OnDestroy {
     this.consumerStatusService.connect()
 
     this.successSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
-      this.toastService.showToast({title: "Document added", content: `Document ${status.filename} was added to paperless.`, actionName: "Open document", action: () => {
+      this.toastService.show({title: "Document added", delay: 10000, content: `Document ${status.filename} was added to paperless.`, actionName: "Open document", action: () => {
         this.router.navigate(['documents', status.document_id])
       }})
     })
diff --git a/src-ui/src/app/components/common/toasts/toasts.component.html b/src-ui/src/app/components/common/toasts/toasts.component.html
index 4e920877e..9d4ae6bb6 100644
--- a/src-ui/src/app/components/common/toasts/toasts.component.html
+++ b/src-ui/src/app/components/common/toasts/toasts.component.html
@@ -1,8 +1,8 @@
 <ngb-toast
   *ngFor="let toast of toasts"
-  [header]="toast.title" [autohide]="true" [delay]="toast.delay || 5000"
+  [header]="toast.title" [autohide]="true" [delay]="toast.delay"
   [class]="toast.classname"
   (hide)="toastService.closeToast(toast)">
   <p>{{toast.content}}</p>
   <p *ngIf="toast.action"><button class="btn btn-sm btn-outline-secondary" (click)="toastService.closeToast(toast); toast.action()">{{toast.actionName}}</button></p>
-</ngb-toast>
\ No newline at end of file
+</ngb-toast>
diff --git a/src-ui/src/environments/environment.ts b/src-ui/src/environments/environment.ts
index da87c55e9..29dcb7ba1 100644
--- a/src-ui/src/environments/environment.ts
+++ b/src-ui/src/environments/environment.ts
@@ -6,7 +6,7 @@ export const environment = {
   production: false,
   apiBaseUrl: "http://localhost:8000/api/",
   appTitle: "Paperless-ng",
-  version: "DEVELOPMENT"
+  version: "DEVELOPMENT",
   wsBaseUrl: "ws://localhost:8000/ws/"
 };
 
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 1cd05d2be..21a4da66b 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -216,7 +216,7 @@ class Consumer(LoggingMixin):
             self.log(
                 "error",
                 f"Error while consuming document {self.filename}: {e}")
-            self._fail(e)
+            self._fail(str(e))
 
         # Prepare the document classifier.
 
diff --git a/src/documents/views.py b/src/documents/views.py
index 8af29ce05..af4861ad4 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -211,7 +211,7 @@ class DocumentViewSet(RetrieveModelMixin,
 
         parser_class = get_parser_class_for_mime_type(mime_type)
         if parser_class:
-            parser = parser_class(logging_group=None)
+            parser = parser_class(progress_callback=None, logging_group=None)
 
             try:
                 return parser.extract_metadata(file, mime_type)

From 77b85b9e93244b8527d0d703dc4b2e3cedff236f Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 00:34:36 +0100
Subject: [PATCH 13/54] remove status widget

---
 src-ui/src/app/app.module.ts                  |  4 +--
 .../dashboard/dashboard.component.html        |  4 +--
 .../consumer-status-widget.component.html     | 12 -------
 .../consumer-status-widget.component.scss     |  0
 .../consumer-status-widget.component.spec.ts  | 25 -------------
 .../consumer-status-widget.component.ts       | 35 -------------------
 6 files changed, 2 insertions(+), 78 deletions(-)
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.scss
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts
 delete mode 100644 src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts

diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts
index ae5cc70b9..49e937c0b 100644
--- a/src-ui/src/app/app.module.ts
+++ b/src-ui/src/app/app.module.ts
@@ -60,7 +60,6 @@ import { NgSelectModule } from '@ng-select/ng-select';
 import { NumberComponent } from './components/common/input/number/number.component';
 import { SafePipe } from './pipes/safe.pipe';
 import { CustomDatePipe } from './pipes/custom-date.pipe';
-import { ConsumerStatusWidgetComponent } from './components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component';
 
 import localeFr from '@angular/common/locales/fr';
 import localeNl from '@angular/common/locales/nl';
@@ -119,8 +118,7 @@ registerLocaleData(localeDe)
     SelectDialogComponent,
     NumberComponent,
     SafePipe,
-    CustomDatePipe,
-    ConsumerStatusWidgetComponent
+    CustomDatePipe
   ],
   imports: [
     BrowserModule,
diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html
index 40fee100f..ec2bbd85c 100644
--- a/src-ui/src/app/components/dashboard/dashboard.component.html
+++ b/src-ui/src/app/components/dashboard/dashboard.component.html
@@ -19,8 +19,6 @@
     <app-statistics-widget></app-statistics-widget>
 
     <app-upload-file-widget></app-upload-file-widget>
-
-    <app-consumer-status-widget></app-consumer-status-widget>
-
+  
   </div>
 </div>
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
deleted file mode 100644
index 9ea715043..000000000
--- a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.html
+++ /dev/null
@@ -1,12 +0,0 @@
-<app-widget-frame title="Document consumer status">
-  <ng-container content>
-    <div class="mb-2 border-bottom" *ngFor="let s of getStatus()">
-      <div class="mb-1"><strong>{{s.filename}}:</strong> {{s.message}}</div>
-      <ngb-progressbar [type]="getType(s.status)" [value]="s.current_progress" [max]="s.max_progress" class="mb-2"></ngb-progressbar>
-      <div *ngIf="isFinished(s)" class="mb-2">
-        <button *ngIf="s.document_id" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{s.document_id}}" (click)="dismiss(s)">Open document</button>
-        <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(s)">Dismiss</button>
-      </div>
-    </div>
-  </ng-container>
-</app-widget-frame>
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.scss
deleted file mode 100644
index e69de29bb..000000000
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts
deleted file mode 100644
index 02aa088f5..000000000
--- a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.spec.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import { ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { ConsumerStatusWidgetComponent } from './consumer-status-widget.component';
-
-describe('ConsumerStatusWidgetComponent', () => {
-  let component: ConsumerStatusWidgetComponent;
-  let fixture: ComponentFixture<ConsumerStatusWidgetComponent>;
-
-  beforeEach(async () => {
-    await TestBed.configureTestingModule({
-      declarations: [ ConsumerStatusWidgetComponent ]
-    })
-    .compileComponents();
-  });
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(ConsumerStatusWidgetComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});
diff --git a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
deleted file mode 100644
index 8e44af6d5..000000000
--- a/src-ui/src/app/components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import { Component, OnInit } from '@angular/core';
-import { ConsumerStatusService, FileStatus } from 'src/app/services/consumer-status.service';
-
-@Component({
-  selector: 'app-consumer-status-widget',
-  templateUrl: './consumer-status-widget.component.html',
-  styleUrls: ['./consumer-status-widget.component.scss']
-})
-export class ConsumerStatusWidgetComponent implements OnInit {
-
-  constructor(private consumerStatusService: ConsumerStatusService) { }
-
-  ngOnInit(): void {
-  }
-
-  getStatus() {
-    return this.consumerStatusService.consumerStatus
-  }
-
-  isFinished(status: FileStatus) {
-    return status.status == "FAILED" || status.status == "SUCCESS"
-  }
-
-  getType(status) {
-    switch (status) {
-      case "WORKING": return "primary"
-      case "FAILED": return "danger"
-      case "SUCCESS": return "success"
-    }
-  }
-
-  dismiss(status: FileStatus) {
-    this.consumerStatusService.dismiss(status)
-  }
-}

From 2fa78f782062b49a45ad7181865eba609ac958ff Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 00:35:07 +0100
Subject: [PATCH 14/54] update Pipfile.lock

---
 Pipfile.lock | 522 +++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 425 insertions(+), 97 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index dab52dbbd..267b18533 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "3c85a487240f18b3feb44f8899395696cb79630f320f0df9ef5ee37b914c89f2"
+            "sha256": "08cd30c074677e99d7b6bada67b7c6b7bb5348a9210753ea3c7841c95cad1037"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -19,6 +19,13 @@
         ]
     },
     "default": {
+        "aioredis": {
+            "hashes": [
+                "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a",
+                "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3"
+            ],
+            "version": "==1.3.1"
+        },
         "arrow": {
             "hashes": [
                 "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5",
@@ -35,6 +42,38 @@
             "markers": "python_version >= '3.5'",
             "version": "==3.3.1"
         },
+        "async-timeout": {
+            "hashes": [
+                "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f",
+                "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"
+            ],
+            "markers": "python_full_version >= '3.5.3'",
+            "version": "==3.0.1"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.3.0"
+        },
+        "autobahn": {
+            "hashes": [
+                "sha256:410a93e0e29882c8b5d5ab05d220b07609b886ef5f23c0b8d39153254ffd6895",
+                "sha256:52ee4236ff9a1fcbbd9500439dcf3284284b37f8a6b31ecc8a36e00cf9f95049"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==20.12.3"
+        },
+        "automat": {
+            "hashes": [
+                "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33",
+                "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111",
+                "sha256:d6d976cf8da698fc85fa7def46e2544493f78cb7ee72d2f4acd1a5c759a3060e"
+            ],
+            "version": "==20.2.0"
+        },
         "blessed": {
             "hashes": [
                 "sha256:0a74a8d3f0366db600d061273df77d44f0db07daade7bb7a4d49c8bc22ed9f74",
@@ -92,6 +131,22 @@
             ],
             "version": "==1.14.4"
         },
+        "channels": {
+            "hashes": [
+                "sha256:056b72e51080a517a0f33a0a30003e03833b551d75394d6636c885d4edb8188f",
+                "sha256:3f15bdd2138bb4796e76ea588a0a344b12a7964ea9b2e456f992fddb988a4317"
+            ],
+            "index": "pypi",
+            "version": "==3.0.3"
+        },
+        "channels-redis": {
+            "hashes": [
+                "sha256:18d63f6462a58011740dc8eeb57ea4b31ec220eb551cb71b27de9c6779a549de",
+                "sha256:2fb31a63b05373f6402da2e6a91a22b9e66eb8b56626c6bfc93e156c734c5ae6"
+            ],
+            "index": "pypi",
+            "version": "==3.2.0"
+        },
         "chardet": {
             "hashes": [
                 "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
@@ -108,6 +163,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==15.0"
         },
+        "constantly": {
+            "hashes": [
+                "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35",
+                "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"
+            ],
+            "version": "==15.1.0"
+        },
         "cryptography": {
             "hashes": [
                 "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d",
@@ -129,6 +191,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
             "version": "==3.3.1"
         },
+        "daphne": {
+            "hashes": [
+                "sha256:0052c9887600c57054a5867d4b0240159fa009faa3bcf6a1627271d9cdcb005a",
+                "sha256:c22b692707f514de9013651ecb687f2abe4f35cf6fe292ece634e9f1737bc7e3"
+            ],
+            "index": "pypi",
+            "version": "==3.0.1"
+        },
         "dateparser": {
             "hashes": [
                 "sha256:7552c994f893b5cb8fcf103b4cd2ff7f57aab9bfd2619fdf0cf571c0740fd90b",
@@ -217,6 +287,60 @@
             "index": "pypi",
             "version": "==20.0.4"
         },
+        "hiredis": {
+            "hashes": [
+                "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
+                "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0",
+                "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0",
+                "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01",
+                "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a",
+                "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b",
+                "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6",
+                "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73",
+                "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee",
+                "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55",
+                "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12",
+                "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b",
+                "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323",
+                "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c",
+                "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655",
+                "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5",
+                "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75",
+                "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb",
+                "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23",
+                "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1",
+                "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f",
+                "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872",
+                "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058",
+                "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454",
+                "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882",
+                "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2",
+                "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132",
+                "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6",
+                "sha256:9f4e67f87e072de981570eaf7cb41444bbac7e92b05c8651dbab6eb1fb8d5a14",
+                "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c",
+                "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363",
+                "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3",
+                "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4",
+                "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919",
+                "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349",
+                "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae",
+                "sha256:b39989b49e8aca9d224324d2650029eda410a4faf43f6afb0eb4f9acb7be6097",
+                "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da",
+                "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f",
+                "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed",
+                "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628",
+                "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64",
+                "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86",
+                "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf",
+                "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c",
+                "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded",
+                "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390",
+                "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.1.0"
+        },
         "humanfriendly": {
             "hashes": [
                 "sha256:066562956639ab21ff2676d1fda0b5987e985c534fc76700a19bd54bcb81121d",
@@ -225,6 +349,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==9.1"
         },
+        "hyperlink": {
+            "hashes": [
+                "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b",
+                "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"
+            ],
+            "version": "==21.0.0"
+        },
         "idna": {
             "hashes": [
                 "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226",
@@ -236,11 +367,11 @@
         },
         "imap-tools": {
             "hashes": [
-                "sha256:7d2d25b35117a3750c3b561dd93cc2fcb24cdc457830a049796c639f4371e317",
-                "sha256:80088839cd1959f20c44206cdad4463ca1e7647ff67cf5b0e31e810fb6aaa6c4"
+                "sha256:4ec9575f21fb5a3efb573e71107806367dd52ecab0bf8f5c3724d1de6762b77b",
+                "sha256:860221dc13eb5d640da240e9bcf7867275c6162b742255789fd84f59124dfdd9"
             ],
             "index": "pypi",
-            "version": "==0.34.0"
+            "version": "==0.36.0"
         },
         "img2pdf": {
             "hashes": [
@@ -257,6 +388,13 @@
             "markers": "python_version < '3.8'",
             "version": "==3.4.0"
         },
+        "incremental": {
+            "hashes": [
+                "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
+                "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3"
+            ],
+            "version": "==17.5.0"
+        },
         "inotify-simple": {
             "hashes": [
                 "sha256:8440ffe49c4ae81a8df57c1ae1eb4b6bfa7acb830099bfb3e305b383005cc128",
@@ -335,6 +473,41 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.6.2"
         },
+        "msgpack": {
+            "hashes": [
+                "sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9",
+                "sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841",
+                "sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439",
+                "sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694",
+                "sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a",
+                "sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f",
+                "sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e",
+                "sha256:7307e86f7ce75b49e65b55660b10b258e9e7b5e0f80d31d7a86a278d8204d1b4",
+                "sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1",
+                "sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c",
+                "sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b",
+                "sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759",
+                "sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326",
+                "sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc",
+                "sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192",
+                "sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83",
+                "sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06",
+                "sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e",
+                "sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9",
+                "sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33",
+                "sha256:c82dc0ba34d620fb94d12a7725e9362958bb1be3938688a061f53ed86bee005a",
+                "sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54",
+                "sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f",
+                "sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887",
+                "sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009",
+                "sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2",
+                "sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c",
+                "sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87",
+                "sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984",
+                "sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6"
+            ],
+            "version": "==1.0.2"
+        },
         "numpy": {
             "hashes": [
                 "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94",
@@ -524,6 +697,42 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "pyasn1": {
+            "hashes": [
+                "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359",
+                "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576",
+                "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf",
+                "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7",
+                "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d",
+                "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00",
+                "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8",
+                "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86",
+                "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12",
+                "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776",
+                "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba",
+                "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2",
+                "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"
+            ],
+            "version": "==0.4.8"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8",
+                "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199",
+                "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811",
+                "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed",
+                "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4",
+                "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e",
+                "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74",
+                "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb",
+                "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45",
+                "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd",
+                "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0",
+                "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d",
+                "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"
+            ],
+            "version": "==0.2.8"
+        },
         "pycparser": {
             "hashes": [
                 "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
@@ -532,6 +741,21 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.20"
         },
+        "pyhamcrest": {
+            "hashes": [
+                "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316",
+                "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.2"
+        },
+        "pyopenssl": {
+            "hashes": [
+                "sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51",
+                "sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b"
+            ],
+            "version": "==20.0.1"
+        },
         "python-dateutil": {
             "hashes": [
                 "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
@@ -558,13 +782,12 @@
         },
         "python-levenshtein": {
             "hashes": [
-                "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1",
-                "sha256:15e26882728c29ccdf74cfc6ac4b49fc22c08b44d152348cb0eb1ec4f3dbf9df",
-                "sha256:3df5e5eb144570ecf5ad38864a2393068798328c7f05e7b167a49391d36a2db1",
-                "sha256:7f049b3ddc4b525bd469febafb98bf5202f789b722e0e4ccbec2ffbe8c07d7b4"
+                "sha256:108edd3c271f1afda8b21a8d9da81886414dfb6940a085fa7903c592e0f8f54b",
+                "sha256:1ff19b712c5974080b003fd26ef365cd93dfc1a5e690be621f79f3e63e00a7cc",
+                "sha256:554e273a88060d177e7b3c1e6ea9158dde11563bfae8f7f661f73f47e5ff0911"
             ],
             "index": "pypi",
-            "version": "==0.12.0"
+            "version": "==0.12.1"
         },
         "python-magic": {
             "hashes": [
@@ -640,50 +863,50 @@
         },
         "reportlab": {
             "hashes": [
-                "sha256:009fa61710647cdc62eb373345248d8ebb93583a058990f7c4f9be46d90aa5b1",
-                "sha256:04a08d284da86882ec3a41a7c719833362ef891b09ee8e2fbb47cee352aa684a",
-                "sha256:07bff6742fba612da8d1b1f783c436338c6fdc6962828159827d5ca7d2b67935",
-                "sha256:09fb11ab1500e679fc1b01199d2fed24435499856e75043a9ac0d31dd48fd881",
-                "sha256:18a876449c9000c391dd3415ebc8454cd7bb9e488977b894886a2d7d018f16cd",
-                "sha256:18eec161411026dde49767bee4e5e8eeb8014879554811a62581dc7433628d5b",
-                "sha256:19353aead39fc115a4d6c598d6fb9fa26da7e69160a0443ebb49b02903e704e8",
-                "sha256:1b85c20e89c22ae902ca973df2afdd2d64d27dc4ffd2b29ebad8c805a213756b",
-                "sha256:1da3d7a35f918cee905facfa94bd00ae6091cadc06dca1b0b31b69ae02d41d1d",
-                "sha256:1e484ce83dae26cb40fcbd312d45b8ba921de7856a00339d867dd4ecf145a1e7",
-                "sha256:33f3cfdc492575f8af3225701301a7e62fc478358729820c9e0091aff5831378",
-                "sha256:3b0026c1129147befd4e5a8cf25da8dea1096fce371e7b2412e36d7254019c06",
-                "sha256:3d7713dddaa8081ed709a1fa2456a43f6a74b0f07d605da8441fd53fef334f69",
-                "sha256:3e2b4d69763103b9dc9b54c0952dc3cee05cedd06e28c0987fad7f84705b12c0",
-                "sha256:4ca5233a19a5ceca23546290f43addec2345789c7d65bb32f8b2668aa148351f",
-                "sha256:5214a289cf01ebbd65e49bae83709671dd9edb601891cf0ae8abf85f3c0b392f",
-                "sha256:52f8237654acbc78ea2fa6fb4a6a06e5b023b6da93f7889adfe2deba09473fad",
-                "sha256:5ed00894e0f8281c0b7c0494b4d3067c641fd90c8e5cf933089ec4cc9a48e491",
-                "sha256:6191961533d49c9d860964d42bada4d7ac3bb28502d984feb8034093f2012fa8",
-                "sha256:6f3ad2b1afe99c436563cd436d8693d4a12e2c4bd45f70c7705759ff7837fe53",
-                "sha256:739b743b7ca1ba4b4d64c321de6fccb49b562d0507ea06c817d9cc4faed5cd22",
-                "sha256:792efba0c0c6e4ee94f6dc95f305451733ee9230a1c7d51cb8e5301a549e0dfb",
-                "sha256:79d63ca40231ca3860859b39a92daa5219035ba9553da89a5e1b218550744121",
-                "sha256:83b28104edd58ad65748d2d0e60e0d97e3b91b3e90b4573ea6fe60de6811972c",
-                "sha256:85650446538cd2f606ca234634142a7ccd74cb6db7cfec250f76a4242e0f2431",
-                "sha256:8850eba6de6eb813036eb8dce353e40d60c8af48bbce107de82770b10d3aa525",
-                "sha256:9da445cb79e3f740756924c053edc952cde11a65ff5af8acfda3c0a1317136ef",
-                "sha256:9fabd5fbd24f5971085ffe53150d663f158f7d3050b25c95736e29ebf676d454",
-                "sha256:a0c377bc45e73c3f15f55d7de69fab270d174749d5b454ab0de502b15430ec2a",
-                "sha256:a1d3f7022a920d4a5e165d264581f1862e1c1b877ceeabb96fe98cec98125ae5",
-                "sha256:a315edef5c5610b0c75790142f49487e89ea34397fc247ae8aa890fe6d6dd057",
-                "sha256:a755cca2dcf023130b03bb671670301a992157d5c3151d838c0b68ef89894536",
-                "sha256:b1b20208ecdfffd7ca027955c4fe8972b28b30a4b3b80cf25099a08d3b20ed7c",
-                "sha256:b26d6f416891cef93411d6d478a25db275766081a5fb66368248293ef459f3be",
-                "sha256:b4ba4c30af7044ee987e61c88a5ffb76031ca0c53666bc85d823b7de55ddbc75",
-                "sha256:b71faf3b6e4d7058e1af1b8afedaf39a962db4a219affc8177009d8244ec10d4",
-                "sha256:cfa854bea525f8c913cb77e2bda724d94b965a0eb3bcfc4a645a9baa29bb86e2",
-                "sha256:dd9687359e466086b9f6fe6d8069034017f8b6ca3080944fae5709767ca6814e",
-                "sha256:de0c675fc2998a7eaa929c356ba49c84f53a892e9ab25e8ee7d8ebbbdcb2ac16",
-                "sha256:e2b4e33fea2ce9d3a14ea39191b169e41eb2ac995274f54ac8fd27519974bce8",
-                "sha256:f3d4a1a273dc141e03b72a553c11bc14dd7a27ec7654a071edcf83eb04f004bc",
-                "sha256:ff547cf4c1de7e104cad1a378431ff81efcb03e90e40871ee686107da5b91442"
+                "sha256:10d0cd2ab669fbad8b766db57066270296396caf515731643e7c7e732159a432",
+                "sha256:21508065492fbb750aa6d07297a2ba4bcacb418c84898fe892e12bdceefc3b9a",
+                "sha256:21c433905569af36bc220490a33f8f78442fd71c1bea3d021f851623c8089a78",
+                "sha256:236c4142474a59e4564b0e7c32e4d7b9cfc0dc2236ffa721ae97a4dbc2667d10",
+                "sha256:287b241a9d4189e7bfffd68aea0efd701d703bccee6686b9779bf16e22e9a2ad",
+                "sha256:2bc48a3cad6dd3cd1b5cd09526dad50afbda14afadd6fb40635a5227246bbe26",
+                "sha256:2e4631c49ec72f6a84502cda1710e988fa3027250e17c26713e543af31fff58a",
+                "sha256:2fd3305a75502d11942a9629cf6af96660a508f19adc4abaac5549909d1db1b3",
+                "sha256:32be7f9a55b8dfd91924730644632d19352fd9e0bb77cd03eba8a34fc846c4f6",
+                "sha256:33dc663ac3713e9d2f5a96b20412e16fd00b688b1b5a8057436e3fce69c2a59d",
+                "sha256:3d83cffa1211a7ae4912bfe3ac36051e31371cfb63b716e458c520da9f645fd9",
+                "sha256:3f9d56f5778b54260ce79878761b1f98267677702233daa26edd9f06f3be85c0",
+                "sha256:49a599f1aae7d91649fb10dbe070614740a60e4d2b65882a002dc20a7eba9f6f",
+                "sha256:4d5d62d49f3632cf437ac28ad52a8274fe6392ea2c48d294d2e92f1258d1479e",
+                "sha256:552ea83656f18e52ee073aee04d7e3f4d0eb71147f6882ae4fc15681d8e6ab1c",
+                "sha256:605f4a49ee313eabae4db4694d073dbb0778a0eccbc6d090e58c5a4db862c2a8",
+                "sha256:6289057afa4023f58d7977d105d09503c2cb1031e248fff52b44e41914d176cf",
+                "sha256:669526b580d52c05efbd45a8c9f1adf9370b8818bc2d48adec661ea7037415d4",
+                "sha256:6fc5f21d1383b552dc5b3eb774aaaf14cb7f8f34f15ea45772f72867dfaeecc6",
+                "sha256:74c7499fdda682028bb1b01d7ca89d813175c0f8c18d3ae9924c6393987ea4ef",
+                "sha256:75c0b6cf47c199f1e69283b0678b47537cc15af31acbad783fffe2c9cfa6a626",
+                "sha256:7b3ec1255c940fabfdb05577a4f148cca54943215cecb24efd5840a8ecb618ce",
+                "sha256:7d39eaca932b1d51de84c387616462913fa988c02a4a462798098f62edcf1e21",
+                "sha256:81e2faddfde7a21154d1a08e28e3d9b820274afa282b7ed17fcf1e9d7b4aa7a1",
+                "sha256:8aca938409fc99a9e79514cb97b3478780b261ff74a058c75a3a12552b1d6b9a",
+                "sha256:8f963222cc0c302ed5a15766ea00ff470c80c47d691e828ef21032df95e8a2ff",
+                "sha256:9be1ec218966e2e9982e73109863d91e3e4b9e7649ae6be0bf98b78c52a5a330",
+                "sha256:aa24f56c504d41982eb212f79669c05d4452f7736618dd33ba4167da0b5578b7",
+                "sha256:aa9caa395823130b360b5f3184eb25c9f1ea4fb51ab0370c2e693dc139cd7d83",
+                "sha256:b2f2bf419229bda5d6c8a10219ad1d984d2a6cd246acf613559d4a5fbcee0aea",
+                "sha256:b73f85e09e4271aee192489f5a8a7c33a1a82f42702cd0886f8f630600a23ad8",
+                "sha256:c3fac0186c18d7d5285f508d79bd86f751526190728ef2a33107568533f9aff0",
+                "sha256:c7ea886aa8dc3d3adb0c32ee87522821dd8ead2bac6d41e0481c1d911049c3c9",
+                "sha256:c9a487b646ac367d15bf6749deffa04ecbe339ba067d29414f206a7516a0a775",
+                "sha256:d0ae3cd60b659da41bb4962facec2ed3c8cfb3cedc85c5993e6299938134fc94",
+                "sha256:d6b900b1dc0d3c7b7f69fde25c8e60cb978b774056383c4a03bb38c829342299",
+                "sha256:d6feecad778e2a3d5a5ea48b027063e9baf73c6454235c1c1556b59c114c90cb",
+                "sha256:de5d98f41ffbe567717c334a9866c0f289a02cef37a78323cb5f8a86eecb6a8c",
+                "sha256:e47daf974b68fdcd2ffb5442ea5f1dd849eeaf6f9d4915128bf5374467e7b63a",
+                "sha256:e52536cba231fb3dd0628abf98b23519778e1247d4d69553d879eea6159b648d",
+                "sha256:f3181284502207e78eb472d00e8f9e84f43a7fe502821d2d76f3dd7da1d5e081",
+                "sha256:fc900dbc8f020305e15781210e64f6e96c1f9ca0fa00c4a0ccc1174cc44c5a5b"
             ],
-            "version": "==3.5.59"
+            "version": "==3.5.60"
         },
         "requests": {
             "hashes": [
@@ -695,35 +918,31 @@
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:076369634ee72b5a5941440661e2f306ff4ac30903802dc52031c7e9199ac640",
-                "sha256:18f7131e62265bf2691ed1d0303c640313894ccfe4278427478c6b2f45094b53",
-                "sha256:26f66b3726b54dfb76ea51c5d9c2431ed17ebc066cb4527662b9e851a3e7ba61",
-                "sha256:2951f87d35e72f007701c6e028aa230f6df6212a3194677c0c950486066a454d",
-                "sha256:2a5348585aa793bc8cc5a72f8e9067c9380834b0aadbd55f924843b071f13282",
-                "sha256:3eeff086f7329521d27249a082ea3c48c085cedb110db5f65968ab55c3ba2e09",
-                "sha256:4395e91b3548005f4a645018435b5a94f8cce232b5b70753020e606c6a750656",
-                "sha256:44e452ea8491225c5783d49577aad0f36202dfd52aec7f82c0fdfe5fbd5f7400",
-                "sha256:490436b44b3a1957cb625e871764b0aa330b34cc416aea4abc6c38ca63d0d682",
-                "sha256:5e6e3c042cea83f2e20a45e563b8eabc1f8f72446251fe23ebefdf111a173a33",
-                "sha256:66f27bf21202a850bcd7b6303916e4907f6e22ec59a14974ede4955aed5c7ed0",
-                "sha256:743b6edd98c98991be46c08e6b21df3861d5ae915f91d59f988384d93f7263e7",
-                "sha256:758619e49cd7c17282e6cc60d5cc73c02c072b47c9a10010bb3bb47e0d976e50",
-                "sha256:7f654befc5ad413690cc58f3f34a3e906caf825195ce0fda00a8e9565e1403e6",
-                "sha256:800aaf63f8838c00e85db2267dd226f89858594843fd03932a9eda95746d2c40",
-                "sha256:80ca024154b84b6ac4cfc86930ba13fdc348a209753bf2c16129db6f9eb8a80b",
-                "sha256:890d7d588f65acb0c4f6c083347c9076916bda5e6bd8400f06244b1afc1009af",
-                "sha256:905d8934d1e27a686698864a5863ff2c0e13a2ae1adb78a8a848aacc8a49927d",
-                "sha256:a83fcd9d59c42a2f66b307e3b0b0f08aa8e6e45be33da055697ea499f0e4f7c2",
-                "sha256:afeb06dc69847927634e58579b9cdc72e1390b79497336b2324b1b173f33bd47",
-                "sha256:b0d13fd56d26cf3de0314a4fd48037108c638fe126d813f5c1222bb0f08b6a76",
-                "sha256:c08b27cb78ee8d2dc781a7affed09859441f5b624f9f92da59ac0791c8774dfc",
-                "sha256:c912247e42114f389858ae05d63f4359d4e667ea72aaabee191aee9ad3f9774a",
-                "sha256:d7fe05fcb44eadd6d6c874c768f085f5de1239db3a3b7be4d3d23d12e4120589",
-                "sha256:d819d625832fb2969911a243e009cfa135cb8ef1e150866e417d6e9d75290087",
-                "sha256:e534f5f3796db6781c87e9835dcd51b7854c8c5a379c9210b93605965c1941fd"
+                "sha256:1adf483e91007a87171d7ce58c34b058eb5dab01b5fee6052f15841778a8ecd8",
+                "sha256:259ec35201e82e2db1ae2496f229e63f46d7f1695ae68eef9350b00dc74ba52f",
+                "sha256:3c4f07f47c04e81b134424d53c3f5e16dfd7f494e44fd7584ba9ce9de2c5e6c1",
+                "sha256:4562dcf4793e61c5d0f89836d07bc37521c3a1889da8f651e2c326463c4bd697",
+                "sha256:7b04691eb2f41d2c68dbda8d1bd3cb4ef421bdc43aaa56aeb6c762224552dfb6",
+                "sha256:826b92bf45b8ad80444814e5f4ac032156dd481e48d7da33d611f8fe96d5f08b",
+                "sha256:83b21ff053b1ff1c018a2d24db6dd3ea339b1acfbaa4d9c881731f43748d8b3b",
+                "sha256:8772b99d683be8f67fcc04789032f1b949022a0e6880ee7b75a7ec97dbbb5d0b",
+                "sha256:895dbf2030aa7337649e36a83a007df3c9811396b4e2fa672a851160f36ce90c",
+                "sha256:8aa1b3ac46b80eaa552b637eeadbbce3be5931e4b5002b964698e33a1b589e1e",
+                "sha256:99349d77f54e11f962d608d94dfda08f0c9e5720d97132233ebdf35be2858b2d",
+                "sha256:9a24d1ccec2a34d4cd3f2a1f86409f3f5954cc23d4d2270ba0d03cf018aa4780",
+                "sha256:9bed8a1ef133c8e2f13966a542cb8125eac7f4b67dcd234197c827ba9c7dd3e0",
+                "sha256:9c6097b6a9b2bafc5e0f31f659e6ab5e131383209c30c9e978c5b8abdac5ed2a",
+                "sha256:9dfa564ef27e8e674aa1cc74378416d580ac4ede1136c13dd555a87996e13422",
+                "sha256:a0334a1802e64d656022c3bfab56a73fbd6bf4b1298343f3688af2151810bbdf",
+                "sha256:ae7191f0253145503fbfa1934cbc83f48260f534343e34115398fb5fc2be55b0",
+                "sha256:c3deb3b19dd9806acf00cf0d400e84562c227723013c33abefbbc3cf906596e9",
+                "sha256:d54dbaadeb1425b7d6a66bf44bee2bb2b899fe3e8850b8e94cfb9c904dcb46d0",
+                "sha256:ddb52d088889f5596bc4d1de981f2eca106b58243b6679e4782f3ba5096fd645",
+                "sha256:ed9d65594948678827f4ff0e7ae23344e2f2b4cabbca057ccaed3118fdc392ca",
+                "sha256:fab31f48282ebf54dd69f6663cd2d9800096bad1bb67bbc9c9ac84eb77b41972"
             ],
             "index": "pypi",
-            "version": "==0.24.0"
+            "version": "==0.24.1"
         },
         "scipy": {
             "hashes": [
@@ -757,6 +976,13 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.5.4"
         },
+        "service-identity": {
+            "hashes": [
+                "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36",
+                "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d"
+            ],
+            "version": "==18.1.0"
+        },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
@@ -804,6 +1030,48 @@
             "index": "pypi",
             "version": "==4.56.0"
         },
+        "twisted": {
+            "extras": [
+                "tls"
+            ],
+            "hashes": [
+                "sha256:0150dae5adc962d15e00054cc6926f1e64763fb8dd26e1632593ac06e592104b",
+                "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f",
+                "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042",
+                "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c",
+                "sha256:15e52271f08f62e2230ff093e0278aa01c9dac057c4557cadadd2429eed86a3e",
+                "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292",
+                "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22",
+                "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec",
+                "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478",
+                "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2",
+                "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29",
+                "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114",
+                "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797",
+                "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa",
+                "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15",
+                "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd",
+                "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274",
+                "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad",
+                "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7",
+                "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a",
+                "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10",
+                "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780",
+                "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504",
+                "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467",
+                "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==20.3.0"
+        },
+        "txaio": {
+            "hashes": [
+                "sha256:1488d31d564a116538cc1265ac3f7979fb6223bb5a9e9f1479436ee2c17d8549",
+                "sha256:a8676d6c68aea1f0e2548c4afdb8e6253873af3bc2659bb5bcd9f39dff7ff90f"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==20.12.1"
+        },
         "typing-extensions": {
             "hashes": [
                 "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
@@ -882,6 +1150,66 @@
             ],
             "markers": "python_version >= '3.6'",
             "version": "==3.4.0"
+        },
+        "zope.interface": {
+            "hashes": [
+                "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
+                "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d",
+                "sha256:09fc3922f235703c0b76f8234867685eee68a24a49fffa2220975f6142db45f1",
+                "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123",
+                "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232",
+                "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549",
+                "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102",
+                "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5",
+                "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45",
+                "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00",
+                "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc",
+                "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7",
+                "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104",
+                "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034",
+                "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3",
+                "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3",
+                "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4",
+                "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86",
+                "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96",
+                "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546",
+                "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb",
+                "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3",
+                "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b",
+                "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b",
+                "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec",
+                "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae",
+                "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e",
+                "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386",
+                "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2",
+                "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a",
+                "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d",
+                "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a",
+                "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24",
+                "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d",
+                "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b",
+                "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50",
+                "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523",
+                "sha256:974f5957e66a7524ea81df7b2686a456bfaf0408dbb7353ddfbedb594eadfef6",
+                "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a",
+                "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095",
+                "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a",
+                "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520",
+                "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65",
+                "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11",
+                "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c",
+                "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7",
+                "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332",
+                "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e",
+                "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c",
+                "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7",
+                "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20",
+                "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc",
+                "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd",
+                "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==5.2.0"
         }
     },
     "develop": {
@@ -1043,11 +1371,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:47ac7d62d5bad8c16422a91f121430ab7656d40ca8fea9c84bcdbdf92e739b03",
-                "sha256:6bc44606d44f711e1d89ad9a5b42394cc6f7eedaffc765ddb5b2d22084c15733"
+                "sha256:0f8198d876bc65edd3699e2542c6ae13cc93fbc4a478c17db190d5d9b6ce790e",
+                "sha256:8fe22be90c0db35d830464918f84beb50f970f4caacae4d88e92692857b78aa7"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==5.5.0"
+            "version": "==5.6.5"
         },
         "filelock": {
             "hashes": [
@@ -1084,11 +1412,11 @@
         },
         "importlib-resources": {
             "hashes": [
-                "sha256:4743f090ed8946e713745ec0e660249ef9fb0b9843eacc5b5ff931d2fd5aa67f",
-                "sha256:ea17df80a0ff04b5dbd3d96dbeab1842acfd1c6c902eaeb8c8858abf2720161e"
+                "sha256:885b8eae589179f661c909d699a546cf10d83692553e34dca1bf5eb06f7f6217",
+                "sha256:bfdad047bce441405a49cf8eb48ddce5e56c696e185f59147a8b79e75e9e6380"
             ],
             "markers": "python_version < '3.7'",
-            "version": "==5.0.0"
+            "version": "==5.1.0"
         },
         "iniconfig": {
             "hashes": [
@@ -1207,11 +1535,11 @@
         },
         "pytest-cov": {
             "hashes": [
-                "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191",
-                "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e"
+                "sha256:359952d9d39b9f822d9d29324483e7ba04a3a17dd7d05aa6beb7ea01e359e5f7",
+                "sha256:bdb9fdb0b85a7cc825269a4c56b48ccaa5c7e365054b6038772c32ddcdc969da"
             ],
             "index": "pypi",
-            "version": "==2.10.1"
+            "version": "==2.11.1"
         },
         "pytest-django": {
             "hashes": [
@@ -1287,10 +1615,10 @@
         },
         "snowballstemmer": {
             "hashes": [
-                "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
-                "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
+                "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2",
+                "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914"
             ],
-            "version": "==2.0.0"
+            "version": "==2.1.0"
         },
         "sphinx": {
             "hashes": [
@@ -1380,11 +1708,11 @@
         },
         "tox": {
             "hashes": [
-                "sha256:5efda30ad73e662c3844ac51ce1381bf28f61063773e06996aa8b6277133a7c0",
-                "sha256:8cccede64802e78aa6c69f81051b25f0706639d1cbbb34d9366ce00c70ee054f"
+                "sha256:0aa777ee466f2ef18e6f58428c793c32378779e0a321dbb8934848bc3e78998c",
+                "sha256:f501808381c01c6d7827c2f17328be59c0a715046e94605ddca15fb91e65827d"
             ],
             "index": "pypi",
-            "version": "==3.21.0"
+            "version": "==3.21.2"
         },
         "typing-extensions": {
             "hashes": [
@@ -1405,11 +1733,11 @@
         },
         "virtualenv": {
             "hashes": [
-                "sha256:205a7577275dd0d9223c730dd498e21a8910600085c3dee97412b041fc4b853b",
-                "sha256:7992b8de87e544a4ab55afc2240bf8388c4e3b5765d03784dad384bfdf9097ee"
+                "sha256:219ee956e38b08e32d5639289aaa5bd190cfbe7dafcb8fa65407fca08e808f9c",
+                "sha256:227a8fed626f2f20a6cdb0870054989f82dd27b2560a911935ba905a2a5e0034"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.3.0"
+            "version": "==20.4.0"
         },
         "zipp": {
             "hashes": [

From f31073a341f4227dfb7ad40df2051403ac5721d2 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 00:50:29 +0100
Subject: [PATCH 15/54] update dependencies

---
 Pipfile          |  1 -
 Pipfile.lock     | 36 ++++++++++++++++++------------------
 requirements.txt | 32 ++++++++++++++++++++++++++------
 3 files changed, 44 insertions(+), 25 deletions(-)

diff --git a/Pipfile b/Pipfile
index adb41f314..140df6d02 100644
--- a/Pipfile
+++ b/Pipfile
@@ -18,7 +18,6 @@ django-q = "~=1.3.4"
 djangorestframework = "~=3.12.2"
 filelock = "*"
 fuzzywuzzy = "*"
-gunicorn = "*"
 imap-tools = "*"
 langdetect = "*"
 pdftotext = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 267b18533..9c30dedfa 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "08cd30c074677e99d7b6bada67b7c6b7bb5348a9210753ea3c7841c95cad1037"
+            "sha256": "ede35846cb5a19613a24fa1ad4ba0e0ea0290d45fc912e7e69136bc56bb64007"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -217,11 +217,11 @@
         },
         "django-cors-headers": {
             "hashes": [
-                "sha256:5665fc1b1aabf1b678885cf6f8f8bd7da36ef0a978375e767d491b48d3055d8f",
-                "sha256:ba898dd478cd4be3a38ebc3d8729fa4d044679f8c91b2684edee41129d7e968a"
+                "sha256:1ac2b1213de75a251e2ba04448da15f99bcfcbe164288ae6b5ff929dc49b372f",
+                "sha256:96069c4aaacace786a34ee7894ff680780ec2644e4268b31181044410fecd12e"
             ],
             "index": "pypi",
-            "version": "==3.6.0"
+            "version": "==3.7.0"
         },
         "django-extensions": {
             "hashes": [
@@ -279,14 +279,6 @@
             "index": "pypi",
             "version": "==0.18.0"
         },
-        "gunicorn": {
-            "hashes": [
-                "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
-                "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
-            ],
-            "index": "pypi",
-            "version": "==20.0.4"
-        },
         "hiredis": {
             "hashes": [
                 "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
@@ -918,10 +910,13 @@
         },
         "scikit-learn": {
             "hashes": [
+                "sha256:0567a2d29ad08af98653300c623bd8477b448fe66ced7198bef4ed195925f082",
+                "sha256:087dfede39efb06ab30618f9ab55a0397f29c38d63cd0ab88d12b500b7d65fd7",
                 "sha256:1adf483e91007a87171d7ce58c34b058eb5dab01b5fee6052f15841778a8ecd8",
                 "sha256:259ec35201e82e2db1ae2496f229e63f46d7f1695ae68eef9350b00dc74ba52f",
                 "sha256:3c4f07f47c04e81b134424d53c3f5e16dfd7f494e44fd7584ba9ce9de2c5e6c1",
                 "sha256:4562dcf4793e61c5d0f89836d07bc37521c3a1889da8f651e2c326463c4bd697",
+                "sha256:4ddd2b6f7449a5d539ff754fa92d75da22de261fd8fdcfb3596799fadf255101",
                 "sha256:7b04691eb2f41d2c68dbda8d1bd3cb4ef421bdc43aaa56aeb6c762224552dfb6",
                 "sha256:826b92bf45b8ad80444814e5f4ac032156dd481e48d7da33d611f8fe96d5f08b",
                 "sha256:83b21ff053b1ff1c018a2d24db6dd3ea339b1acfbaa4d9c881731f43748d8b3b",
@@ -934,8 +929,13 @@
                 "sha256:9c6097b6a9b2bafc5e0f31f659e6ab5e131383209c30c9e978c5b8abdac5ed2a",
                 "sha256:9dfa564ef27e8e674aa1cc74378416d580ac4ede1136c13dd555a87996e13422",
                 "sha256:a0334a1802e64d656022c3bfab56a73fbd6bf4b1298343f3688af2151810bbdf",
+                "sha256:a29460499c1e62b7a830bb57ca42e615375a6ab1bcad053cd25b493588348ea8",
+                "sha256:a36e159a0521e13bbe15ca8c8d038b3a1dd4c7dad18d276d76992e03b92cf643",
                 "sha256:ae7191f0253145503fbfa1934cbc83f48260f534343e34115398fb5fc2be55b0",
+                "sha256:c13ebac42236b1c46397162471ea1c46af68413000e28b9309f8c05722c65a09",
                 "sha256:c3deb3b19dd9806acf00cf0d400e84562c227723013c33abefbbc3cf906596e9",
+                "sha256:c658432d8a20e95398f6bb95ff9731ce9dfa343fdf21eea7ec6a7edfacd4b4d9",
+                "sha256:c7f4eb77504ac586d8ac1bde1b0c04b504487210f95297235311a0ab7edd7e38",
                 "sha256:d54dbaadeb1425b7d6a66bf44bee2bb2b899fe3e8850b8e94cfb9c904dcb46d0",
                 "sha256:ddb52d088889f5596bc4d1de981f2eca106b58243b6679e4782f3ba5096fd645",
                 "sha256:ed9d65594948678827f4ff0e7ae23344e2f2b4cabbca057ccaed3118fdc392ca",
@@ -1371,11 +1371,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:0f8198d876bc65edd3699e2542c6ae13cc93fbc4a478c17db190d5d9b6ce790e",
-                "sha256:8fe22be90c0db35d830464918f84beb50f970f4caacae4d88e92692857b78aa7"
+                "sha256:1a40c9696830342e67d30057753323478b848e89334e97e0a64b442b18ae7ece",
+                "sha256:dd98418eceb374c475e85bd49ae826a9b304dabcd794640f35a1d185f4767dd9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==5.6.5"
+            "version": "==5.7.0"
         },
         "filelock": {
             "hashes": [
@@ -1527,11 +1527,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8",
-                "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"
+                "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9",
+                "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839"
             ],
             "index": "pypi",
-            "version": "==6.2.1"
+            "version": "==6.2.2"
         },
         "pytest-cov": {
             "hashes": [
diff --git a/requirements.txt b/requirements.txt
index 3b7a158c1..c70d13ff4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,16 +7,25 @@
 
 -i https://pypi.python.org/simple
 --extra-index-url https://www.piwheels.org/simple
+aioredis==1.3.1
 arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 asgiref==3.3.1; python_version >= '3.5'
+async-timeout==3.0.1; python_full_version >= '3.5.3'
+attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+autobahn==20.12.3; python_version >= '3.6'
+automat==20.2.0
 blessed==1.17.12
 certifi==2020.12.5
 cffi==1.14.4
+channels-redis==3.2.0
+channels==3.0.3
 chardet==4.0.0; python_version >= '3.1'
 coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+constantly==15.1.0
 cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+daphne==3.0.1
 dateparser==0.7.6
-django-cors-headers==3.6.0
+django-cors-headers==3.7.0
 django-extensions==3.1.0
 django-filter==2.4.0
 django-picklefield==3.0.1; python_version >= '3'
@@ -25,17 +34,20 @@ django==3.1.5
 djangorestframework==3.12.2
 filelock==3.0.12
 fuzzywuzzy==0.18.0
-gunicorn==20.0.4
+hiredis==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+hyperlink==21.0.0
 idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-imap-tools==0.34.0
+imap-tools==0.36.0
 img2pdf==0.4.0
 importlib-metadata==3.4.0; python_version < '3.8'
+incremental==17.5.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
 joblib==1.0.0; python_version >= '3.6'
 langdetect==1.0.8
 lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+msgpack==1.0.2
 numpy==1.19.5; python_version >= '3.6'
 ocrmypdf==11.4.5
 pathvalidate==2.3.2
@@ -45,25 +57,32 @@ pikepdf==2.2.5
 pillow==8.1.0
 pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 psycopg2-binary==2.8.6
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pyhamcrest==2.0.2; python_version >= '3.5'
+pyopenssl==20.0.1
 python-dateutil==2.8.1
 python-dotenv==0.15.0
 python-gnupg==0.4.6
-python-levenshtein==0.12.0
+python-levenshtein==0.12.1
 python-magic==0.4.18
 pytz==2020.5
 redis==3.5.3
 regex==2020.11.13
-reportlab==3.5.59
+reportlab==3.5.60
 requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-scikit-learn==0.24.0
+scikit-learn==0.24.1
 scipy==1.5.4; python_version >= '3.6'
+service-identity==18.1.0
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sortedcontainers==2.3.0
 sqlparse==0.4.1; python_version >= '3.5'
 threadpoolctl==2.1.0; python_version >= '3.5'
 tika==1.24
 tqdm==4.56.0
+twisted[tls]==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+txaio==20.12.1; python_version >= '3.6'
 typing-extensions==3.7.4.3; python_version < '3.8'
 tzlocal==2.1
 urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
@@ -72,3 +91,4 @@ wcwidth==0.2.5
 whitenoise==5.2.0
 whoosh==2.7.4
 zipp==3.4.0; python_version >= '3.6'
+zope.interface==5.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

From 339e96b63c369223b8134b271034eb909819a58f Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 00:51:20 +0100
Subject: [PATCH 16/54] task ids

---
 src/documents/consumer.py | 6 +++++-
 src/documents/tasks.py    | 7 +++++--
 src/documents/views.py    | 6 +++++-
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 71b1294f5..f4a40cbb9 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -35,6 +35,7 @@ class Consumer(LoggingMixin):
                        message, document_id=None):
         payload = {
             'filename': os.path.basename(filename),
+            'task_id': self.task_id,
             'current_progress': current_progress,
             'max_progress': max_progress,
             'status': status,
@@ -58,6 +59,7 @@ class Consumer(LoggingMixin):
         self.override_correspondent_id = None
         self.override_tag_ids = None
         self.override_document_type_id = None
+        self.task_id = None
 
         self.channel_layer = get_channel_layer()
 
@@ -136,7 +138,8 @@ class Consumer(LoggingMixin):
                          override_title=None,
                          override_correspondent_id=None,
                          override_document_type_id=None,
-                         override_tag_ids=None):
+                         override_tag_ids=None,
+                         task_id=None):
         """
         Return the document object if it was successfully created.
         """
@@ -147,6 +150,7 @@ class Consumer(LoggingMixin):
         self.override_correspondent_id = override_correspondent_id
         self.override_document_type_id = override_document_type_id
         self.override_tag_ids = override_tag_ids
+        self.task_id = task_id
 
         self._send_progress(self.filename, 0, 100, 'WORKING',
                             'Received new file.')
diff --git a/src/documents/tasks.py b/src/documents/tasks.py
index 38ff532b5..c67b2b3fa 100644
--- a/src/documents/tasks.py
+++ b/src/documents/tasks.py
@@ -62,7 +62,8 @@ def consume_file(path,
                  override_title=None,
                  override_correspondent_id=None,
                  override_document_type_id=None,
-                 override_tag_ids=None):
+                 override_tag_ids=None,
+                 task_id=None):
 
     document = Consumer().try_consume_file(
         path,
@@ -70,7 +71,9 @@ def consume_file(path,
         override_title=override_title,
         override_correspondent_id=override_correspondent_id,
         override_document_type_id=override_document_type_id,
-        override_tag_ids=override_tag_ids)
+        override_tag_ids=override_tag_ids,
+        task_id=task_id
+    )
 
     if document:
         return "Success. New document id {} created".format(
diff --git a/src/documents/views.py b/src/documents/views.py
index f0c7f10d1..2012e879a 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -1,6 +1,7 @@
 import logging
 import os
 import tempfile
+import uuid
 from datetime import datetime
 from time import mktime
 
@@ -385,6 +386,8 @@ class PostDocumentView(APIView):
             f.write(doc_data)
             os.utime(f.name, times=(t, t))
 
+            task_id = str(uuid.uuid4())
+
             async_task("documents.tasks.consume_file",
                        f.name,
                        override_filename=doc_name,
@@ -392,8 +395,9 @@ class PostDocumentView(APIView):
                        override_correspondent_id=correspondent_id,
                        override_document_type_id=document_type_id,
                        override_tag_ids=tag_ids,
+                       task_id=task_id,
                        task_name=os.path.basename(doc_name)[:100])
-        return Response("OK")
+        return Response({"task_id": task_id})
 
 
 class SelectionDataView(APIView):

From ede3bd1391804d54d0b25e9b45b6fa446a651fef Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 00:51:45 +0100
Subject: [PATCH 17/54] rework of the front end components

---
 src-ui/src/app/app.component.ts               |  4 +-
 .../upload-file-widget.component.html         | 12 ++-
 .../upload-file-widget.component.ts           | 69 +++++++------
 .../data/websocket-consumer-status-message.ts | 11 +++
 .../app/services/consumer-status.service.ts   | 97 +++++++++++++++----
 5 files changed, 137 insertions(+), 56 deletions(-)
 create mode 100644 src-ui/src/app/data/websocket-consumer-status-message.ts

diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts
index 8ca276dd4..43c23f2af 100644
--- a/src-ui/src/app/app.component.ts
+++ b/src-ui/src/app/app.component.ts
@@ -32,7 +32,7 @@ export class AppComponent implements OnInit, OnDestroy {
 
     this.successSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
       this.toastService.show({title: "Document added", delay: 10000, content: `Document ${status.filename} was added to paperless.`, actionName: "Open document", action: () => {
-        this.router.navigate(['documents', status.document_id])
+        this.router.navigate(['documents', status.documentId])
       }})
     })
 
@@ -42,6 +42,4 @@ export class AppComponent implements OnInit, OnDestroy {
 
   }
 
-
-
 }
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index aa317bd52..59afb74f9 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -9,10 +9,16 @@
 
       </ngx-file-drop>
     </form>
-    <div *ngIf="uploadVisible" class="mt-3">
-      <p i18n>{uploadStatus.length, plural, =1 {Uploading file...} =other {Uploading {{uploadStatus.length}} files...}}</p>
-      <ngb-progressbar [value]="loadedSum" [max]="totalSum" [striped]="true" [animated]="uploadStatus.length > 0">
+    <div *ngFor="let status of getStatus()">
+      <p>{{status.filename}}: {{status.message}}</p>
+      <ngb-progressbar [value]="status.getProgress()" [max]="1" [striped]="true" [animated]="!isFinished(status)" [type]="getType(status)">
       </ngb-progressbar>
+
+      <div *ngIf="isFinished(status)" class="mb-2">
+        <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">Open document</button>
+        <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(status)">Dismiss</button>
+      </div>
+    
     </div>
   </div>
 </app-widget-frame>
\ No newline at end of file
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index 19d9909a9..ac270fe30 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -1,15 +1,10 @@
 import { HttpEventType } from '@angular/common/http';
 import { Component, OnInit } from '@angular/core';
 import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop';
+import { ConsumerStatusService, FileStatus, FileStatusPhase } from 'src/app/services/consumer-status.service';
 import { DocumentService } from 'src/app/services/rest/document.service';
-import { ToastService } from 'src/app/services/toast.service';
 
 
-interface UploadStatus {
-  loaded: number
-  total: number
-}
-
 @Component({
   selector: 'app-upload-file-widget',
   templateUrl: './upload-file-widget.component.html',
@@ -17,8 +12,35 @@ interface UploadStatus {
 })
 export class UploadFileWidgetComponent implements OnInit {
 
-  constructor(private documentService: DocumentService, private toastService: ToastService) { }
+  constructor(
+    private documentService: DocumentService,
+    private consumerStatusService: ConsumerStatusService
+  ) { }
 
+  getStatus() {
+    return this.consumerStatusService.consumerStatus
+  }
+
+  isFinished(status: FileStatus) {
+    return status.phase == FileStatusPhase.FAILED || status.phase == FileStatusPhase.SUCCESS
+  }
+
+  getType(status: FileStatus) {
+    switch (status.phase) {
+      case FileStatusPhase.PROCESSING:
+      case FileStatusPhase.UPLOADING:
+          return "primary"
+      case FileStatusPhase.FAILED:
+        return "danger"
+      case FileStatusPhase.SUCCESS:
+        return "success"
+    }
+  }
+
+  dismiss(status: FileStatus) {
+    this.consumerStatusService.dismiss(status)
+  }
+  
   ngOnInit(): void {
   }
 
@@ -28,54 +50,37 @@ export class UploadFileWidgetComponent implements OnInit {
   public fileLeave(event){
   }
 
-  uploadStatus: UploadStatus[] = []
-  completedFiles = 0
-
-  uploadVisible = false
-
-  get loadedSum() {
-    return this.uploadStatus.map(s => s.loaded).reduce((a,b) => a+b, this.completedFiles > 0 ? 1 : 0)
-  }
-
-  get totalSum() {
-    return this.uploadStatus.map(s => s.total).reduce((a,b) => a+b, 1)
-  }
-
   public dropped(files: NgxFileDropEntry[]) {
     for (const droppedFile of files) {
       if (droppedFile.fileEntry.isFile) {
-      let uploadStatusObject: UploadStatus = {loaded: 0, total: 1}
-      this.uploadStatus.push(uploadStatusObject)
-      this.uploadVisible = true
 
       const fileEntry = droppedFile.fileEntry as FileSystemFileEntry;
         fileEntry.file((file: File) => {
           let formData = new FormData()
           formData.append('document', file, file.name)
+          let status = this.consumerStatusService.newFileUpload()
+          status.filename = file.name
 
           this.documentService.uploadDocument(formData).subscribe(event => {
             if (event.type == HttpEventType.UploadProgress) {
-              uploadStatusObject.loaded = event.loaded
-              uploadStatusObject.total = event.total
+              status.updateProgress(FileStatusPhase.UPLOADING, event.loaded, event.total)
             } else if (event.type == HttpEventType.Response) {
-              this.uploadStatus.splice(this.uploadStatus.indexOf(uploadStatusObject), 1)
-              this.completedFiles += 1
-              this.toastService.showInfo($localize`The document has been uploaded and will be processed by the consumer shortly.`)
+              status.taskId = event.body["task_id"]
             }
 
           }, error => {
-            this.uploadStatus.splice(this.uploadStatus.indexOf(uploadStatusObject), 1)
-            this.completedFiles += 1
+            status.updateProgress(FileStatusPhase.FAILED)
             switch (error.status) {
               case 400: {
-                this.toastService.showInfo($localize`There was an error while uploading the document: ${error.error.document}`)
+                status.message = error.error.document
                 break;
               }
               default: {
-                this.toastService.showInfo($localize`An error has occurred while uploading the document. Sorry!`)
+                status.message = $localize`An error has occurred while uploading the document. Sorry!`
                 break;
               }
             }
+
           })
         });
       }
diff --git a/src-ui/src/app/data/websocket-consumer-status-message.ts b/src-ui/src/app/data/websocket-consumer-status-message.ts
new file mode 100644
index 000000000..49117b101
--- /dev/null
+++ b/src-ui/src/app/data/websocket-consumer-status-message.ts
@@ -0,0 +1,11 @@
+export interface WebsocketConsumerStatusMessage {
+
+  filename?: string
+  task_id?: string
+  current_progress?: number
+  max_progress?: number
+  status?: string
+  message?: string
+  document_id: number
+
+}
\ No newline at end of file
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index 070420b0f..8aaae1e93 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -1,13 +1,57 @@
 import { Injectable } from '@angular/core';
 import { Subject } from 'rxjs';
+import { WebsocketConsumerStatusMessage } from '../data/websocket-consumer-status-message';
+
+export enum FileStatusPhase {
+  STARTED = 0,
+  UPLOADING = 1,
+  PROCESSING = 2,
+  SUCCESS = 3,
+  FAILED = 4
+}
+
+export class FileStatus {
+
+  filename: string
+
+  taskId: string
+
+  phase: FileStatusPhase = FileStatusPhase.STARTED  
+
+  currentPhaseProgress: number
+
+  currentPhaseMaxProgress: number
+
+  message: string
+
+  documentId: number
+
+  getProgress(): number {
+    switch (this.phase) {
+      case FileStatusPhase.STARTED:
+        return 0.0
+      case FileStatusPhase.UPLOADING:
+        return this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.2
+      case FileStatusPhase.PROCESSING:
+        return this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.8 + 0.2
+      case FileStatusPhase.SUCCESS:
+      case FileStatusPhase.FAILED:
+        return 1.0
+    }
+  }
+
+  updateProgress(status: FileStatusPhase, currentProgress?: number, maxProgress?: number) {
+    if (status >= this.phase) {
+      this.phase = status
+      if (currentProgress) {
+        this.currentPhaseProgress = currentProgress
+      }
+      if (maxProgress) {
+        this.currentPhaseMaxProgress = maxProgress
+      }
+    }
+  }
 
-export interface FileStatus {
-  filename?: string
-  current_progress?: number
-  max_progress?: number
-  status?: string
-  message?: string
-  document_id?: number
 }
 
 @Injectable({
@@ -23,24 +67,41 @@ export class ConsumerStatusService {
   private documentConsumptionFinishedSubject = new Subject<FileStatus>()
   private documentConsumptionFailedSubject = new Subject<FileStatus>()
 
+  private get(taskId: string, filename?: string) {
+    let status = this.consumerStatus.find(e => e.taskId == taskId) || this.consumerStatus.find(e => e.filename == filename)
+    if (!status) {
+      status = new FileStatus()
+      this.consumerStatus.push(status)
+    }
+    status.taskId = taskId
+    status.filename = filename
+    return status
+  }
+
+  newFileUpload(): FileStatus {
+    let status = new FileStatus()
+    this.consumerStatus.push(status)
+    return status
+  }
+
   connect() {
     this.disconnect()
     this.statusWebSocked = new WebSocket("ws://localhost:8000/ws/status/");
     this.statusWebSocked.onmessage = (ev) => {
-      let statusUpdate: FileStatus = JSON.parse(ev['data'])
+      let statusMessage: WebsocketConsumerStatusMessage = JSON.parse(ev['data'])
 
-      let index = this.consumerStatus.findIndex(fs => fs.filename == statusUpdate.filename)
-      if (index > -1) {
-        this.consumerStatus[index] = statusUpdate
-      } else {
-        this.consumerStatus.push(statusUpdate)
-      }
+      let status = this.get(statusMessage.task_id, statusMessage.filename)
+      status.updateProgress(FileStatusPhase.PROCESSING, statusMessage.current_progress, statusMessage.max_progress)
+      status.message = statusMessage.message
+      status.documentId = statusMessage.document_id
 
-      if (statusUpdate.status == "SUCCESS") {
-        this.documentConsumptionFinishedSubject.next(statusUpdate)
+      if (statusMessage.status == "SUCCESS") {
+        status.phase = FileStatusPhase.SUCCESS
+        this.documentConsumptionFinishedSubject.next(status)
       }
-      if (statusUpdate.status == "FAILED") {
-        this.documentConsumptionFailedSubject.next(statusUpdate)
+      if (statusMessage.status == "FAILED") {
+        status.phase = FileStatusPhase.FAILED
+        this.documentConsumptionFailedSubject.next(status)
       }
     }
   }

From b581e4221645bcf1952bfed438623fce9b2f6e79 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 01:10:39 +0100
Subject: [PATCH 18/54] functions for combined upload progress bars

---
 .../upload-file-widget.component.html         |  1 +
 .../upload-file-widget.component.ts           | 25 ++++++++++++++++---
 .../app/services/consumer-status.service.ts   | 15 +++++++++--
 3 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index 59afb74f9..263918a73 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -9,6 +9,7 @@
 
       </ngx-file-drop>
     </form>
+    <p>Uploading {{getStatusUploading().length}} files...</p>
     <div *ngFor="let status of getStatus()">
       <p>{{status.filename}}: {{status.message}}</p>
       <ngb-progressbar [value]="status.getProgress()" [max]="1" [striped]="true" [animated]="!isFinished(status)" [type]="getType(status)">
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index ac270fe30..bd0e4e290 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -18,7 +18,23 @@ export class UploadFileWidgetComponent implements OnInit {
   ) { }
 
   getStatus() {
-    return this.consumerStatusService.consumerStatus
+    return this.consumerStatusService.getConsumerStatus()
+  }
+
+  getStatusUploading() {
+    return this.consumerStatusService.getConsumerStatus(FileStatusPhase.UPLOADING)
+  }
+
+  getTotalUploadProgress() {
+    let current = 0
+    let max = 0
+
+    this.getStatusUploading().forEach(status => {
+      current += status.currentPhaseProgress
+      max += status.currentPhaseMaxProgress
+    })
+
+    return current / Math.max(max, 1)
   }
 
   isFinished(status: FileStatus) {
@@ -58,14 +74,17 @@ export class UploadFileWidgetComponent implements OnInit {
         fileEntry.file((file: File) => {
           let formData = new FormData()
           formData.append('document', file, file.name)
-          let status = this.consumerStatusService.newFileUpload()
-          status.filename = file.name
+          let status = this.consumerStatusService.newFileUpload(file.name)
+          
+          status.message = "Connecting..."
 
           this.documentService.uploadDocument(formData).subscribe(event => {
             if (event.type == HttpEventType.UploadProgress) {
               status.updateProgress(FileStatusPhase.UPLOADING, event.loaded, event.total)
+              status.message = "Uploading..."
             } else if (event.type == HttpEventType.Response) {
               status.taskId = event.body["task_id"]
+              status.message = "Upload complete."
             }
 
           }, error => {
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index 8aaae1e93..69e58e2dd 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -63,7 +63,9 @@ export class ConsumerStatusService {
 
   private statusWebSocked: WebSocket
 
-  consumerStatus: FileStatus[] = []
+  private consumerStatus: FileStatus[] = []
+
+
   private documentConsumptionFinishedSubject = new Subject<FileStatus>()
   private documentConsumptionFailedSubject = new Subject<FileStatus>()
 
@@ -78,12 +80,21 @@ export class ConsumerStatusService {
     return status
   }
 
-  newFileUpload(): FileStatus {
+  newFileUpload(filename: string): FileStatus {
     let status = new FileStatus()
+    status.filename = filename
     this.consumerStatus.push(status)
     return status
   }
 
+  getConsumerStatus(phase?: FileStatusPhase) {
+    if (phase) {
+      return this.consumerStatus.filter(s => s.phase == phase)
+    } else {
+      return this.consumerStatus
+    }
+  }
+
   connect() {
     this.disconnect()
     this.statusWebSocked = new WebSocket("ws://localhost:8000/ws/status/");

From 7dedb99daed97632de0bebb7fc0f6aa65dd6a9dc Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 01:15:27 +0100
Subject: [PATCH 19/54] fix status reports for consumption directory status
 messages

---
 src/documents/consumer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index f4a40cbb9..aef8c6345 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -1,6 +1,7 @@
 import datetime
 import hashlib
 import os
+import uuid
 from subprocess import Popen
 
 import magic
@@ -150,7 +151,7 @@ class Consumer(LoggingMixin):
         self.override_correspondent_id = override_correspondent_id
         self.override_document_type_id = override_document_type_id
         self.override_tag_ids = override_tag_ids
-        self.task_id = task_id
+        self.task_id = task_id or str(uuid.uuid4())
 
         self._send_progress(self.filename, 0, 100, 'WORKING',
                             'Received new file.')

From 5651a03b1417f6b5f040e2563ec54456d5dffb3f Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 00:17:27 -0800
Subject: [PATCH 20/54] Drop over background color

---
 .../upload-file-widget/upload-file-widget.component.scss     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
index e69de29bb..461f9d5b3 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
@@ -0,0 +1,5 @@
+@import "/src/theme";
+
+::ng-deep .ngx-file-drop__drop-zone--over {
+  background-color: $primaryFaded !important;
+}

From 53309f017f7e38940bcafce445ee77b30f9f9173 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 02:32:45 -0800
Subject: [PATCH 21/54] Lovely file upload alerts

---
 .../upload-file-widget.component.html         | 26 +++++++++--------
 .../upload-file-widget.component.scss         | 28 +++++++++++++++++++
 .../upload-file-widget.component.ts           |  6 ++--
 .../app/services/consumer-status.service.ts   |  5 ++--
 4 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index 263918a73..b4c4fc01b 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -6,20 +6,22 @@
         (onFileOver)="fileOver($event)" (onFileLeave)="fileLeave($event)" dropZoneClassName="bg-light card"
         multiple="true" contentClassName="justify-content-center d-flex align-items-center p-5" [showBrowseBtn]=true
         browseBtnClassName="btn btn-sm btn-outline-primary ml-2" i18n-dropZoneLabel i18n-browseBtnLabel>
-
       </ngx-file-drop>
     </form>
-    <p>Uploading {{getStatusUploading().length}} files...</p>
     <div *ngFor="let status of getStatus()">
-      <p>{{status.filename}}: {{status.message}}</p>
-      <ngb-progressbar [value]="status.getProgress()" [max]="1" [striped]="true" [animated]="!isFinished(status)" [type]="getType(status)">
-      </ngb-progressbar>
-
-      <div *ngIf="isFinished(status)" class="mb-2">
-        <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">Open document</button>
-        <button class="btn btn-sm btn-outline-secondary" (click)="dismiss(status)">Dismiss</button>
-      </div>
-    
+      <ngb-alert type="secondary" class="mt-2 mb-0" [dismissible]="isFinished(status)" (closed)="dismiss(status)">
+        <h6 class="alert-heading">{{status.filename}}</h6>
+        <p class="mb-0" *ngIf="!isFinished(status) || (isFinished(status) && !status.documentId)">{{status.message}}</p>
+        <ngb-progressbar [value]="status.getProgress()" [max]="1" [type]="getStatusColor(status)"></ngb-progressbar>
+        <div *ngIf="isFinished(status)">
+          <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">
+            <small>Open document</small>
+            <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
+              <path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
+            </svg>
+          </button>
+        </div>
+      </ngb-alert>
     </div>
   </div>
-</app-widget-frame>
\ No newline at end of file
+</app-widget-frame>
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
index 461f9d5b3..807eac66b 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
@@ -3,3 +3,31 @@
 ::ng-deep .ngx-file-drop__drop-zone--over {
   background-color: $primaryFaded !important;
 }
+
+form {
+  position: relative;
+}
+
+.alert {
+  transition: all 0.2s ease;
+}
+
+.alert-heading {
+  font-size: 80%;
+  font-weight: bold;
+}
+
+.btn-sm {
+  line-height: 1;
+}
+
+::ng-deep .progress {
+  position: absolute;
+  top: 0;
+  right: 0;
+  bottom: 0;
+  left: 0;
+  height: auto;
+  mix-blend-mode: soft-light;
+  pointer-events: none;
+}
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index bd0e4e290..e9a62cf9c 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -41,7 +41,7 @@ export class UploadFileWidgetComponent implements OnInit {
     return status.phase == FileStatusPhase.FAILED || status.phase == FileStatusPhase.SUCCESS
   }
 
-  getType(status: FileStatus) {
+  getStatusColor(status: FileStatus) {
     switch (status.phase) {
       case FileStatusPhase.PROCESSING:
       case FileStatusPhase.UPLOADING:
@@ -56,7 +56,7 @@ export class UploadFileWidgetComponent implements OnInit {
   dismiss(status: FileStatus) {
     this.consumerStatusService.dismiss(status)
   }
-  
+
   ngOnInit(): void {
   }
 
@@ -75,7 +75,7 @@ export class UploadFileWidgetComponent implements OnInit {
           let formData = new FormData()
           formData.append('document', file, file.name)
           let status = this.consumerStatusService.newFileUpload(file.name)
-          
+
           status.message = "Connecting..."
 
           this.documentService.uploadDocument(formData).subscribe(event => {
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index 69e58e2dd..d5ac1ba47 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -16,7 +16,7 @@ export class FileStatus {
 
   taskId: string
 
-  phase: FileStatusPhase = FileStatusPhase.STARTED  
+  phase: FileStatusPhase = FileStatusPhase.STARTED
 
   currentPhaseProgress: number
 
@@ -33,7 +33,8 @@ export class FileStatus {
       case FileStatusPhase.UPLOADING:
         return this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.2
       case FileStatusPhase.PROCESSING:
-        return this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.8 + 0.2
+        if (this.currentPhaseProgress > 100) this.currentPhaseProgress = 0
+        return (this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.8) + 0.2
       case FileStatusPhase.SUCCESS:
       case FileStatusPhase.FAILED:
         return 1.0

From 532a3d955ccb69acce81268091d68009abefbbbc Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 02:52:16 -0800
Subject: [PATCH 22/54] Dismiss all

---
 .../upload-file-widget.component.html                | 12 ++++++++++--
 .../upload-file-widget.component.scss                |  9 +++++++++
 .../upload-file-widget.component.ts                  |  4 ++++
 src-ui/src/app/services/consumer-status.service.ts   |  4 ++++
 4 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index b4c4fc01b..d00114934 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -1,5 +1,13 @@
-<app-widget-frame title="Upload new documents" i18n-title>
-
+<app-widget-frame title="Upload new documents" i18n-title [class.has-multiple-status]="getStatus().length > 1">
+  <div header-buttons>
+    <button type="button" class="btn btn-link dismiss-all" [disabled]="!getStatus().length" (click)="dismissAll()">
+      <small class="mr-1">Hide all</small>
+      <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-check2-all" viewBox="0 0 16 16">
+        <path d="M12.354 4.354a.5.5 0 0 0-.708-.708L5 10.293 1.854 7.146a.5.5 0 1 0-.708.708l3.5 3.5a.5.5 0 0 0 .708 0l7-7zm-4.208 7l-.896-.897.707-.707.543.543 6.646-6.647a.5.5 0 0 1 .708.708l-7 7a.5.5 0 0 1-.708 0z"/>
+        <path d="M5.354 7.146l.896.897-.707.707-.897-.896a.5.5 0 1 1 .708-.708z"/>
+      </svg>
+    </button>
+  </div>
   <div content>
     <form>
       <ngx-file-drop dropZoneLabel="Drop documents here or" browseBtnLabel="Browse files" (onFileDrop)="dropped($event)"
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
index 807eac66b..10a83e308 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
@@ -21,6 +21,15 @@ form {
   line-height: 1;
 }
 
+.dismiss-all {
+  transition: opacity 0.2s ease;
+  opacity: 0;
+}
+
+.has-multiple-status .dismiss-all {
+  opacity: 1;
+}
+
 ::ng-deep .progress {
   position: absolute;
   top: 0;
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index e9a62cf9c..19ab8d04b 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -57,6 +57,10 @@ export class UploadFileWidgetComponent implements OnInit {
     this.consumerStatusService.dismiss(status)
   }
 
+  dismissAll() {
+    this.consumerStatusService.dismissAll()
+  }
+
   ngOnInit(): void {
   }
 
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index d5ac1ba47..b2804103d 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -133,6 +133,10 @@ export class ConsumerStatusService {
     }
   }
 
+  dismissAll() {
+    this.consumerStatus = []
+  }
+
   onDocumentConsumptionFinished() {
     return this.documentConsumptionFinishedSubject
   }

From 4eeb5642f5aefe81c6a5bdc1b9475291eb78120f Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 15:02:46 +0100
Subject: [PATCH 23/54] i18n, better error handling

---
 src/documents/consumer.py              |  94 ++++++-----
 src/documents/serialisers.py           |   6 +-
 src/locale/en-us/LC_MESSAGES/django.po | 222 +++++++++++++++----------
 3 files changed, 194 insertions(+), 128 deletions(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index aef8c6345..2fbbe7a0a 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -25,6 +25,8 @@ from .signals import (
     document_consumption_started
 )
 
+from django.utils.translation import gettext as _
+
 
 class ConsumerError(Exception):
     pass
@@ -32,10 +34,10 @@ class ConsumerError(Exception):
 
 class Consumer(LoggingMixin):
 
-    def _send_progress(self, filename, current_progress, max_progress, status,
+    def _send_progress(self, current_progress, max_progress, status,
                        message, document_id=None):
         payload = {
-            'filename': os.path.basename(filename),
+            'filename': os.path.basename(self.filename),
             'task_id': self.task_id,
             'current_progress': current_progress,
             'max_progress': max_progress,
@@ -47,10 +49,10 @@ class Consumer(LoggingMixin):
                                                      {'type': 'status_update',
                                                       'data': payload})
 
-    def _fail(self, message):
-        self._send_progress(self.filename, 100, 100, 'FAILED',
-                            message)
-        raise ConsumerError(f"{self.filename}: {message}")
+    def _fail(self, message, log_message=None):
+        self._send_progress(100, 100, 'FAILED', message)
+        self.log("error", log_message or message)
+        raise ConsumerError(f"{self.filename}: {log_message or message}")
 
     def __init__(self):
         super().__init__()
@@ -66,11 +68,10 @@ class Consumer(LoggingMixin):
 
     def pre_check_file_exists(self):
         if not os.path.isfile(self.path):
-            self.log(
-                "error",
-                "Cannot consume {}: It is not a file.".format(self.path)
+            self._fail(
+                _("File not found"),
+                f"Cannot consume {self.path}: It is not a file."
             )
-            self._fail("File not found")
 
     def pre_check_duplicate(self):
         with open(self.path, "rb") as f:
@@ -78,11 +79,10 @@ class Consumer(LoggingMixin):
         if Document.objects.filter(Q(checksum=checksum) | Q(archive_checksum=checksum)).exists():  # NOQA: E501
             if settings.CONSUMER_DELETE_DUPLICATES:
                 os.unlink(self.path)
-            self.log(
-                "error",
-                "Not consuming {}: It is a duplicate.".format(self.filename)
+            self._fail(
+                _("Document is a duplicate"),
+                f"Not consuming {self.filename}: It is a duplicate."
             )
-            self._fail("Document is a duplicate")
 
     def pre_check_directories(self):
         os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
@@ -95,14 +95,16 @@ class Consumer(LoggingMixin):
             return
 
         if not os.path.isfile(settings.PRE_CONSUME_SCRIPT):
-            raise ConsumerError(
+            self._fail(
+                _("Pre-consume script does not exist."),
                 f"Configured pre-consume script "
                 f"{settings.PRE_CONSUME_SCRIPT} does not exist.")
 
         try:
             Popen((settings.PRE_CONSUME_SCRIPT, self.path)).wait()
         except Exception as e:
-            raise ConsumerError(
+            self._fail(
+                _("Error while executing pre-consume script"),
                 f"Error while executing pre-consume script: {e}"
             )
 
@@ -111,9 +113,11 @@ class Consumer(LoggingMixin):
             return
 
         if not os.path.isfile(settings.POST_CONSUME_SCRIPT):
-            raise ConsumerError(
+            self._fail(
+                _("Post-consume script does not exist."),
                 f"Configured post-consume script "
-                f"{settings.POST_CONSUME_SCRIPT} does not exist.")
+                f"{settings.POST_CONSUME_SCRIPT} does not exist."
+            )
 
         try:
             Popen((
@@ -129,8 +133,9 @@ class Consumer(LoggingMixin):
                     "name", flat=True)))
             )).wait()
         except Exception as e:
-            raise ConsumerError(
-                f"Error while executing pre-consume script: {e}"
+            self._fail(
+                _("Error while executing post-consume script"),
+                f"Error while executing post-consume script: {e}"
             )
 
     def try_consume_file(self,
@@ -153,8 +158,7 @@ class Consumer(LoggingMixin):
         self.override_tag_ids = override_tag_ids
         self.task_id = task_id or str(uuid.uuid4())
 
-        self._send_progress(self.filename, 0, 100, 'WORKING',
-                            'Received new file.')
+        self._send_progress(0, 100, 'WORKING', _('Received new file'))
 
         # this is for grouping logging entries for this particular file
         # together.
@@ -177,10 +181,13 @@ class Consumer(LoggingMixin):
 
         parser_class = get_parser_class_for_mime_type(mime_type)
         if not parser_class:
-            self._fail(f"Unsupported mime type {mime_type}")
+            self._fail(
+                _("File type %(type)s not supported") %
+                {'type': mime_type},
+                f"Unsupported mime type {mime_type}"
+            )
         else:
-            self.log("debug",
-                     f"Parser: {parser_class.__name__}")
+            self.log("debug", f"Parser: {parser_class.__name__}")
 
         # Notify all listeners that we're going to do some work.
 
@@ -195,7 +202,7 @@ class Consumer(LoggingMixin):
         def progress_callback(current_progress, max_progress, message):
             # recalculate progress to be within 20 and 80
             p = int((current_progress / max_progress) * 50 + 20)
-            self._send_progress(self.filename, p, 100, "WORKING", message)
+            self._send_progress(p, 100, "WORKING", message)
 
         # This doesn't parse the document yet, but gives us a parser.
 
@@ -206,32 +213,36 @@ class Consumer(LoggingMixin):
 
         # Parse the document. This may take some time.
 
+        text = None
+        date = None
+        thumbnail = None
+        archive_path = None
+
         try:
-            self._send_progress(self.filename, 20, 100, 'WORKING',
-                                'Parsing document...')
+            self._send_progress(20, 100, 'WORKING', _('Parsing document...'))
             self.log("debug", "Parsing {}...".format(self.filename))
             document_parser.parse(self.path, mime_type, self.filename)
 
             self.log("debug", f"Generating thumbnail for {self.filename}...")
-            self._send_progress(self.filename, 70, 100, 'WORKING',
-                                'Generating thumbnail...')
+            self._send_progress(70, 100, 'WORKING',
+                                _('Generating thumbnail...'))
             thumbnail = document_parser.get_optimised_thumbnail(
                 self.path, mime_type)
 
             text = document_parser.get_text()
             date = document_parser.get_date()
             if not date:
-                self._send_progress(self.filename, 90, 100, 'WORKING',
-                                    'Getting date from document...')
+                self._send_progress(90, 100, 'WORKING',
+                                    _('Getting date from document...'))
                 date = parse_date(self.filename, text)
             archive_path = document_parser.get_archive_path()
 
         except ParseError as e:
             document_parser.cleanup()
-            self.log(
-                "error",
-                f"Error while consuming document {self.filename}: {e}")
-            self._fail(str(e))
+            self._fail(
+                str(e),
+                f"Error while consuming document {self.filename}: {e}"
+            )
 
         # Prepare the document classifier.
 
@@ -247,8 +258,7 @@ class Consumer(LoggingMixin):
                 "warning",
                 f"Cannot classify documents: {e}.")
             classifier = None
-        self._send_progress(self.filename, 95, 100, 'WORKING',
-                            'Storing the document...')
+        self._send_progress(95, 100, 'WORKING', _('Saving document...'))
         # now that everything is done, we can start to store the document
         # in the system. This will be a transaction and reasonably fast.
         try:
@@ -302,12 +312,11 @@ class Consumer(LoggingMixin):
                 os.unlink(self.path)
 
         except Exception as e:
-            self.log(
-                "error",
+            self._fail(
+                str(e),
                 f"The following error occured while consuming "
                 f"{self.filename}: {e}"
             )
-            self._fail(str(e))
         finally:
             document_parser.cleanup()
 
@@ -318,8 +327,7 @@ class Consumer(LoggingMixin):
             "Document {} consumption finished".format(document)
         )
 
-        self._send_progress(self.filename, 100, 100, 'SUCCESS',
-                            'Finished.', document.id)
+        self._send_progress(100, 100, 'SUCCESS', _('Finished.'), document.id)
 
         return document
 
diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py
index 70a44d3fd..aa4ef4bf8 100644
--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@@ -8,6 +8,8 @@ from .models import Correspondent, Tag, Document, Log, DocumentType, \
     SavedView, SavedViewFilterRule
 from .parsers import is_mime_type_supported
 
+from django.utils.translation import gettext as _
+
 
 # https://www.django-rest-framework.org/api-guide/serializers/#example
 class DynamicFieldsModelSerializer(serializers.ModelSerializer):
@@ -378,7 +380,9 @@ class PostDocumentSerializer(serializers.Serializer):
 
         if not is_mime_type_supported(mime_type):
             raise serializers.ValidationError(
-                "This file type is not supported.")
+                _("File type %(type)s not supported") %
+                {'type': mime_type}
+            )
 
         return document.name, document_data
 
diff --git a/src/locale/en-us/LC_MESSAGES/django.po b/src/locale/en-us/LC_MESSAGES/django.po
index 414caba52..5451b8dde 100644
--- a/src/locale/en-us/LC_MESSAGES/django.po
+++ b/src/locale/en-us/LC_MESSAGES/django.po
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-01-10 21:41+0000\n"
+"POT-Creation-Date: 2021-01-26 14:59+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -21,320 +21,374 @@ msgstr ""
 msgid "Documents"
 msgstr ""
 
-#: documents/models.py:32
-msgid "Any word"
+#: documents/consumer.py:72
+msgid "File not found"
+msgstr "Datei nicht gefunden"
+
+#: documents/consumer.py:83
+msgid "Document is a duplicate"
+msgstr "Dokument existiert bereits"
+
+#: documents/consumer.py:99
+msgid "Pre-consume script does not exist."
+msgstr ""
+
+#: documents/consumer.py:107
+msgid "Error while executing pre-consume script"
+msgstr ""
+
+#: documents/consumer.py:117
+msgid "Post-consume script does not exist."
+msgstr ""
+
+#: documents/consumer.py:137
+msgid "Error while executing post-consume script"
+msgstr ""
+
+#: documents/consumer.py:161
+msgid "Received new file"
+msgstr ""
+
+#: documents/consumer.py:185 documents/serialisers.py:383
+#, fuzzy, python-format
+#| msgid "File type {type} not supported."
+msgid "File type %(type)s not supported"
+msgstr "Dateityp {type} wird nicht unterstützt"
+
+#: documents/consumer.py:222
+msgid "Parsing document..."
+msgstr ""
+
+#: documents/consumer.py:228
+msgid "Generating thumbnail..."
+msgstr ""
+
+#: documents/consumer.py:236
+msgid "Getting date from document..."
+msgstr ""
+
+#: documents/consumer.py:261
+msgid "Saving document..."
+msgstr ""
+
+#: documents/consumer.py:330
+msgid "Finished."
 msgstr ""
 
 #: documents/models.py:33
-msgid "All words"
+msgid "Any word"
 msgstr ""
 
 #: documents/models.py:34
-msgid "Exact match"
+msgid "All words"
 msgstr ""
 
 #: documents/models.py:35
-msgid "Regular expression"
+msgid "Exact match"
 msgstr ""
 
 #: documents/models.py:36
-msgid "Fuzzy word"
+msgid "Regular expression"
 msgstr ""
 
 #: documents/models.py:37
+msgid "Fuzzy word"
+msgstr ""
+
+#: documents/models.py:38
 msgid "Automatic"
 msgstr ""
 
-#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25
+#: documents/models.py:42 documents/models.py:352 paperless_mail/models.py:25
 #: paperless_mail/models.py:109
 msgid "name"
 msgstr ""
 
-#: documents/models.py:45
+#: documents/models.py:46
 msgid "match"
 msgstr ""
 
-#: documents/models.py:49
+#: documents/models.py:50
 msgid "matching algorithm"
 msgstr ""
 
-#: documents/models.py:55
+#: documents/models.py:56
 msgid "is insensitive"
 msgstr ""
 
-#: documents/models.py:80 documents/models.py:140
+#: documents/models.py:75 documents/models.py:135
 msgid "correspondent"
 msgstr ""
 
-#: documents/models.py:81
+#: documents/models.py:76
 msgid "correspondents"
 msgstr ""
 
-#: documents/models.py:103
+#: documents/models.py:98
 msgid "color"
 msgstr ""
 
-#: documents/models.py:107
+#: documents/models.py:102
 msgid "is inbox tag"
 msgstr ""
 
-#: documents/models.py:109
+#: documents/models.py:104
 msgid ""
 "Marks this tag as an inbox tag: All newly consumed documents will be tagged "
 "with inbox tags."
 msgstr ""
 
-#: documents/models.py:114
+#: documents/models.py:109
 msgid "tag"
 msgstr ""
 
-#: documents/models.py:115 documents/models.py:171
+#: documents/models.py:110 documents/models.py:166
 msgid "tags"
 msgstr ""
 
-#: documents/models.py:121 documents/models.py:153
+#: documents/models.py:116 documents/models.py:148
 msgid "document type"
 msgstr ""
 
-#: documents/models.py:122
+#: documents/models.py:117
 msgid "document types"
 msgstr ""
 
-#: documents/models.py:130
+#: documents/models.py:125
 msgid "Unencrypted"
 msgstr ""
 
-#: documents/models.py:131
+#: documents/models.py:126
 msgid "Encrypted with GNU Privacy Guard"
 msgstr ""
 
-#: documents/models.py:144
+#: documents/models.py:139
 msgid "title"
 msgstr ""
 
-#: documents/models.py:157
+#: documents/models.py:152
 msgid "content"
 msgstr ""
 
-#: documents/models.py:159
+#: documents/models.py:154
 msgid ""
 "The raw, text-only data of the document. This field is primarily used for "
 "searching."
 msgstr ""
 
-#: documents/models.py:164
+#: documents/models.py:159
 msgid "mime type"
 msgstr ""
 
-#: documents/models.py:175
+#: documents/models.py:170
 msgid "checksum"
 msgstr ""
 
-#: documents/models.py:179
+#: documents/models.py:174
 msgid "The checksum of the original document."
 msgstr ""
 
-#: documents/models.py:183
+#: documents/models.py:178
 msgid "archive checksum"
 msgstr ""
 
-#: documents/models.py:188
+#: documents/models.py:183
 msgid "The checksum of the archived document."
 msgstr ""
 
-#: documents/models.py:192 documents/models.py:332
+#: documents/models.py:187 documents/models.py:330
 msgid "created"
 msgstr ""
 
-#: documents/models.py:196
+#: documents/models.py:191
 msgid "modified"
 msgstr ""
 
-#: documents/models.py:200
+#: documents/models.py:195
 msgid "storage type"
 msgstr ""
 
-#: documents/models.py:208
+#: documents/models.py:203
 msgid "added"
 msgstr ""
 
-#: documents/models.py:212
+#: documents/models.py:207
 msgid "filename"
 msgstr ""
 
-#: documents/models.py:217
+#: documents/models.py:212
 msgid "Current filename in storage"
 msgstr ""
 
-#: documents/models.py:221
+#: documents/models.py:216
 msgid "archive serial number"
 msgstr ""
 
-#: documents/models.py:226
+#: documents/models.py:221
 msgid "The position of this document in your physical document archive."
 msgstr ""
 
-#: documents/models.py:232
+#: documents/models.py:227
 msgid "document"
 msgstr ""
 
-#: documents/models.py:233
+#: documents/models.py:228
 msgid "documents"
 msgstr ""
 
-#: documents/models.py:315
+#: documents/models.py:313
 msgid "debug"
 msgstr ""
 
-#: documents/models.py:316
+#: documents/models.py:314
 msgid "information"
 msgstr ""
 
-#: documents/models.py:317
+#: documents/models.py:315
 msgid "warning"
 msgstr ""
 
-#: documents/models.py:318
+#: documents/models.py:316
 msgid "error"
 msgstr ""
 
-#: documents/models.py:319
+#: documents/models.py:317
 msgid "critical"
 msgstr ""
 
-#: documents/models.py:323
+#: documents/models.py:321
 msgid "group"
 msgstr ""
 
-#: documents/models.py:326
+#: documents/models.py:324
 msgid "message"
 msgstr ""
 
-#: documents/models.py:329
+#: documents/models.py:327
 msgid "level"
 msgstr ""
 
-#: documents/models.py:336
+#: documents/models.py:334
 msgid "log"
 msgstr ""
 
-#: documents/models.py:337
+#: documents/models.py:335
 msgid "logs"
 msgstr ""
 
-#: documents/models.py:348 documents/models.py:398
+#: documents/models.py:346 documents/models.py:396
 msgid "saved view"
 msgstr ""
 
-#: documents/models.py:349
+#: documents/models.py:347
 msgid "saved views"
 msgstr ""
 
-#: documents/models.py:352
+#: documents/models.py:350
 msgid "user"
 msgstr ""
 
-#: documents/models.py:358
+#: documents/models.py:356
 msgid "show on dashboard"
 msgstr ""
 
-#: documents/models.py:361
+#: documents/models.py:359
 msgid "show in sidebar"
 msgstr ""
 
-#: documents/models.py:365
+#: documents/models.py:363
 msgid "sort field"
 msgstr ""
 
-#: documents/models.py:368
+#: documents/models.py:366
 msgid "sort reverse"
 msgstr ""
 
-#: documents/models.py:374
+#: documents/models.py:372
 msgid "title contains"
 msgstr ""
 
-#: documents/models.py:375
+#: documents/models.py:373
 msgid "content contains"
 msgstr ""
 
-#: documents/models.py:376
+#: documents/models.py:374
 msgid "ASN is"
 msgstr ""
 
-#: documents/models.py:377
+#: documents/models.py:375
 msgid "correspondent is"
 msgstr ""
 
-#: documents/models.py:378
+#: documents/models.py:376
 msgid "document type is"
 msgstr ""
 
-#: documents/models.py:379
+#: documents/models.py:377
 msgid "is in inbox"
 msgstr ""
 
-#: documents/models.py:380
+#: documents/models.py:378
 msgid "has tag"
 msgstr ""
 
-#: documents/models.py:381
+#: documents/models.py:379
 msgid "has any tag"
 msgstr ""
 
-#: documents/models.py:382
+#: documents/models.py:380
 msgid "created before"
 msgstr ""
 
-#: documents/models.py:383
+#: documents/models.py:381
 msgid "created after"
 msgstr ""
 
-#: documents/models.py:384
+#: documents/models.py:382
 msgid "created year is"
 msgstr ""
 
-#: documents/models.py:385
+#: documents/models.py:383
 msgid "created month is"
 msgstr ""
 
-#: documents/models.py:386
+#: documents/models.py:384
 msgid "created day is"
 msgstr ""
 
-#: documents/models.py:387
+#: documents/models.py:385
 msgid "added before"
 msgstr ""
 
-#: documents/models.py:388
+#: documents/models.py:386
 msgid "added after"
 msgstr ""
 
-#: documents/models.py:389
+#: documents/models.py:387
 msgid "modified before"
 msgstr ""
 
-#: documents/models.py:390
+#: documents/models.py:388
 msgid "modified after"
 msgstr ""
 
-#: documents/models.py:391
+#: documents/models.py:389
 msgid "does not have tag"
 msgstr ""
 
-#: documents/models.py:402
+#: documents/models.py:400
 msgid "rule type"
 msgstr ""
 
-#: documents/models.py:406
+#: documents/models.py:404
 msgid "value"
 msgstr ""
 
-#: documents/models.py:412
+#: documents/models.py:410
 msgid "filter rule"
 msgstr ""
 
-#: documents/models.py:413
+#: documents/models.py:411
 msgid "filter rules"
 msgstr ""
 
@@ -378,23 +432,23 @@ msgstr ""
 msgid "Sign in"
 msgstr ""
 
-#: paperless/settings.py:268
+#: paperless/settings.py:286
 msgid "English"
 msgstr ""
 
-#: paperless/settings.py:269
+#: paperless/settings.py:287
 msgid "German"
 msgstr ""
 
-#: paperless/settings.py:270
+#: paperless/settings.py:288
 msgid "Dutch"
 msgstr ""
 
-#: paperless/settings.py:271
+#: paperless/settings.py:289
 msgid "French"
 msgstr ""
 
-#: paperless/urls.py:108
+#: paperless/urls.py:114
 msgid "Paperless-ng administration"
 msgstr ""
 

From 868fd4155a4f7b194357bf88f0a622ce169b84fa Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 15:19:56 +0100
Subject: [PATCH 24/54] bug fixes, test case fixes

---
 src/documents/consumer.py            |  6 +++---
 src/documents/parsers.py             |  2 +-
 src/documents/tests/test_consumer.py | 20 +++++++++++---------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 2fbbe7a0a..d0f7399c8 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -37,7 +37,7 @@ class Consumer(LoggingMixin):
     def _send_progress(self, current_progress, max_progress, status,
                        message, document_id=None):
         payload = {
-            'filename': os.path.basename(self.filename),
+            'filename': os.path.basename(self.filename) if self.filename else None,  # NOQA: E501
             'task_id': self.task_id,
             'current_progress': current_progress,
             'max_progress': max_progress,
@@ -70,7 +70,7 @@ class Consumer(LoggingMixin):
         if not os.path.isfile(self.path):
             self._fail(
                 _("File not found"),
-                f"Cannot consume {self.path}: It is not a file."
+                f"Cannot consume {self.path}: File not found."
             )
 
     def pre_check_duplicate(self):
@@ -80,7 +80,7 @@ class Consumer(LoggingMixin):
             if settings.CONSUMER_DELETE_DUPLICATES:
                 os.unlink(self.path)
             self._fail(
-                _("Document is a duplicate"),
+                _("Document already exists"),
                 f"Not consuming {self.filename}: It is a duplicate."
             )
 
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index f8fd117d9..ddad6897a 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -261,7 +261,7 @@ class DocumentParser(LoggingMixin):
     `paperless_tesseract.parsers` for inspiration.
     """
 
-    def __init__(self, logging_group, progress_callback):
+    def __init__(self, logging_group, progress_callback=None):
         super().__init__()
         self.logging_group = logging_group
         os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py
index a6861a541..799250d08 100644
--- a/src/documents/tests/test_consumer.py
+++ b/src/documents/tests/test_consumer.py
@@ -170,7 +170,7 @@ class DummyParser(DocumentParser):
         raise NotImplementedError()
 
     def __init__(self, logging_group, scratch_dir, archive_path):
-        super(DummyParser, self).__init__(logging_group)
+        super(DummyParser, self).__init__(logging_group, None)
         _, self.fake_thumb = tempfile.mkstemp(suffix=".png", dir=scratch_dir)
         self.archive_path = archive_path
 
@@ -212,10 +212,10 @@ def fake_magic_from_file(file, mime=False):
 @mock.patch("documents.consumer.magic.from_file", fake_magic_from_file)
 class TestConsumer(DirectoriesMixin, TestCase):
 
-    def make_dummy_parser(self, logging_group):
+    def make_dummy_parser(self, logging_group, progress_callback=None):
         return DummyParser(logging_group, self.dirs.scratch_dir, self.get_test_archive_file())
 
-    def make_faulty_parser(self, logging_group):
+    def make_faulty_parser(self, logging_group, progress_callback=None):
         return FaultyParser(logging_group, self.dirs.scratch_dir)
 
     def setUp(self):
@@ -312,7 +312,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
         try:
             self.consumer.try_consume_file("non-existing-file")
         except ConsumerError as e:
-            self.assertTrue(str(e).endswith('It is not a file'))
+            self.assertTrue(str(e).endswith('File not found.'))
             return
 
         self.fail("Should throw exception")
@@ -350,7 +350,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
         try:
             self.consumer.try_consume_file(self.get_test_file())
         except ConsumerError as e:
-            self.assertEqual("Unsupported mime type application/pdf of file sample.pdf", str(e))
+            self.assertEqual(str(e), "sample.pdf: Unsupported mime type application/pdf")
             return
 
         self.fail("Should throw exception")
@@ -366,7 +366,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
         try:
             self.consumer.try_consume_file(self.get_test_file())
         except ConsumerError as e:
-            self.assertEqual(str(e), "Does not compute.")
+            self.assertEqual(str(e), "sample.pdf: Error while consuming document sample.pdf: Does not compute.")
             return
 
         self.fail("Should throw exception.")
@@ -378,7 +378,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
         try:
             self.consumer.try_consume_file(filename)
         except ConsumerError as e:
-            self.assertEqual(str(e), "NO.")
+            self.assertEqual(str(e), "sample.pdf: The following error occured while consuming sample.pdf: NO.")
         else:
             self.fail("Should raise exception")
 
@@ -482,6 +482,7 @@ class PreConsumeTestCase(TestCase):
     @override_settings(PRE_CONSUME_SCRIPT="does-not-exist")
     def test_pre_consume_script_not_found(self, m):
         c = Consumer()
+        c.filename = "somefile.pdf"
         c.path = "path-to-file"
         self.assertRaises(ConsumerError, c.run_pre_consume_script)
 
@@ -523,8 +524,9 @@ class PostConsumeTestCase(TestCase):
     @override_settings(POST_CONSUME_SCRIPT="does-not-exist")
     def test_post_consume_script_not_found(self):
         doc = Document.objects.create(title="Test", mime_type="application/pdf")
-
-        self.assertRaises(ConsumerError, Consumer().run_post_consume_script, doc)
+        c = Consumer()
+        c.filename = "somefile.pdf"
+        self.assertRaises(ConsumerError, c.run_post_consume_script, doc)
 
     @mock.patch("documents.consumer.Popen")
     def test_post_consume_script_simple(self, m):

From 313f2886d747973dd7d0400d827604c8653d699e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 15:26:25 +0100
Subject: [PATCH 25/54] fix test cases

---
 src/documents/tests/test_consumer.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py
index 799250d08..54d20ca8b 100644
--- a/src/documents/tests/test_consumer.py
+++ b/src/documents/tests/test_consumer.py
@@ -228,7 +228,11 @@ class TestConsumer(DirectoriesMixin, TestCase):
             "mime_types": {"application/pdf": ".pdf"},
             "weight": 0
         })]
+        self.addCleanup(patcher.stop)
 
+        # this prevents websocket message reports during testing.
+        patcher = mock.patch("documents.consumer.Consumer._send_progress")
+        patcher.start()
         self.addCleanup(patcher.stop)
 
         self.consumer = Consumer()
@@ -479,8 +483,9 @@ class PreConsumeTestCase(TestCase):
         m.assert_not_called()
 
     @mock.patch("documents.consumer.Popen")
+    @mock.patch("documents.consumer.Consumer._send_progress")
     @override_settings(PRE_CONSUME_SCRIPT="does-not-exist")
-    def test_pre_consume_script_not_found(self, m):
+    def test_pre_consume_script_not_found(self, m, m2):
         c = Consumer()
         c.filename = "somefile.pdf"
         c.path = "path-to-file"
@@ -504,7 +509,6 @@ class PreConsumeTestCase(TestCase):
                 self.assertEqual(command[1], "path-to-file")
 
 
-
 class PostConsumeTestCase(TestCase):
 
     @mock.patch("documents.consumer.Popen")
@@ -520,9 +524,9 @@ class PostConsumeTestCase(TestCase):
 
         m.assert_not_called()
 
-
     @override_settings(POST_CONSUME_SCRIPT="does-not-exist")
-    def test_post_consume_script_not_found(self):
+    @mock.patch("documents.consumer.Consumer._send_progress")
+    def test_post_consume_script_not_found(self, m):
         doc = Document.objects.create(title="Test", mime_type="application/pdf")
         c = Consumer()
         c.filename = "somefile.pdf"

From 3b5d55c428861dabd54fe8ac536d4b584297629e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 15:43:07 +0100
Subject: [PATCH 26/54] pycodestyle

---
 src/paperless/asgi.py      | 6 +++---
 src/paperless/consumers.py | 6 ++++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/paperless/asgi.py b/src/paperless/asgi.py
index 45565c68a..2f6cc2d5f 100644
--- a/src/paperless/asgi.py
+++ b/src/paperless/asgi.py
@@ -8,10 +8,10 @@ from django.core.asgi import get_asgi_application
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings")
 django_asgi_app = get_asgi_application()
 
-from channels.auth import AuthMiddlewareStack
-from channels.routing import ProtocolTypeRouter, URLRouter
+from channels.auth import AuthMiddlewareStack  # NOQA: E402
+from channels.routing import ProtocolTypeRouter, URLRouter  # NOQA: E402
 
-from paperless.urls import websocket_urlpatterns
+from paperless.urls import websocket_urlpatterns  # NOQA: E402
 
 application = ProtocolTypeRouter({
     "http": get_asgi_application(),
diff --git a/src/paperless/consumers.py b/src/paperless/consumers.py
index fbb7b72d0..21a0e3ede 100644
--- a/src/paperless/consumers.py
+++ b/src/paperless/consumers.py
@@ -7,10 +7,12 @@ from channels.generic.websocket import WebsocketConsumer
 class StatusConsumer(WebsocketConsumer):
     def connect(self):
         self.accept()
-        async_to_sync(self.channel_layer.group_add)('status_updates', self.channel_name)
+        async_to_sync(self.channel_layer.group_add)(
+            'status_updates', self.channel_name)
 
     def disconnect(self, close_code):
-        async_to_sync(self.channel_layer.group_discard)('status_updates', self.channel_name)
+        async_to_sync(self.channel_layer.group_discard)(
+            'status_updates', self.channel_name)
 
     def status_update(self, event):
         self.send(json.dumps(event['data']))

From f0d49a1382830099023ebd9e77a9592a2f381608 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 15:55:47 +0100
Subject: [PATCH 27/54] enable docker image builds on feature branches

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 425243163..03b789bb7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -225,7 +225,7 @@ jobs:
 
   # build and push image to docker hub.
   build-docker-image:
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-'))
+    if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-'))
     runs-on: ubuntu-latest
     needs: [frontend, tests]
     steps:

From 18d105b4f8e315b4ea1dbc7cda93c855fcbe8efc Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 18:00:59 +0100
Subject: [PATCH 28/54] pinned scikit learn version

---
 Pipfile      |   3 +-
 Pipfile.lock | 170 +++++++++++++++++++++++++--------------------------
 2 files changed, 85 insertions(+), 88 deletions(-)

diff --git a/Pipfile b/Pipfile
index 140df6d02..e30abc064 100644
--- a/Pipfile
+++ b/Pipfile
@@ -32,7 +32,8 @@ python-Levenshtein = "*"
 python-magic = "*"
 psycopg2-binary = "*"
 redis = "*"
-scikit-learn="~=0.24.0"
+# Pinned because aarch64 wheels and updates cause warnings when loading the classifier model.
+scikit-learn="==0.24.0"
 whitenoise = "~=5.2.0"
 watchdog = "*"
 whoosh="~=2.7.4"
diff --git a/Pipfile.lock b/Pipfile.lock
index 9c30dedfa..f60ac0223 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "ede35846cb5a19613a24fa1ad4ba0e0ea0290d45fc912e7e69136bc56bb64007"
+            "sha256": "61ffe4b723ce6b04e67636f1572f833ee9adacdcf74a883c558ed4b0c906e75a"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -910,39 +910,35 @@
         },
         "scikit-learn": {
             "hashes": [
-                "sha256:0567a2d29ad08af98653300c623bd8477b448fe66ced7198bef4ed195925f082",
-                "sha256:087dfede39efb06ab30618f9ab55a0397f29c38d63cd0ab88d12b500b7d65fd7",
-                "sha256:1adf483e91007a87171d7ce58c34b058eb5dab01b5fee6052f15841778a8ecd8",
-                "sha256:259ec35201e82e2db1ae2496f229e63f46d7f1695ae68eef9350b00dc74ba52f",
-                "sha256:3c4f07f47c04e81b134424d53c3f5e16dfd7f494e44fd7584ba9ce9de2c5e6c1",
-                "sha256:4562dcf4793e61c5d0f89836d07bc37521c3a1889da8f651e2c326463c4bd697",
-                "sha256:4ddd2b6f7449a5d539ff754fa92d75da22de261fd8fdcfb3596799fadf255101",
-                "sha256:7b04691eb2f41d2c68dbda8d1bd3cb4ef421bdc43aaa56aeb6c762224552dfb6",
-                "sha256:826b92bf45b8ad80444814e5f4ac032156dd481e48d7da33d611f8fe96d5f08b",
-                "sha256:83b21ff053b1ff1c018a2d24db6dd3ea339b1acfbaa4d9c881731f43748d8b3b",
-                "sha256:8772b99d683be8f67fcc04789032f1b949022a0e6880ee7b75a7ec97dbbb5d0b",
-                "sha256:895dbf2030aa7337649e36a83a007df3c9811396b4e2fa672a851160f36ce90c",
-                "sha256:8aa1b3ac46b80eaa552b637eeadbbce3be5931e4b5002b964698e33a1b589e1e",
-                "sha256:99349d77f54e11f962d608d94dfda08f0c9e5720d97132233ebdf35be2858b2d",
-                "sha256:9a24d1ccec2a34d4cd3f2a1f86409f3f5954cc23d4d2270ba0d03cf018aa4780",
-                "sha256:9bed8a1ef133c8e2f13966a542cb8125eac7f4b67dcd234197c827ba9c7dd3e0",
-                "sha256:9c6097b6a9b2bafc5e0f31f659e6ab5e131383209c30c9e978c5b8abdac5ed2a",
-                "sha256:9dfa564ef27e8e674aa1cc74378416d580ac4ede1136c13dd555a87996e13422",
-                "sha256:a0334a1802e64d656022c3bfab56a73fbd6bf4b1298343f3688af2151810bbdf",
-                "sha256:a29460499c1e62b7a830bb57ca42e615375a6ab1bcad053cd25b493588348ea8",
-                "sha256:a36e159a0521e13bbe15ca8c8d038b3a1dd4c7dad18d276d76992e03b92cf643",
-                "sha256:ae7191f0253145503fbfa1934cbc83f48260f534343e34115398fb5fc2be55b0",
-                "sha256:c13ebac42236b1c46397162471ea1c46af68413000e28b9309f8c05722c65a09",
-                "sha256:c3deb3b19dd9806acf00cf0d400e84562c227723013c33abefbbc3cf906596e9",
-                "sha256:c658432d8a20e95398f6bb95ff9731ce9dfa343fdf21eea7ec6a7edfacd4b4d9",
-                "sha256:c7f4eb77504ac586d8ac1bde1b0c04b504487210f95297235311a0ab7edd7e38",
-                "sha256:d54dbaadeb1425b7d6a66bf44bee2bb2b899fe3e8850b8e94cfb9c904dcb46d0",
-                "sha256:ddb52d088889f5596bc4d1de981f2eca106b58243b6679e4782f3ba5096fd645",
-                "sha256:ed9d65594948678827f4ff0e7ae23344e2f2b4cabbca057ccaed3118fdc392ca",
-                "sha256:fab31f48282ebf54dd69f6663cd2d9800096bad1bb67bbc9c9ac84eb77b41972"
+                "sha256:076369634ee72b5a5941440661e2f306ff4ac30903802dc52031c7e9199ac640",
+                "sha256:18f7131e62265bf2691ed1d0303c640313894ccfe4278427478c6b2f45094b53",
+                "sha256:26f66b3726b54dfb76ea51c5d9c2431ed17ebc066cb4527662b9e851a3e7ba61",
+                "sha256:2951f87d35e72f007701c6e028aa230f6df6212a3194677c0c950486066a454d",
+                "sha256:2a5348585aa793bc8cc5a72f8e9067c9380834b0aadbd55f924843b071f13282",
+                "sha256:3eeff086f7329521d27249a082ea3c48c085cedb110db5f65968ab55c3ba2e09",
+                "sha256:4395e91b3548005f4a645018435b5a94f8cce232b5b70753020e606c6a750656",
+                "sha256:44e452ea8491225c5783d49577aad0f36202dfd52aec7f82c0fdfe5fbd5f7400",
+                "sha256:490436b44b3a1957cb625e871764b0aa330b34cc416aea4abc6c38ca63d0d682",
+                "sha256:5e6e3c042cea83f2e20a45e563b8eabc1f8f72446251fe23ebefdf111a173a33",
+                "sha256:66f27bf21202a850bcd7b6303916e4907f6e22ec59a14974ede4955aed5c7ed0",
+                "sha256:743b6edd98c98991be46c08e6b21df3861d5ae915f91d59f988384d93f7263e7",
+                "sha256:758619e49cd7c17282e6cc60d5cc73c02c072b47c9a10010bb3bb47e0d976e50",
+                "sha256:7f654befc5ad413690cc58f3f34a3e906caf825195ce0fda00a8e9565e1403e6",
+                "sha256:800aaf63f8838c00e85db2267dd226f89858594843fd03932a9eda95746d2c40",
+                "sha256:80ca024154b84b6ac4cfc86930ba13fdc348a209753bf2c16129db6f9eb8a80b",
+                "sha256:890d7d588f65acb0c4f6c083347c9076916bda5e6bd8400f06244b1afc1009af",
+                "sha256:905d8934d1e27a686698864a5863ff2c0e13a2ae1adb78a8a848aacc8a49927d",
+                "sha256:a83fcd9d59c42a2f66b307e3b0b0f08aa8e6e45be33da055697ea499f0e4f7c2",
+                "sha256:afeb06dc69847927634e58579b9cdc72e1390b79497336b2324b1b173f33bd47",
+                "sha256:b0d13fd56d26cf3de0314a4fd48037108c638fe126d813f5c1222bb0f08b6a76",
+                "sha256:c08b27cb78ee8d2dc781a7affed09859441f5b624f9f92da59ac0791c8774dfc",
+                "sha256:c912247e42114f389858ae05d63f4359d4e667ea72aaabee191aee9ad3f9774a",
+                "sha256:d7fe05fcb44eadd6d6c874c768f085f5de1239db3a3b7be4d3d23d12e4120589",
+                "sha256:d819d625832fb2969911a243e009cfa135cb8ef1e150866e417d6e9d75290087",
+                "sha256:e534f5f3796db6781c87e9835dcd51b7854c8c5a379c9210b93605965c1941fd"
             ],
             "index": "pypi",
-            "version": "==0.24.1"
+            "version": "==0.24.0"
         },
         "scipy": {
             "hashes": [
@@ -1268,60 +1264,60 @@
         },
         "coverage": {
             "hashes": [
-                "sha256:08b3ba72bd981531fd557f67beee376d6700fba183b167857038997ba30dd297",
-                "sha256:262066798d786ad67a13c7ba869e3ce0e39609f99f6d6c80160ad602c4808e32",
-                "sha256:2757fa64e11ec12220968f65d086b7a29b6583d16e9a544c889b22ba98555ef1",
-                "sha256:3102bb2c206700a7d28181dbe04d66b30780cde1d1c02c5f3c165cf3d2489497",
-                "sha256:3498b27d8236057def41de3585f317abae235dd3a11d33e01736ffedb2ef8606",
-                "sha256:378ac77af41350a8c6b8801a66021b52da8a05fd77e578b7380e876c0ce4f528",
-                "sha256:38f16b1317b8dd82df67ed5daa5f5e7c959e46579840d77a67a4ceb9cef0a50b",
-                "sha256:3911c2ef96e5ddc748a3c8b4702c61986628bb719b8378bf1e4a6184bbd48fe4",
-                "sha256:3a3c3f8863255f3c31db3889f8055989527173ef6192a283eb6f4db3c579d830",
-                "sha256:3b14b1da110ea50c8bcbadc3b82c3933974dbeea1832e814aab93ca1163cd4c1",
-                "sha256:535dc1e6e68fad5355f9984d5637c33badbdc987b0c0d303ee95a6c979c9516f",
-                "sha256:6f61319e33222591f885c598e3e24f6a4be3533c1d70c19e0dc59e83a71ce27d",
-                "sha256:723d22d324e7997a651478e9c5a3120a0ecbc9a7e94071f7e1954562a8806cf3",
-                "sha256:76b2775dda7e78680d688daabcb485dc87cf5e3184a0b3e012e1d40e38527cc8",
-                "sha256:782a5c7df9f91979a7a21792e09b34a658058896628217ae6362088b123c8500",
-                "sha256:7e4d159021c2029b958b2363abec4a11db0ce8cd43abb0d9ce44284cb97217e7",
-                "sha256:8dacc4073c359f40fcf73aede8428c35f84639baad7e1b46fce5ab7a8a7be4bb",
-                "sha256:8f33d1156241c43755137288dea619105477961cfa7e47f48dbf96bc2c30720b",
-                "sha256:8ffd4b204d7de77b5dd558cdff986a8274796a1e57813ed005b33fd97e29f059",
-                "sha256:93a280c9eb736a0dcca19296f3c30c720cb41a71b1f9e617f341f0a8e791a69b",
-                "sha256:9a4f66259bdd6964d8cf26142733c81fb562252db74ea367d9beb4f815478e72",
-                "sha256:9a9d4ff06804920388aab69c5ea8a77525cf165356db70131616acd269e19b36",
-                "sha256:a2070c5affdb3a5e751f24208c5c4f3d5f008fa04d28731416e023c93b275277",
-                "sha256:a4857f7e2bc6921dbd487c5c88b84f5633de3e7d416c4dc0bb70256775551a6c",
-                "sha256:a607ae05b6c96057ba86c811d9c43423f35e03874ffb03fbdcd45e0637e8b631",
-                "sha256:a66ca3bdf21c653e47f726ca57f46ba7fc1f260ad99ba783acc3e58e3ebdb9ff",
-                "sha256:ab110c48bc3d97b4d19af41865e14531f300b482da21783fdaacd159251890e8",
-                "sha256:b239711e774c8eb910e9b1ac719f02f5ae4bf35fa0420f438cdc3a7e4e7dd6ec",
-                "sha256:be0416074d7f253865bb67630cf7210cbc14eb05f4099cc0f82430135aaa7a3b",
-                "sha256:c46643970dff9f5c976c6512fd35768c4a3819f01f61169d8cdac3f9290903b7",
-                "sha256:c5ec71fd4a43b6d84ddb88c1df94572479d9a26ef3f150cef3dacefecf888105",
-                "sha256:c6e5174f8ca585755988bc278c8bb5d02d9dc2e971591ef4a1baabdf2d99589b",
-                "sha256:c89b558f8a9a5a6f2cfc923c304d49f0ce629c3bd85cb442ca258ec20366394c",
-                "sha256:cc44e3545d908ecf3e5773266c487ad1877be718d9dc65fc7eb6e7d14960985b",
-                "sha256:cc6f8246e74dd210d7e2b56c76ceaba1cc52b025cd75dbe96eb48791e0250e98",
-                "sha256:cd556c79ad665faeae28020a0ab3bda6cd47d94bec48e36970719b0b86e4dcf4",
-                "sha256:ce6f3a147b4b1a8b09aae48517ae91139b1b010c5f36423fa2b866a8b23df879",
-                "sha256:ceb499d2b3d1d7b7ba23abe8bf26df5f06ba8c71127f188333dddcf356b4b63f",
-                "sha256:cef06fb382557f66d81d804230c11ab292d94b840b3cb7bf4450778377b592f4",
-                "sha256:e448f56cfeae7b1b3b5bcd99bb377cde7c4eb1970a525c770720a352bc4c8044",
-                "sha256:e52d3d95df81c8f6b2a1685aabffadf2d2d9ad97203a40f8d61e51b70f191e4e",
-                "sha256:eb33c4c858d06bd8d79713c7628d3f2b50fb1c62071e2e88cb44876be03eabe1",
-                "sha256:ee2f1d1c223c3d2c24e3afbb2dd38be3f03b1a8d6a83ee3d9eb8c36a52bee899",
-                "sha256:f2c6888eada180814b8583c3e793f3f343a692fc802546eed45f40a001b1169f",
-                "sha256:f51dbba78d68a44e99d484ca8c8f604f17e957c1ca09c3ebc2c7e3bbd9ba0448",
-                "sha256:f54de00baf200b4539a5a092a759f000b5f45fd226d6d25a76b0dff71177a714",
-                "sha256:fa10fee7e32213f5c7b0d6428ea92e3a3fdd6d725590238a3f92c0de1c78b9d2",
-                "sha256:fabeeb121735d47d8eab8671b6b031ce08514c86b7ad8f7d5490a7b6dcd6267d",
-                "sha256:fac3c432851038b3e6afe086f777732bcf7f6ebbfd90951fa04ee53db6d0bcdd",
-                "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7",
-                "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae"
+                "sha256:03ed2a641e412e42cc35c244508cf186015c217f0e4d496bf6d7078ebe837ae7",
+                "sha256:04b14e45d6a8e159c9767ae57ecb34563ad93440fc1b26516a89ceb5b33c1ad5",
+                "sha256:0cdde51bfcf6b6bd862ee9be324521ec619b20590787d1655d005c3fb175005f",
+                "sha256:0f48fc7dc82ee14aeaedb986e175a429d24129b7eada1b7e94a864e4f0644dde",
+                "sha256:107d327071061fd4f4a2587d14c389a27e4e5c93c7cba5f1f59987181903902f",
+                "sha256:1375bb8b88cb050a2d4e0da901001347a44302aeadb8ceb4b6e5aa373b8ea68f",
+                "sha256:14a9f1887591684fb59fdba8feef7123a0da2424b0652e1b58dd5b9a7bb1188c",
+                "sha256:16baa799ec09cc0dcb43a10680573269d407c159325972dd7114ee7649e56c66",
+                "sha256:1b811662ecf72eb2d08872731636aee6559cae21862c36f74703be727b45df90",
+                "sha256:1ccae21a076d3d5f471700f6d30eb486da1626c380b23c70ae32ab823e453337",
+                "sha256:2f2cf7a42d4b7654c9a67b9d091ec24374f7c58794858bff632a2039cb15984d",
+                "sha256:322549b880b2d746a7672bf6ff9ed3f895e9c9f108b714e7360292aa5c5d7cf4",
+                "sha256:32ab83016c24c5cf3db2943286b85b0a172dae08c58d0f53875235219b676409",
+                "sha256:3fe50f1cac369b02d34ad904dfe0771acc483f82a1b54c5e93632916ba847b37",
+                "sha256:4a780807e80479f281d47ee4af2eb2df3e4ccf4723484f77da0bb49d027e40a1",
+                "sha256:4a8eb7785bd23565b542b01fb39115a975fefb4a82f23d407503eee2c0106247",
+                "sha256:4b40b794775df10d7e3ea677108dd581bfec796235750c617d461874178a67f6",
+                "sha256:5bee3970617b3d74759b2d2df2f6a327d372f9732f9ccbf03fa591b5f7581e39",
+                "sha256:60a3307a84ec60578accd35d7f0c71a3a971430ed7eca6567399d2b50ef37b8c",
+                "sha256:6625e52b6f346a283c3d563d1fd8bae8956daafc64bb5bbd2b8f8a07608e3994",
+                "sha256:66a5aae8233d766a877c5ef293ec5ab9520929c2578fd2069308a98b7374ea8c",
+                "sha256:68fb816a5dd901c6aff352ce49e2a0ffadacdf9b6fae282a69e7a16a02dad5fb",
+                "sha256:6b588b5cf51dc0fd1c9e19f622457cc74b7d26fe295432e434525f1c0fae02bc",
+                "sha256:6c4d7165a4e8f41eca6b990c12ee7f44fef3932fac48ca32cecb3a1b2223c21f",
+                "sha256:6d2e262e5e8da6fa56e774fb8e2643417351427604c2b177f8e8c5f75fc928ca",
+                "sha256:6d9c88b787638a451f41f97446a1c9fd416e669b4d9717ae4615bd29de1ac135",
+                "sha256:755c56beeacac6a24c8e1074f89f34f4373abce8b662470d3aa719ae304931f3",
+                "sha256:7e40d3f8eb472c1509b12ac2a7e24158ec352fc8567b77ab02c0db053927e339",
+                "sha256:812eaf4939ef2284d29653bcfee9665f11f013724f07258928f849a2306ea9f9",
+                "sha256:84df004223fd0550d0ea7a37882e5c889f3c6d45535c639ce9802293b39cd5c9",
+                "sha256:859f0add98707b182b4867359e12bde806b82483fb12a9ae868a77880fc3b7af",
+                "sha256:87c4b38288f71acd2106f5d94f575bc2136ea2887fdb5dfe18003c881fa6b370",
+                "sha256:89fc12c6371bf963809abc46cced4a01ca4f99cba17be5e7d416ed7ef1245d19",
+                "sha256:9564ac7eb1652c3701ac691ca72934dd3009997c81266807aef924012df2f4b3",
+                "sha256:9754a5c265f991317de2bac0c70a746efc2b695cf4d49f5d2cddeac36544fb44",
+                "sha256:a565f48c4aae72d1d3d3f8e8fb7218f5609c964e9c6f68604608e5958b9c60c3",
+                "sha256:a636160680c6e526b84f85d304e2f0bb4e94f8284dd765a1911de9a40450b10a",
+                "sha256:a839e25f07e428a87d17d857d9935dd743130e77ff46524abb992b962eb2076c",
+                "sha256:ae9702c099546e72000d76758b5efec2dd937ba5d746ec8d0563d2fca0f9bc2e",
+                "sha256:b62046592b44263fa7570f1117d372ae3f310222af1fc1407416f037fb3af21b",
+                "sha256:b7f7421841f8db443855d2854e25914a79a1ff48ae92f70d0a5c2f8907ab98c9",
+                "sha256:ba7ca81b6d60a9f7a0b4b4e175dcc38e8fef4992673d9d6e6879fd6de00dd9b8",
+                "sha256:bb32ca14b4d04e172c541c69eec5f385f9a075b38fb22d765d8b0ce3af3a0c22",
+                "sha256:c0ff1c1b4d13e2240821ef23c1efb1f009207cb3f56e16986f713c2b0e7cd37f",
+                "sha256:c669b440ce46ae3abe9b2d44a913b5fd86bb19eb14a8701e88e3918902ecd345",
+                "sha256:c67734cff78383a1f23ceba3b3239c7deefc62ac2b05fa6a47bcd565771e5880",
+                "sha256:c6809ebcbf6c1049002b9ac09c127ae43929042ec1f1dbd8bb1615f7cd9f70a0",
+                "sha256:cd601187476c6bed26a0398353212684c427e10a903aeafa6da40c63309d438b",
+                "sha256:ebfa374067af240d079ef97b8064478f3bf71038b78b017eb6ec93ede1b6bcec",
+                "sha256:fbb17c0d0822684b7d6c09915677a32319f16ff1115df5ec05bdcaaee40b35f3",
+                "sha256:fff1f3a586246110f34dc762098b5afd2de88de507559e63553d7da643053786"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==5.3.1"
+            "version": "==5.4"
         },
         "coveralls": {
             "hashes": [
@@ -1371,11 +1367,11 @@
         },
         "faker": {
             "hashes": [
-                "sha256:1a40c9696830342e67d30057753323478b848e89334e97e0a64b442b18ae7ece",
-                "sha256:dd98418eceb374c475e85bd49ae826a9b304dabcd794640f35a1d185f4767dd9"
+                "sha256:0783729c61501d52efea2967aff6e6fcb8370f0f6b5a558f2a81233642ae529a",
+                "sha256:6b2995ffff6c2b02bc5daad96f8c24c021e5bd491d9d53d31bcbd66f348181d4"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==5.7.0"
+            "version": "==5.8.0"
         },
         "filelock": {
             "hashes": [

From 7c02a8b414a56fb4ab4f9a0a8962a9f2d94bdd4a Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 26 Jan 2021 18:01:24 +0100
Subject: [PATCH 29/54] requirements

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index c70d13ff4..a5a564a99 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -72,7 +72,7 @@ redis==3.5.3
 regex==2020.11.13
 reportlab==3.5.60
 requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-scikit-learn==0.24.1
+scikit-learn==0.24.0
 scipy==1.5.4; python_version >= '3.6'
 service-identity==18.1.0
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'

From 55584a6a876cc4ac2a263482347fcfe758265b95 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 09:50:48 -0800
Subject: [PATCH 30/54] Fix alert resizing

---
 .../upload-file-widget.component.html                  |  4 ++--
 .../upload-file-widget.component.scss                  | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index d00114934..213f46fd6 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -19,10 +19,10 @@
     <div *ngFor="let status of getStatus()">
       <ngb-alert type="secondary" class="mt-2 mb-0" [dismissible]="isFinished(status)" (closed)="dismiss(status)">
         <h6 class="alert-heading">{{status.filename}}</h6>
-        <p class="mb-0" *ngIf="!isFinished(status) || (isFinished(status) && !status.documentId)">{{status.message}}</p>
+        <p class="mb-0 pb-1" *ngIf="!isFinished(status) || (isFinished(status) && !status.documentId)">{{status.message}}</p>
         <ngb-progressbar [value]="status.getProgress()" [max]="1" [type]="getStatusColor(status)"></ngb-progressbar>
         <div *ngIf="isFinished(status)">
-          <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary mr-2" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">
+          <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary btn-open" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">
             <small>Open document</small>
             <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
               <path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
index 10a83e308..2af4a4029 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
@@ -8,17 +8,17 @@ form {
   position: relative;
 }
 
-.alert {
-  transition: all 0.2s ease;
-}
-
 .alert-heading {
   font-size: 80%;
   font-weight: bold;
 }
 
-.btn-sm {
+.btn-open {
   line-height: 1;
+
+  svg {
+    margin-top: -1px;
+  }
 }
 
 .dismiss-all {

From d60b532b8f07582f2b3670d729587e820386aa7d Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 10:09:13 -0800
Subject: [PATCH 31/54] Dark mode support

---
 .../upload-file-widget.component.scss                  |  4 ----
 src-ui/src/styles.scss                                 |  4 ++++
 src-ui/src/theme_dark.scss                             | 10 ++++++++++
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
index 2af4a4029..ed1bca319 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
@@ -1,9 +1,5 @@
 @import "/src/theme";
 
-::ng-deep .ngx-file-drop__drop-zone--over {
-  background-color: $primaryFaded !important;
-}
-
 form {
   position: relative;
 }
diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss
index 8cf4a93f6..34f575a05 100644
--- a/src-ui/src/styles.scss
+++ b/src-ui/src/styles.scss
@@ -111,3 +111,7 @@ body {
     font-size: 16px;
   }
 }
+
+.ngx-file-drop__drop-zone--over {
+  background-color: $primaryFaded !important;
+}
diff --git a/src-ui/src/theme_dark.scss b/src-ui/src/theme_dark.scss
index 9a698143d..bd4227fc6 100644
--- a/src-ui/src/theme_dark.scss
+++ b/src-ui/src/theme_dark.scss
@@ -352,6 +352,16 @@ $border-color-dark-mode: #47494f;
   .bg-dark {
     background-color: $bg-light-dark-mode !important;
   }
+
+  .ngx-file-drop__drop-zone--over {
+    background-color: darken($primary-dark-mode, 35%) !important;
+  }
+
+  .alert-secondary {
+    background-color: $bg-light-dark-mode;
+    border-color: darken($bg-light-dark-mode, 10%);
+    color: $text-color-dark-mode-accent;
+  }
 }
 
 body.color-scheme-dark {

From 44600961d8671d7a36f635a27c580238de3fb5e1 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 10:17:45 -0800
Subject: [PATCH 32/54] Fix initial upload progress > 100%

---
 src-ui/src/app/services/consumer-status.service.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index b2804103d..181e45e15 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -33,7 +33,6 @@ export class FileStatus {
       case FileStatusPhase.UPLOADING:
         return this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.2
       case FileStatusPhase.PROCESSING:
-        if (this.currentPhaseProgress > 100) this.currentPhaseProgress = 0
         return (this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.8) + 0.2
       case FileStatusPhase.SUCCESS:
       case FileStatusPhase.FAILED:
@@ -44,7 +43,7 @@ export class FileStatus {
   updateProgress(status: FileStatusPhase, currentProgress?: number, maxProgress?: number) {
     if (status >= this.phase) {
       this.phase = status
-      if (currentProgress) {
+      if (currentProgress != undefined) {
         this.currentPhaseProgress = currentProgress
       }
       if (maxProgress) {

From 52f17a9ba63472d8863e577e3f72c39e911cca95 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 10:17:58 -0800
Subject: [PATCH 33/54] Fix dark mode primary progress color

---
 src-ui/src/theme_dark.scss | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src-ui/src/theme_dark.scss b/src-ui/src/theme_dark.scss
index bd4227fc6..4e850f017 100644
--- a/src-ui/src/theme_dark.scss
+++ b/src-ui/src/theme_dark.scss
@@ -362,6 +362,10 @@ $border-color-dark-mode: #47494f;
     border-color: darken($bg-light-dark-mode, 10%);
     color: $text-color-dark-mode-accent;
   }
+
+  .progress-bar.bg-primary {
+    background-color: darken($primary-dark-mode, 5%) !important;
+  }
 }
 
 body.color-scheme-dark {

From 51dc40ae734cc419ae058a7dc0614a02ece4c659 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 26 Jan 2021 14:48:39 -0800
Subject: [PATCH 34/54] Maximum limit of alerts

---
 .../upload-file-widget.component.html         | 38 ++++++++++++-------
 .../upload-file-widget.component.scss         |  6 +++
 .../upload-file-widget.component.ts           |  9 ++++-
 3 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index 213f46fd6..c8104dae1 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -1,4 +1,4 @@
-<app-widget-frame title="Upload new documents" i18n-title [class.has-multiple-status]="getStatus().length > 1">
+<app-widget-frame title="Upload new documents" i18n-title [class.has-multiple-status]="(getStatus().length + getStatusesHidden().length)> 1">
   <div header-buttons>
     <button type="button" class="btn btn-link dismiss-all" [disabled]="!getStatus().length" (click)="dismissAll()">
       <small class="mr-1">Hide all</small>
@@ -17,19 +17,31 @@
       </ngx-file-drop>
     </form>
     <div *ngFor="let status of getStatus()">
-      <ngb-alert type="secondary" class="mt-2 mb-0" [dismissible]="isFinished(status)" (closed)="dismiss(status)">
-        <h6 class="alert-heading">{{status.filename}}</h6>
-        <p class="mb-0 pb-1" *ngIf="!isFinished(status) || (isFinished(status) && !status.documentId)">{{status.message}}</p>
-        <ngb-progressbar [value]="status.getProgress()" [max]="1" [type]="getStatusColor(status)"></ngb-progressbar>
-        <div *ngIf="isFinished(status)">
-          <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary btn-open" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">
-            <small>Open document</small>
-            <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
-              <path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
-            </svg>
-          </button>
+      <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>
+    </div>
+    <div *ngIf="getStatusesHidden().length" class="alerts-hidden">
+      <p *ngIf="!alertsExpanded" class="mt-3 mb-0 text-center">{{getStatusesHidden().length}} more hidden <button class="btn btn-sm btn-link py-0" (click)="alertsExpanded = !alertsExpanded" aria-controls="hiddenAlerts" [attr.aria-expanded]="alertsExpanded">Show all</button></p>
+      <div #hiddenAlerts="ngbCollapse" [(ngbCollapse)]="!alertsExpanded">
+        <div *ngFor="let status of getStatusesHidden()">
+          <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>
         </div>
-      </ngb-alert>
+      </div>
     </div>
   </div>
 </app-widget-frame>
+
+<ng-template #consumerAlert let-status>
+  <ngb-alert type="secondary" class="mt-2 mb-0" [dismissible]="isFinished(status)" (closed)="dismiss(status)">
+    <h6 class="alert-heading">{{status.filename}}</h6>
+    <p class="mb-0 pb-1" *ngIf="!isFinished(status) || (isFinished(status) && !status.documentId)">{{status.message}}</p>
+    <ngb-progressbar [value]="status.getProgress()" [max]="1" [type]="getStatusColor(status)"></ngb-progressbar>
+    <div *ngIf="isFinished(status)">
+      <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary btn-open" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">
+        <small>Open document</small>
+        <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
+          <path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
+        </svg>
+      </button>
+    </div>
+  </ngb-alert>
+</ng-template>
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
index ed1bca319..5e404bb8e 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
@@ -9,6 +9,12 @@ form {
   font-weight: bold;
 }
 
+.alerts-hidden {
+  .btn {
+    line-height: 1;
+  }
+}
+
 .btn-open {
   line-height: 1;
 
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index 19ab8d04b..27f67f976 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -4,6 +4,7 @@ import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop';
 import { ConsumerStatusService, FileStatus, FileStatusPhase } from 'src/app/services/consumer-status.service';
 import { DocumentService } from 'src/app/services/rest/document.service';
 
+const MAX_ALERTS = 5
 
 @Component({
   selector: 'app-upload-file-widget',
@@ -11,6 +12,7 @@ import { DocumentService } from 'src/app/services/rest/document.service';
   styleUrls: ['./upload-file-widget.component.scss']
 })
 export class UploadFileWidgetComponent implements OnInit {
+  alertsExpanded = false
 
   constructor(
     private documentService: DocumentService,
@@ -18,7 +20,12 @@ export class UploadFileWidgetComponent implements OnInit {
   ) { }
 
   getStatus() {
-    return this.consumerStatusService.getConsumerStatus()
+    return this.consumerStatusService.getConsumerStatus().slice(0, MAX_ALERTS)
+  }
+
+  getStatusesHidden() {
+    if (this.consumerStatusService.getConsumerStatus().length < MAX_ALERTS) return []
+    else return this.consumerStatusService.getConsumerStatus().slice(MAX_ALERTS)
   }
 
   getStatusUploading() {

From 68a9dd7e052908c7ad70e4ce0f46e60252b478c7 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 15:23:29 +0100
Subject: [PATCH 35/54] fix an issue with upload status identification

---
 src-ui/src/app/services/consumer-status.service.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index 181e45e15..90ba4f0ef 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -43,10 +43,10 @@ export class FileStatus {
   updateProgress(status: FileStatusPhase, currentProgress?: number, maxProgress?: number) {
     if (status >= this.phase) {
       this.phase = status
-      if (currentProgress != undefined) {
+      if (currentProgress != null) {
         this.currentPhaseProgress = currentProgress
       }
-      if (maxProgress) {
+      if (maxProgress != null) {
         this.currentPhaseMaxProgress = maxProgress
       }
     }
@@ -70,7 +70,7 @@ export class ConsumerStatusService {
   private documentConsumptionFailedSubject = new Subject<FileStatus>()
 
   private get(taskId: string, filename?: string) {
-    let status = this.consumerStatus.find(e => e.taskId == taskId) || this.consumerStatus.find(e => e.filename == filename)
+    let status = this.consumerStatus.find(e => e.taskId == taskId) || this.consumerStatus.find(e => e.filename == filename && e.taskId == null)
     if (!status) {
       status = new FileStatus()
       this.consumerStatus.push(status)

From cc74be9ccc574548ffb99efba796afc568d83a18 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 15:50:37 +0100
Subject: [PATCH 36/54] better error messages

---
 .../upload-file-widget/upload-file-widget.component.ts    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index 27f67f976..8c2dfec07 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -87,15 +87,15 @@ export class UploadFileWidgetComponent implements OnInit {
           formData.append('document', file, file.name)
           let status = this.consumerStatusService.newFileUpload(file.name)
 
-          status.message = "Connecting..."
+          status.message = $localize`Connecting...`
 
           this.documentService.uploadDocument(formData).subscribe(event => {
             if (event.type == HttpEventType.UploadProgress) {
               status.updateProgress(FileStatusPhase.UPLOADING, event.loaded, event.total)
-              status.message = "Uploading..."
+              status.message = $localize`Uploading...`
             } else if (event.type == HttpEventType.Response) {
               status.taskId = event.body["task_id"]
-              status.message = "Upload complete."
+              status.message = $localize`Waiting for consumer...`
             }
 
           }, error => {
@@ -106,7 +106,7 @@ export class UploadFileWidgetComponent implements OnInit {
                 break;
               }
               default: {
-                status.message = $localize`An error has occurred while uploading the document. Sorry!`
+                status.message = `${error.status} ${error.statusText}`
                 break;
               }
             }

From 5a4cb5fe4a805b26301b039ce0f1d50625dc02be Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 16:04:06 +0100
Subject: [PATCH 37/54] proper messages when uploading fails

---
 .../upload-file-widget/upload-file-widget.component.ts    | 5 ++---
 src-ui/src/app/services/consumer-status.service.ts        | 8 +++++++-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index 8c2dfec07..1c5ed5438 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -99,14 +99,13 @@ export class UploadFileWidgetComponent implements OnInit {
             }
 
           }, error => {
-            status.updateProgress(FileStatusPhase.FAILED)
             switch (error.status) {
               case 400: {
-                status.message = error.error.document
+                this.consumerStatusService.fail(status, error.error.document)
                 break;
               }
               default: {
-                status.message = `${error.status} ${error.statusText}`
+                this.consumerStatusService.fail(status, `${error.status} ${error.statusText}`)
                 break;
               }
             }
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index 90ba4f0ef..a3c942499 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -117,6 +117,12 @@ export class ConsumerStatusService {
     }
   }
 
+  fail(status: FileStatus, message: string) {
+    status.message = message
+    status.phase = FileStatusPhase.FAILED
+    this.documentConsumptionFailedSubject.next(status)
+  }
+
   disconnect() {
     if (this.statusWebSocked) {
       this.statusWebSocked.close()
@@ -133,7 +139,7 @@ export class ConsumerStatusService {
   }
 
   dismissAll() {
-    this.consumerStatus = []
+    this.consumerStatus = this.consumerStatus.filter(status => status.phase < FileStatusPhase.SUCCESS)
   }
 
   onDocumentConsumptionFinished() {

From 91f7c4e6855c384f133e2714e3d6104205a486ea Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 17:28:11 +0100
Subject: [PATCH 38/54] settings and localization

---
 src-ui/messages.xlf                           | 166 ++++++++++++++----
 src-ui/src/app/app.component.ts               |  41 ++++-
 .../upload-file-widget.component.html         |   4 +-
 .../manage/settings/settings.component.html   |  14 ++
 .../manage/settings/settings.component.ts     |   8 +
 .../app/services/consumer-status.service.ts   |  18 +-
 src-ui/src/app/services/settings.service.ts   |  12 +-
 7 files changed, 214 insertions(+), 49 deletions(-)

diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf
index 6175cf700..ec718acc6 100644
--- a/src-ui/messages.xlf
+++ b/src-ui/messages.xlf
@@ -2,6 +2,48 @@
 <xliff version="1.2" xmlns="urn:oasis:names:tc:xliff:document:1.2">
   <file source-language="en-US" datatype="plaintext" original="ng2.template">
     <body>
+      <trans-unit id="9103526311244275943" datatype="html">
+        <source>Document added</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/app.component.ts</context>
+          <context context-type="linenumber">51</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="9204248378636247318" datatype="html">
+        <source>Document <x id="PH" equiv-text="status.filename"/> was added to paperless.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/app.component.ts</context>
+          <context context-type="linenumber">51</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="1931214133925051574" datatype="html">
+        <source>Open document</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/app.component.ts</context>
+          <context context-type="linenumber">51</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="8582620835547864448" datatype="html">
+        <source>Could not add <x id="PH" equiv-text="status.filename"/>: <x id="PH_1" equiv-text="status.message"/></source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/app.component.ts</context>
+          <context context-type="linenumber">59</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="1710712016675379662" datatype="html">
+        <source>New document detected</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/app.component.ts</context>
+          <context context-type="linenumber">65</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="587031278561344416" datatype="html">
+        <source>Document <x id="PH" equiv-text="status.filename"/> is being processed by paperless.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/app.component.ts</context>
+          <context context-type="linenumber">65</context>
+        </context-group>
+      </trans-unit>
       <trans-unit id="4733307402565258070" datatype="html">
         <source>Documents</source>
         <context-group purpose="location">
@@ -482,35 +524,35 @@
         <source>Saved view &quot;<x id="PH" equiv-text="savedView.name"/>&quot; deleted.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
-          <context context-type="linenumber">63</context>
+          <context context-type="linenumber">67</context>
         </context-group>
       </trans-unit>
       <trans-unit id="5647210819299459618" datatype="html">
         <source>Settings saved successfully.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
-          <context context-type="linenumber">79</context>
+          <context context-type="linenumber">87</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6839066544204061364" datatype="html">
         <source>Use system language</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
-          <context context-type="linenumber">83</context>
+          <context context-type="linenumber">91</context>
         </context-group>
       </trans-unit>
       <trans-unit id="7729897675462249787" datatype="html">
         <source>Use date format of display language</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
-          <context context-type="linenumber">87</context>
+          <context context-type="linenumber">95</context>
         </context-group>
       </trans-unit>
       <trans-unit id="8488620293789898901" datatype="html">
         <source>Error while storing settings on server: <x id="PH" equiv-text="JSON.stringify(error.error)"/></source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.ts</context>
-          <context context-type="linenumber">103</context>
+          <context context-type="linenumber">111</context>
         </context-group>
       </trans-unit>
       <trans-unit id="121cc5391cd2a5115bc2b3160379ee5b36cd7716" datatype="html">
@@ -531,7 +573,7 @@
         <source>Saved views</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">114</context>
+          <context context-type="linenumber">128</context>
         </context-group>
       </trans-unit>
       <trans-unit id="bbe41ac2ea4a6c00ea941a41b33105048f8e9f13" datatype="html">
@@ -639,60 +681,109 @@
           <context context-type="linenumber">98</context>
         </context-group>
       </trans-unit>
+      <trans-unit id="8bcabdf6b16cad0313a86c7e940c5e3ad7f9f8ab" datatype="html">
+        <source>Notifications</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
+          <context context-type="linenumber">102</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="69c5a98f8aa92e4db060f10dcd37781c8f40a48f" datatype="html">
+        <source>Consumer status</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
+          <context context-type="linenumber">106</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="2ad4d76b36341c589d94004ad2a213fd4d6f5ca0" datatype="html">
+        <source>Show notifications when new documents are detected</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
+          <context context-type="linenumber">109</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="f2361d3f65b6c77ef0a15fad8af8e858b043ace3" datatype="html">
+        <source>Show notifications when document consumption completes successfully</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
+          <context context-type="linenumber">110</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="2bcbcbe99e207803e21183580b98d90410dd8718" datatype="html">
+        <source>Show notifications when document consumption fails</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
+          <context context-type="linenumber">111</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="af113f7c9f7e13145c3461f61a1aedf12d57bd71" datatype="html">
+        <source>Suppress notifications on dashboard</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
+          <context context-type="linenumber">112</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="59fe5c9233a2aaab3079a2300e4dfa439ddb1890" datatype="html">
+        <source>This will suppress all consumer related status messages on the dashboard.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
+          <context context-type="linenumber">112</context>
+        </context-group>
+      </trans-unit>
       <trans-unit id="3863a86cd9e69a61d143d3daf51df44203df4a82" datatype="html">
         <source>Bulk editing</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">102</context>
+          <context context-type="linenumber">116</context>
         </context-group>
       </trans-unit>
       <trans-unit id="c0ac61661c6c326d6e0e00c231b95cf2ac0c6586" datatype="html">
         <source>Show confirmation dialogs</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">106</context>
+          <context context-type="linenumber">120</context>
         </context-group>
       </trans-unit>
       <trans-unit id="291bbe56ecbe945dcf05580a57d679fa7bd1e06a" datatype="html">
         <source>Deleting documents will always ask for confirmation.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">106</context>
+          <context context-type="linenumber">120</context>
         </context-group>
       </trans-unit>
       <trans-unit id="8cfddc13e04f5545ac63f419ef363505d6f78c2e" datatype="html">
         <source>Apply on close</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">107</context>
+          <context context-type="linenumber">121</context>
         </context-group>
       </trans-unit>
       <trans-unit id="8cb90334f5dfd7fc67205085f59381e2a334ccfc" datatype="html">
         <source>Appears on</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">126</context>
+          <context context-type="linenumber">140</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6717cf1acf04728fc2b7c39f6d3297f8ff15fde5" datatype="html">
         <source>Show on dashboard</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">129</context>
+          <context context-type="linenumber">143</context>
         </context-group>
       </trans-unit>
       <trans-unit id="541bfc5b123b3f8867fd681eaceefb663a811973" datatype="html">
         <source>Show in sidebar</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">133</context>
+          <context context-type="linenumber">147</context>
         </context-group>
       </trans-unit>
       <trans-unit id="abba764a7a595d04dc8c3b26e04b3780d4fdb540" datatype="html">
         <source>No saved views defined.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/manage/settings/settings.component.html</context>
-          <context context-type="linenumber">143</context>
+          <context context-type="linenumber">157</context>
         </context-group>
       </trans-unit>
       <trans-unit id="ef60a738a565f498b858e903e42bc5ffc3cc1299" datatype="html">
@@ -1323,25 +1414,25 @@
           <context context-type="linenumber">4</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="8705589528094706681" datatype="html">
-        <source>The document has been uploaded and will be processed by the consumer shortly.</source>
+      <trans-unit id="3852289441366561594" datatype="html">
+        <source>Connecting...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">63</context>
+          <context context-type="linenumber">90</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="4956689020592747108" datatype="html">
-        <source>There was an error while uploading the document: <x id="PH" equiv-text="error.error.document"/></source>
+      <trans-unit id="1245343823699368872" datatype="html">
+        <source>Uploading...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">71</context>
+          <context context-type="linenumber">95</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="7554858521017940575" datatype="html">
-        <source>An error has occurred while uploading the document. Sorry!</source>
+      <trans-unit id="3994065460580948013" datatype="html">
+        <source>Waiting for consumer...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">75</context>
+          <context context-type="linenumber">98</context>
         </context-group>
       </trans-unit>
       <trans-unit id="e022072b3e4dd77e3f09960817ef3359a49963b3" datatype="html">
@@ -1351,25 +1442,32 @@
           <context context-type="linenumber">1</context>
         </context-group>
       </trans-unit>
+      <trans-unit id="bd4a8607e4a002d939cffb347ec056664dfb2c73" datatype="html">
+        <source>Dismiss completed</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
+          <context context-type="linenumber">4</context>
+        </context-group>
+      </trans-unit>
       <trans-unit id="52d5d56b6667ba3f7c759cbc0d4c39d34a99701e" datatype="html">
         <source>Drop documents here or</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
-          <context context-type="linenumber">5</context>
+          <context context-type="linenumber">13</context>
         </context-group>
       </trans-unit>
       <trans-unit id="865c511f4a24558ed0e954f9bbbff557bbb8954d" datatype="html">
         <source>Browse files</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
-          <context context-type="linenumber">5</context>
+          <context context-type="linenumber">13</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="33c76d75ce25ce3b05ab22877f1b6b09dcf603ae" datatype="html">
-        <source>{VAR_PLURAL, plural, =1 {Uploading file...} =other {Uploading <x id="INTERPOLATION"/> files...}}</source>
+      <trans-unit id="710254a196a2649674438edf8a15b7ab1f48271b" datatype="html">
+        <source>Open document</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
-          <context context-type="linenumber">13</context>
+          <context context-type="linenumber">40</context>
         </context-group>
       </trans-unit>
       <trans-unit id="45854ddec74086b271e62be6a363f4fa5036f7fc" datatype="html">
@@ -1467,42 +1565,42 @@
         <source>English (US)</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/services/settings.service.ts</context>
-          <context context-type="linenumber">74</context>
+          <context context-type="linenumber">82</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1858110241312746425" datatype="html">
         <source>German</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/services/settings.service.ts</context>
-          <context context-type="linenumber">75</context>
+          <context context-type="linenumber">83</context>
         </context-group>
       </trans-unit>
       <trans-unit id="3071065188816255493" datatype="html">
         <source>Dutch</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/services/settings.service.ts</context>
-          <context context-type="linenumber">76</context>
+          <context context-type="linenumber">84</context>
         </context-group>
       </trans-unit>
       <trans-unit id="7633754075223722162" datatype="html">
         <source>French</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/services/settings.service.ts</context>
-          <context context-type="linenumber">77</context>
+          <context context-type="linenumber">85</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1519954996184640001" datatype="html">
         <source>Error</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/services/toast.service.ts</context>
-          <context context-type="linenumber">31</context>
+          <context context-type="linenumber">35</context>
         </context-group>
       </trans-unit>
       <trans-unit id="5037437391296624618" datatype="html">
         <source>Information</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/services/toast.service.ts</context>
-          <context context-type="linenumber">35</context>
+          <context context-type="linenumber">39</context>
         </context-group>
       </trans-unit>
       <trans-unit id="7517688192215738656" datatype="html">
diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts
index 43c23f2af..836a6f66a 100644
--- a/src-ui/src/app/app.component.ts
+++ b/src-ui/src/app/app.component.ts
@@ -1,9 +1,9 @@
-import { SettingsService } from './services/settings.service';
+import { SettingsService, SETTINGS_KEYS } from './services/settings.service';
 import { Component, OnDestroy, OnInit } from '@angular/core';
 import { Router } from '@angular/router';
 import { Subscription } from 'rxjs';
 import { ConsumerStatusService } from './services/consumer-status.service';
-import { Toast, ToastService } from './services/toast.service';
+import { ToastService } from './services/toast.service';
 
 @Component({
   selector: 'app-root',
@@ -12,6 +12,7 @@ import { Toast, ToastService } from './services/toast.service';
 })
 export class AppComponent implements OnInit, OnDestroy {
 
+  newDocumentSubscription: Subscription;
   successSubscription: Subscription;
   failedSubscription: Subscription;
 
@@ -23,23 +24,47 @@ export class AppComponent implements OnInit, OnDestroy {
 
   ngOnDestroy(): void {
     this.consumerStatusService.disconnect()
-    this.successSubscription.unsubscribe()
-    this.failedSubscription.unsubscribe()
+    if (this.successSubscription) {
+      this.successSubscription.unsubscribe()
+    }
+    if (this.failedSubscription) {
+      this.failedSubscription.unsubscribe()
+    }
+    if (this.newDocumentSubscription) {
+      this.newDocumentSubscription.unsubscribe()
+    }
+  }
+
+  private showNotification(key) {
+    if (this.router.url == '/dashboard' && this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD)) {
+      return false
+    }
+    return this.settings.get(key)
   }
 
   ngOnInit(): void {
     this.consumerStatusService.connect()
 
+    
     this.successSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => {
-      this.toastService.show({title: "Document added", delay: 10000, content: `Document ${status.filename} was added to paperless.`, actionName: "Open document", action: () => {
-        this.router.navigate(['documents', status.documentId])
-      }})
+      if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)) {
+        this.toastService.show({title: $localize`Document added`, delay: 10000, content: $localize`Document ${status.filename} was added to paperless.`, actionName: $localize`Open document`, action: () => {
+          this.router.navigate(['documents', status.documentId])
+        }})
+      }
     })
 
     this.failedSubscription = this.consumerStatusService.onDocumentConsumptionFailed().subscribe(status => {
-      this.toastService.showError(`Could not consume ${status.filename}: ${status.message}`)
+      if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)) {
+        this.toastService.showError($localize`Could not add ${status.filename}\: ${status.message}`)
+      }
     })
 
+    this.newDocumentSubscription = this.consumerStatusService.onDocumentDetected().subscribe(status => {
+      if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT)) {
+        this.toastService.show({title: $localize`New document detected`, delay: 5000, content: $localize`Document ${status.filename} is being processed by paperless.`})
+      }
+    })
   }
 
 }
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index c8104dae1..a9658bcf7 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -1,7 +1,7 @@
 <app-widget-frame title="Upload new documents" i18n-title [class.has-multiple-status]="(getStatus().length + getStatusesHidden().length)> 1">
   <div header-buttons>
     <button type="button" class="btn btn-link dismiss-all" [disabled]="!getStatus().length" (click)="dismissAll()">
-      <small class="mr-1">Hide all</small>
+      <small class="mr-1" i18n>Dismiss completed</small>
       <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-check2-all" viewBox="0 0 16 16">
         <path d="M12.354 4.354a.5.5 0 0 0-.708-.708L5 10.293 1.854 7.146a.5.5 0 1 0-.708.708l3.5 3.5a.5.5 0 0 0 .708 0l7-7zm-4.208 7l-.896-.897.707-.707.543.543 6.646-6.647a.5.5 0 0 1 .708.708l-7 7a.5.5 0 0 1-.708 0z"/>
         <path d="M5.354 7.146l.896.897-.707.707-.897-.896a.5.5 0 1 1 .708-.708z"/>
@@ -37,7 +37,7 @@
     <ngb-progressbar [value]="status.getProgress()" [max]="1" [type]="getStatusColor(status)"></ngb-progressbar>
     <div *ngIf="isFinished(status)">
       <button *ngIf="status.documentId" class="btn btn-sm btn-outline-primary btn-open" routerLink="/documents/{{status.documentId}}" (click)="dismiss(status)">
-        <small>Open document</small>
+        <small i18n>Open document</small>
         <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-arrow-right-short" viewBox="0 0 16 16">
           <path fill-rule="evenodd" d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z"/>
         </svg>
diff --git a/src-ui/src/app/components/manage/settings/settings.component.html b/src-ui/src/app/components/manage/settings/settings.component.html
index 4ed5ad1ae..910867ace 100644
--- a/src-ui/src/app/components/manage/settings/settings.component.html
+++ b/src-ui/src/app/components/manage/settings/settings.component.html
@@ -99,6 +99,20 @@
           </div>
         </div>
 
+        <h4 class="mt-4" i18n>Notifications</h4>
+
+        <div class="form-row form-group">
+          <div class="col-md-3 col-form-label">
+            <span i18n>Consumer status</span>
+          </div>
+          <div class="col">
+            <app-input-check i18n-title title="Show notifications when new documents are detected" formControlName="notificationsConsumerNewDocument"></app-input-check>
+            <app-input-check i18n-title title="Show notifications when document consumption completes successfully" formControlName="notificationsConsumerSuccess"></app-input-check>
+            <app-input-check i18n-title title="Show notifications when document consumption fails" formControlName="notificationsConsumerFailed"></app-input-check>
+            <app-input-check i18n-title title="Suppress notifications on dashboard" formControlName="notificationsConsumerSuppressOnDashboard" i18n-hint hint="This will suppress all consumer related status messages on the dashboard."></app-input-check>
+          </div>
+        </div>
+
         <h4 class="mt-4" i18n>Bulk editing</h4>
 
         <div class="form-row form-group">
diff --git a/src-ui/src/app/components/manage/settings/settings.component.ts b/src-ui/src/app/components/manage/settings/settings.component.ts
index 47f714c21..47c7b8d7b 100644
--- a/src-ui/src/app/components/manage/settings/settings.component.ts
+++ b/src-ui/src/app/components/manage/settings/settings.component.ts
@@ -26,6 +26,10 @@ export class SettingsComponent implements OnInit {
     'displayLanguage': new FormControl(this.settings.getLanguage()),
     'dateLocale': new FormControl(this.settings.get(SETTINGS_KEYS.DATE_LOCALE)),
     'dateFormat': new FormControl(this.settings.get(SETTINGS_KEYS.DATE_FORMAT)),
+    'notificationsConsumerNewDocument': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT)),
+    'notificationsConsumerSuccess': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)),
+    'notificationsConsumerFailed': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)),
+    'notificationsConsumerSuppressOnDashboard': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD)),
   })
 
   savedViews: PaperlessSavedView[]
@@ -73,6 +77,10 @@ export class SettingsComponent implements OnInit {
     this.settings.set(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER, this.settingsForm.value.useNativePdfViewer)
     this.settings.set(SETTINGS_KEYS.DATE_LOCALE, this.settingsForm.value.dateLocale)
     this.settings.set(SETTINGS_KEYS.DATE_FORMAT, this.settingsForm.value.dateFormat)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT, this.settingsForm.value.notificationsConsumerNewDocument)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS, this.settingsForm.value.notificationsConsumerSuccess)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED, this.settingsForm.value.notificationsConsumerFailed)
+    this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD, this.settingsForm.value.notificationsConsumerSuppressOnDashboard)
     this.settings.setLanguage(this.settingsForm.value.displayLanguage)
     this.documentListViewService.updatePageSize()
     this.settings.updateDarkModeSettings()
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index a3c942499..feb04f3d6 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -65,19 +65,21 @@ export class ConsumerStatusService {
 
   private consumerStatus: FileStatus[] = []
 
-
+  private documentDetectedSubject = new Subject<FileStatus>()
   private documentConsumptionFinishedSubject = new Subject<FileStatus>()
   private documentConsumptionFailedSubject = new Subject<FileStatus>()
 
   private get(taskId: string, filename?: string) {
     let status = this.consumerStatus.find(e => e.taskId == taskId) || this.consumerStatus.find(e => e.filename == filename && e.taskId == null)
+    let created = false
     if (!status) {
       status = new FileStatus()
       this.consumerStatus.push(status)
+      created = true
     }
     status.taskId = taskId
     status.filename = filename
-    return status
+    return {'status': status, 'created': created}
   }
 
   newFileUpload(filename: string): FileStatus {
@@ -101,11 +103,17 @@ export class ConsumerStatusService {
     this.statusWebSocked.onmessage = (ev) => {
       let statusMessage: WebsocketConsumerStatusMessage = JSON.parse(ev['data'])
 
-      let status = this.get(statusMessage.task_id, statusMessage.filename)
+      let statusMessageGet = this.get(statusMessage.task_id, statusMessage.filename)
+      let status = statusMessageGet.status
+      let created = statusMessageGet.created
+
       status.updateProgress(FileStatusPhase.PROCESSING, statusMessage.current_progress, statusMessage.max_progress)
       status.message = statusMessage.message
       status.documentId = statusMessage.document_id
 
+      if (created && statusMessage.status == 'STARTING') {
+        this.documentDetectedSubject.next(status)
+      }
       if (statusMessage.status == "SUCCESS") {
         status.phase = FileStatusPhase.SUCCESS
         this.documentConsumptionFinishedSubject.next(status)
@@ -150,4 +158,8 @@ export class ConsumerStatusService {
     return this.documentConsumptionFailedSubject
   }
 
+  onDocumentDetected() {
+    return this.documentDetectedSubject
+  }
+
 }
diff --git a/src-ui/src/app/services/settings.service.ts b/src-ui/src/app/services/settings.service.ts
index 041fb51ca..f3e437ada 100644
--- a/src-ui/src/app/services/settings.service.ts
+++ b/src-ui/src/app/services/settings.service.ts
@@ -23,7 +23,11 @@ export const SETTINGS_KEYS = {
   DARK_MODE_ENABLED: 'general-settings:dark-mode:enabled',
   USE_NATIVE_PDF_VIEWER: 'general-settings:document-details:native-pdf-viewer',
   DATE_LOCALE: 'general-settings:date-display:date-locale',
-  DATE_FORMAT: 'general-settings:date-display:date-format'
+  DATE_FORMAT: 'general-settings:date-display:date-format',
+  NOTIFICATIONS_CONSUMER_NEW_DOCUMENT: 'general-settings:notifications:consumer-new-documents',
+  NOTIFICATIONS_CONSUMER_SUCCESS: 'general-settings:notifications:consumer-success',
+  NOTIFICATIONS_CONSUMER_FAILED: 'general-settings:notifications:consumer-failed',
+  NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD: 'general-settings:notifications:consumer-suppress-on-dashboard',
 }
 
 const SETTINGS: PaperlessSettings[] = [
@@ -34,7 +38,11 @@ const SETTINGS: PaperlessSettings[] = [
   {key: SETTINGS_KEYS.DARK_MODE_ENABLED, type: "boolean", default: false},
   {key: SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER, type: "boolean", default: false},
   {key: SETTINGS_KEYS.DATE_LOCALE, type: "string", default: ""},
-  {key: SETTINGS_KEYS.DATE_FORMAT, type: "string", default: "mediumDate"}
+  {key: SETTINGS_KEYS.DATE_FORMAT, type: "string", default: "mediumDate"},
+  {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT, type: "boolean", default: true},
+  {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS, type: "boolean", default: true},
+  {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED, type: "boolean", default: true},
+  {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD, type: "boolean", default: true},
 ]
 
 @Injectable({

From b0ed06003b1073a1457bf00815be4cea18c1415a Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 17:56:06 +0100
Subject: [PATCH 39/54] better error messages

---
 src/documents/consumer.py          | 2 +-
 src/paperless_tesseract/parsers.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index d0f7399c8..05043201b 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -158,7 +158,7 @@ class Consumer(LoggingMixin):
         self.override_tag_ids = override_tag_ids
         self.task_id = task_id or str(uuid.uuid4())
 
-        self._send_progress(0, 100, 'WORKING', _('Received new file'))
+        self._send_progress(0, 100, 'STARTING', _('Received new file'))
 
         # this is for grouping logging entries for this particular file
         # together.
diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py
index 31e956284..52b50e983 100644
--- a/src/paperless_tesseract/parsers.py
+++ b/src/paperless_tesseract/parsers.py
@@ -190,11 +190,11 @@ class RasterisedDocumentParser(DocumentParser):
             # Also, no archived file.
             if not self.text:
                 # However, if we don't have anything, fail:
-                raise ParseError(e)
+                raise ParseError(e.__class__.__name__ + ": " + str(e))
 
         except Exception as e:
             # Anything else is probably serious.
-            raise ParseError(e)
+            raise ParseError(e.__class__.__name__ + ": " + str(e))
 
         if not self.text:
             # This may happen for files that don't have any text.

From 71d5af233d6b3c077fe9be09e4106d01957f5efa Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 17:57:16 +0100
Subject: [PATCH 40/54] update messages

---
 src/locale/en-us/LC_MESSAGES/django.po | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/locale/en-us/LC_MESSAGES/django.po b/src/locale/en-us/LC_MESSAGES/django.po
index 5451b8dde..8513c577b 100644
--- a/src/locale/en-us/LC_MESSAGES/django.po
+++ b/src/locale/en-us/LC_MESSAGES/django.po
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-01-26 14:59+0100\n"
+"POT-Creation-Date: 2021-01-27 17:57+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -26,8 +26,8 @@ msgid "File not found"
 msgstr "Datei nicht gefunden"
 
 #: documents/consumer.py:83
-msgid "Document is a duplicate"
-msgstr "Dokument existiert bereits"
+msgid "Document already exists"
+msgstr ""
 
 #: documents/consumer.py:99
 msgid "Pre-consume script does not exist."
@@ -674,3 +674,6 @@ msgstr ""
 #: paperless_mail/models.py:205
 msgid "assign this correspondent"
 msgstr ""
+
+#~ msgid "Document is a duplicate"
+#~ msgstr "Dokument existiert bereits"

From 3b541b22e858153f253c3aceb847bac6f1777c3e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 18:34:59 +0100
Subject: [PATCH 41/54] improved the logic of the "dismiss completed" button

---
 .../upload-file-widget/upload-file-widget.component.html | 8 ++++----
 .../upload-file-widget/upload-file-widget.component.scss | 9 ---------
 .../upload-file-widget/upload-file-widget.component.ts   | 3 +++
 src-ui/src/app/services/consumer-status.service.ts       | 4 ++++
 4 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index a9658bcf7..45d63264a 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -1,12 +1,12 @@
-<app-widget-frame title="Upload new documents" i18n-title [class.has-multiple-status]="(getStatus().length + getStatusesHidden().length)> 1">
+<app-widget-frame title="Upload new documents" i18n-title>
   <div header-buttons>
-    <button type="button" class="btn btn-link dismiss-all" [disabled]="!getStatus().length" (click)="dismissAll()">
-      <small class="mr-1" i18n>Dismiss completed</small>
+    <a *ngIf="getStatusCompleted().length > 0" (click)="dismissAll()" [routerLink]="" >
+      <span i18n>Dismiss completed</span>&nbsp;
       <svg xmlns="http://www.w3.org/2000/svg" width="1rem" height="1rem" fill="currentColor" class="bi bi-check2-all" viewBox="0 0 16 16">
         <path d="M12.354 4.354a.5.5 0 0 0-.708-.708L5 10.293 1.854 7.146a.5.5 0 1 0-.708.708l3.5 3.5a.5.5 0 0 0 .708 0l7-7zm-4.208 7l-.896-.897.707-.707.543.543 6.646-6.647a.5.5 0 0 1 .708.708l-7 7a.5.5 0 0 1-.708 0z"/>
         <path d="M5.354 7.146l.896.897-.707.707-.897-.896a.5.5 0 1 1 .708-.708z"/>
       </svg>
-    </button>
+    </a>
   </div>
   <div content>
     <form>
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
index 5e404bb8e..b37606ff3 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss
@@ -23,15 +23,6 @@ form {
   }
 }
 
-.dismiss-all {
-  transition: opacity 0.2s ease;
-  opacity: 0;
-}
-
-.has-multiple-status .dismiss-all {
-  opacity: 1;
-}
-
 ::ng-deep .progress {
   position: absolute;
   top: 0;
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index 1c5ed5438..a6870e0e7 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -32,6 +32,9 @@ export class UploadFileWidgetComponent implements OnInit {
     return this.consumerStatusService.getConsumerStatus(FileStatusPhase.UPLOADING)
   }
 
+  getStatusCompleted() {
+    return this.consumerStatusService.getConsumerStatusCompleted()
+  }
   getTotalUploadProgress() {
     let current = 0
     let max = 0
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index feb04f3d6..c2169d51f 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -97,6 +97,10 @@ export class ConsumerStatusService {
     }
   }
 
+  getConsumerStatusCompleted() {
+    return this.consumerStatus.filter(s => s.phase == FileStatusPhase.FAILED || s.phase == FileStatusPhase.SUCCESS)
+  }
+
   connect() {
     this.disconnect()
     this.statusWebSocked = new WebSocket("ws://localhost:8000/ws/status/");

From 9167357908a761b1b2a1677d8ff3a38b4d41e442 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 18:47:02 +0100
Subject: [PATCH 42/54] update list view when documents are added

---
 .../document-list/document-list.component.ts  | 21 ++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts
index 509c0a735..0d1562c24 100644
--- a/src-ui/src/app/components/document-list/document-list.component.ts
+++ b/src-ui/src/app/components/document-list/document-list.component.ts
@@ -1,9 +1,11 @@
-import { AfterViewInit, Component, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
+import { AfterViewInit, Component, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
+import { Subscription } from 'rxjs';
 import { PaperlessDocument } from 'src/app/data/paperless-document';
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view';
 import { SortableDirective, SortEvent } from 'src/app/directives/sortable.directive';
+import { ConsumerStatusService } from 'src/app/services/consumer-status.service';
 import { DocumentListViewService } from 'src/app/services/document-list-view.service';
 import { DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service';
 import { SavedViewService } from 'src/app/services/rest/saved-view.service';
@@ -16,7 +18,7 @@ import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-vi
   templateUrl: './document-list.component.html',
   styleUrls: ['./document-list.component.scss']
 })
-export class DocumentListComponent implements OnInit {
+export class DocumentListComponent implements OnInit, OnDestroy {
 
   constructor(
     public list: DocumentListViewService,
@@ -24,7 +26,9 @@ export class DocumentListComponent implements OnInit {
     public route: ActivatedRoute,
     private router: Router,
     private toastService: ToastService,
-    private modalService: NgbModal) { }
+    private modalService: NgbModal,
+    private consumerStatusService: ConsumerStatusService
+  ) { }
 
   @ViewChild("filterEditor")
   private filterEditor: FilterEditorComponent
@@ -35,6 +39,8 @@ export class DocumentListComponent implements OnInit {
 
   filterRulesModified: boolean = false
 
+  private consumptionFinishedSubscription: Subscription
+
   get isFiltered() {
     return this.list.filterRules?.length > 0
   }
@@ -63,6 +69,9 @@ export class DocumentListComponent implements OnInit {
     if (localStorage.getItem('document-list:displayMode') != null) {
       this.displayMode = localStorage.getItem('document-list:displayMode')
     }
+    this.consumptionFinishedSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(() => {
+      this.list.reload()
+    })
     this.route.paramMap.subscribe(params => {
       this.list.clear()
       if (params.has('id')) {
@@ -83,6 +92,12 @@ export class DocumentListComponent implements OnInit {
     })
   }
 
+  ngOnDestroy() {
+    if (this.consumptionFinishedSubscription) {
+      this.consumptionFinishedSubscription.unsubscribe()
+    }
+  }
+
   loadViewConfig(view: PaperlessSavedView) {
     this.list.load(view)
     this.list.reload()

From d9e6f6e07c327c26aec21d2e2dbd904625323850 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 19:11:18 +0100
Subject: [PATCH 43/54] add gunicorn again

---
 Pipfile          |  1 +
 Pipfile.lock     | 30 +++++++++++++++++++-----------
 requirements.txt |  3 ++-
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/Pipfile b/Pipfile
index e30abc064..884279142 100644
--- a/Pipfile
+++ b/Pipfile
@@ -18,6 +18,7 @@ django-q = "~=1.3.4"
 djangorestframework = "~=3.12.2"
 filelock = "*"
 fuzzywuzzy = "*"
+gunicorn = "*"
 imap-tools = "*"
 langdetect = "*"
 pdftotext = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index f60ac0223..1deb2913a 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "61ffe4b723ce6b04e67636f1572f833ee9adacdcf74a883c558ed4b0c906e75a"
+            "sha256": "80ebe7269056bd328736aea532b3baccb3314763cc76fabd8e6ca28d84c3b1c0"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -279,6 +279,14 @@
             "index": "pypi",
             "version": "==0.18.0"
         },
+        "gunicorn": {
+            "hashes": [
+                "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
+                "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
+            ],
+            "index": "pypi",
+            "version": "==20.0.4"
+        },
         "hiredis": {
             "hashes": [
                 "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
@@ -1086,11 +1094,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
-                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+                "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
+                "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.26.2"
+            "version": "==1.26.3"
         },
         "watchdog": {
             "hashes": [
@@ -1351,11 +1359,11 @@
         },
         "execnet": {
             "hashes": [
-                "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50",
-                "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"
+                "sha256:7a13113028b1e1cc4c6492b28098b3c6576c9dccc7973bfe47b342afadafb2ac",
+                "sha256:b73c5565e517f24b62dea8a5ceac178c661c4309d3aa0c3e420856c072c411b4"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.7.1"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.8.0"
         },
         "factory-boy": {
             "hashes": [
@@ -1721,11 +1729,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
-                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"
+                "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
+                "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.26.2"
+            "version": "==1.26.3"
         },
         "virtualenv": {
             "hashes": [
diff --git a/requirements.txt b/requirements.txt
index a5a564a99..e1a382426 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -34,6 +34,7 @@ django==3.1.5
 djangorestframework==3.12.2
 filelock==3.0.12
 fuzzywuzzy==0.18.0
+gunicorn==20.0.4
 hiredis==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 hyperlink==21.0.0
@@ -85,7 +86,7 @@ twisted[tls]==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.
 txaio==20.12.1; python_version >= '3.6'
 typing-extensions==3.7.4.3; python_version < '3.8'
 tzlocal==2.1
-urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+urllib3==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
 watchdog==1.0.2
 wcwidth==0.2.5
 whitenoise==5.2.0

From a0a781a3d5d990ac89325a30fa0345f16e4f78b3 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 19:11:50 +0100
Subject: [PATCH 44/54] i18n

---
 src-ui/messages.xlf                           | 33 +++++++++++--------
 .../upload-file-widget.component.html         |  2 +-
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf
index ec718acc6..84ca4888b 100644
--- a/src-ui/messages.xlf
+++ b/src-ui/messages.xlf
@@ -48,21 +48,21 @@
         <source>Documents</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/document-list/document-list.component.ts</context>
-          <context context-type="linenumber">43</context>
+          <context context-type="linenumber">49</context>
         </context-group>
       </trans-unit>
       <trans-unit id="2155249406916744630" datatype="html">
         <source>View &quot;<x id="PH" equiv-text="this.list.savedView.name"/>&quot; saved successfully.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/document-list/document-list.component.ts</context>
-          <context context-type="linenumber">94</context>
+          <context context-type="linenumber">109</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6837554170707123455" datatype="html">
         <source>View &quot;<x id="PH" equiv-text="savedView.name"/>&quot; created successfully.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/document-list/document-list.component.ts</context>
-          <context context-type="linenumber">115</context>
+          <context context-type="linenumber">130</context>
         </context-group>
       </trans-unit>
       <trans-unit id="9ca82952a6bc860b5391d5975322d8af8ceddfa4" datatype="html">
@@ -1418,21 +1418,21 @@
         <source>Connecting...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">90</context>
+          <context context-type="linenumber">93</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1245343823699368872" datatype="html">
         <source>Uploading...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">95</context>
+          <context context-type="linenumber">98</context>
         </context-group>
       </trans-unit>
       <trans-unit id="3994065460580948013" datatype="html">
         <source>Waiting for consumer...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">98</context>
+          <context context-type="linenumber">101</context>
         </context-group>
       </trans-unit>
       <trans-unit id="e022072b3e4dd77e3f09960817ef3359a49963b3" datatype="html">
@@ -1442,13 +1442,6 @@
           <context context-type="linenumber">1</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="bd4a8607e4a002d939cffb347ec056664dfb2c73" datatype="html">
-        <source>Dismiss completed</source>
-        <context-group purpose="location">
-          <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
-          <context context-type="linenumber">4</context>
-        </context-group>
-      </trans-unit>
       <trans-unit id="52d5d56b6667ba3f7c759cbc0d4c39d34a99701e" datatype="html">
         <source>Drop documents here or</source>
         <context-group purpose="location">
@@ -1463,6 +1456,20 @@
           <context context-type="linenumber">13</context>
         </context-group>
       </trans-unit>
+      <trans-unit id="bd4a8607e4a002d939cffb347ec056664dfb2c73" datatype="html">
+        <source>Dismiss completed</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
+          <context context-type="linenumber">4</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="7cf3abd55bf1d2095435a050325927f083e6034e" datatype="html">
+        <source><x id="INTERPOLATION" equiv-text="{{getStatusesHidden().length}}"/> more hidden</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
+          <context context-type="linenumber">23</context>
+        </context-group>
+      </trans-unit>
       <trans-unit id="710254a196a2649674438edf8a15b7ab1f48271b" datatype="html">
         <source>Open document</source>
         <context-group purpose="location">
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index 45d63264a..5cabb3c85 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -20,7 +20,7 @@
       <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>
     </div>
     <div *ngIf="getStatusesHidden().length" class="alerts-hidden">
-      <p *ngIf="!alertsExpanded" class="mt-3 mb-0 text-center">{{getStatusesHidden().length}} more hidden <button class="btn btn-sm btn-link py-0" (click)="alertsExpanded = !alertsExpanded" aria-controls="hiddenAlerts" [attr.aria-expanded]="alertsExpanded">Show all</button></p>
+      <p *ngIf="!alertsExpanded" class="mt-3 mb-0 text-center"><span i18n>{{getStatusesHidden().length}} more hidden</span> <button class="btn btn-sm btn-link py-0" (click)="alertsExpanded = !alertsExpanded" aria-controls="hiddenAlerts" [attr.aria-expanded]="alertsExpanded" i18n>Show all</button></p>
       <div #hiddenAlerts="ngbCollapse" [(ngbCollapse)]="!alertsExpanded">
         <div *ngFor="let status of getStatusesHidden()">
           <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>

From 7414b3d23aed1633a6f2ba60aa9ac57266fc3ab9 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 19:57:04 +0100
Subject: [PATCH 45/54] update configuration

---
 Pipfile                             |   2 +-
 Pipfile.lock                        | 238 ++++++++++++++++++----------
 ansible/tasks/main.yml              |   2 +-
 docker/gunicorn.conf.py             |   2 +-
 docker/supervisord.conf             |   4 +-
 requirements.txt                    |  17 +-
 scripts/paperless-webserver.service |   2 +-
 7 files changed, 167 insertions(+), 100 deletions(-)

diff --git a/Pipfile b/Pipfile
index 884279142..c97ec297b 100644
--- a/Pipfile
+++ b/Pipfile
@@ -44,7 +44,7 @@ tqdm = "*"
 tika = "*"
 channels = "~=3.0"
 channels-redis = "*"
-daphne = "~=3.0"
+uvicorn = {extras = ["standard"], version = "*"}
 
 [dev-packages]
 coveralls = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 1deb2913a..5ae121105 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "80ebe7269056bd328736aea532b3baccb3314763cc76fabd8e6ca28d84c3b1c0"
+            "sha256": "33586bca793586bd4e413ae634780628b6df65a0d67b581c86ab4d2a16a817e0"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -155,6 +155,15 @@
             "markers": "python_version >= '3.1'",
             "version": "==4.0.0"
         },
+        "click": {
+            "hashes": [
+                "sha256:a3747c864f8e400a3664f5f4fd6dae11b4605bf6b727dae7b6f22ba9bd0a194a",
+                "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
+                "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==7.1.2"
+        },
         "coloredlogs": {
             "hashes": [
                 "sha256:5e78691e2673a8e294499e1832bb13efcfb44a86b92e18109fa18951093218ab",
@@ -196,7 +205,7 @@
                 "sha256:0052c9887600c57054a5867d4b0240159fa009faa3bcf6a1627271d9cdcb005a",
                 "sha256:c22b692707f514de9013651ecb687f2abe4f35cf6fe292ece634e9f1737bc7e3"
             ],
-            "index": "pypi",
+            "markers": "python_version >= '3.6'",
             "version": "==3.0.1"
         },
         "dateparser": {
@@ -287,6 +296,14 @@
             "index": "pypi",
             "version": "==20.0.4"
         },
+        "h11": {
+            "hashes": [
+                "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6",
+                "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.12.0"
+        },
         "hiredis": {
             "hashes": [
                 "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680",
@@ -341,6 +358,25 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.1.0"
         },
+        "httptools": {
+            "hashes": [
+                "sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be",
+                "sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d",
+                "sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce",
+                "sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2",
+                "sha256:7f6c82262d3bdde886a29ae0d65d4fae6b3ac6fba763891ddb72e72e1dbe7075",
+                "sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6",
+                "sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f",
+                "sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009",
+                "sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce",
+                "sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a",
+                "sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c",
+                "sha256:cb46a65a0ed99c38dfcbf9f5be8be6cf9cb497e527505fefac7cbd38a467f3c6",
+                "sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4",
+                "sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437"
+            ],
+            "version": "==0.1.1"
+        },
         "humanfriendly": {
             "hashes": [
                 "sha256:066562956639ab21ff2676d1fda0b5987e985c534fc76700a19bd54bcb81121d",
@@ -380,14 +416,6 @@
             ],
             "version": "==0.4.0"
         },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771",
-                "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==3.4.0"
-        },
         "incremental": {
             "hashes": [
                 "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
@@ -754,6 +782,7 @@
                 "sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51",
                 "sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==20.0.1"
         },
         "python-dateutil": {
@@ -804,6 +833,32 @@
             ],
             "version": "==2020.5"
         },
+        "pyyaml": {
+            "hashes": [
+                "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf",
+                "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696",
+                "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393",
+                "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77",
+                "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922",
+                "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5",
+                "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8",
+                "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10",
+                "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc",
+                "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018",
+                "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e",
+                "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253",
+                "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183",
+                "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb",
+                "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185",
+                "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db",
+                "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46",
+                "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b",
+                "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63",
+                "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df",
+                "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"
+            ],
+            "version": "==5.4.1"
+        },
         "redis": {
             "hashes": [
                 "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2",
@@ -950,35 +1005,29 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:168c45c0c32e23f613db7c9e4e780bc61982d71dcd406ead746c7c7c2f2004ce",
-                "sha256:213bc59191da2f479984ad4ec39406bf949a99aba70e9237b916ce7547b6ef42",
-                "sha256:25b241034215247481f53355e05f9e25462682b13bd9191359075682adcd9554",
-                "sha256:2c872de0c69ed20fb1a9b9cf6f77298b04a26f0b8720a5457be08be254366c6e",
-                "sha256:3397c129b479846d7eaa18f999369a24322d008fac0782e7828fa567358c36ce",
-                "sha256:368c0f69f93186309e1b4beb8e26d51dd6f5010b79264c0f1e9ca00cd92ea8c9",
-                "sha256:3d5db5d815370c28d938cf9b0809dade4acf7aba57eaf7ef733bfedc9b2474c4",
-                "sha256:4598cf03136067000855d6b44d7a1f4f46994164bcd450fb2c3d481afc25dd06",
-                "sha256:4a453d5e5689de62e5d38edf40af3f17560bfd63c9c5bd228c18c1f99afa155b",
-                "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47",
-                "sha256:634568a3018bc16a83cda28d4f7aed0d803dd5618facb36e977e53b2df868443",
-                "sha256:65923bc3809524e46fb7eb4d6346552cbb6a1ffc41be748535aa502a2e3d3389",
-                "sha256:6b0ceb23560f46dd236a8ad4378fc40bad1783e997604ba845e131d6c680963e",
-                "sha256:8c8d6ca19c8497344b810b0b0344f8375af5f6bb9c98bd42e33f747417ab3f57",
-                "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62",
-                "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d",
-                "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437",
-                "sha256:b5e9d3e4474644915809d6aa1416ff20430a3ed9ae723a5d295da5ddb24985e2",
-                "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2",
-                "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54",
-                "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474",
-                "sha256:e360cb2299028d0b0d0f65a5c5e51fc16a335f1603aa2357c25766c8dab56938",
-                "sha256:e98d49a5717369d8241d6cf33ecb0ca72deee392414118198a8e5b4c35c56340",
-                "sha256:ed572470af2438b526ea574ff8f05e7f39b44ac37f712105e57fc4d53a6fb660",
-                "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012",
-                "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c"
+                "sha256:155225621df90fcd151e25d51c50217e412de717475999ebb76e17e310176981",
+                "sha256:1bc5b446600c4ff7ab36bade47180673141322f0febaa555f1c433fe04f2a0e3",
+                "sha256:2f1c2ebca6fd867160e70102200b1bd07b3b2d31a3e6af3c58d688c15d0d07b7",
+                "sha256:313785c4dab65060f9648112d025f6d2fec69a8a889c714328882d678a95f053",
+                "sha256:31ab217b5c27ab429d07428a76002b33662f98986095bbce5d55e0788f7e8b15",
+                "sha256:3d4303e3e21d07d9557b26a1707bb9fc065510ee8501c9bf22a0157249a82fd0",
+                "sha256:4f1d9cc977ac6a4a63c124045c1e8bf67ec37098f67c699887a93736961a00ae",
+                "sha256:58731bbe0103e96b89b2f41516699db9b63066e4317e31b8402891571f6d358f",
+                "sha256:8629135ee00cc2182ac8be8e75643b9f02235942443732c2ed69ab48edcb6614",
+                "sha256:876badc33eec20709d4e042a09834f5953ebdac4088d45a4f3a1f18b56885718",
+                "sha256:8840a9adb4ede3751f49761653d3ebf664f25195fdd42ada394ffea8903dd51d",
+                "sha256:aef3a2dbc436bbe8f6e0b635f0b5fe5ed024b522eee4637dbbe0b974129ca734",
+                "sha256:b8af26839ae343655f3ca377a5d5e5466f1d3b3ac7432a43449154fe958ae0e0",
+                "sha256:c0911f3180de343643f369dc5cfedad6ba9f939c2d516bddea4a6871eb000722",
+                "sha256:cb6dc9f82dfd95f6b9032a8d7ea70efeeb15d5b5fd6ed4e8537bb3c673580566",
+                "sha256:cdbc47628184a0ebeb5c08f1892614e1bd4a51f6e0d609c6eed253823a960f5b",
+                "sha256:d902d3a5ad7f28874c0a82db95246d24ca07ad932741df668595fe00a4819870",
+                "sha256:eab389aba0ad8b5e6b5abdc3337ade46823df75f80a8edd4c67833567577cb3d",
+                "sha256:eb7928275f3560d47e5538e15e9f32b3d64cd30ea8f85f3e82987425476f53f6",
+                "sha256:f68d5761a2d2376e2b194c8e9192bbf7c51306ca176f1a0889990a52ef0d551f"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.5.4"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.6.0"
         },
         "service-identity": {
             "hashes": [
@@ -1076,15 +1125,6 @@
             "markers": "python_version >= '3.6'",
             "version": "==20.12.1"
         },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
-                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
-                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==3.7.4.3"
-        },
         "tzlocal": {
             "hashes": [
                 "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44",
@@ -1100,6 +1140,33 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
             "version": "==1.26.3"
         },
+        "uvicorn": {
+            "extras": [
+                "standard"
+            ],
+            "hashes": [
+                "sha256:1079c50a06f6338095b4f203e7861dbff318dde5f22f3a324fc6e94c7654164c",
+                "sha256:ef1e0bb5f7941c6fe324e06443ddac0331e1632a776175f87891c7bd02694355"
+            ],
+            "index": "pypi",
+            "version": "==0.13.3"
+        },
+        "uvloop": {
+            "hashes": [
+                "sha256:08b109f0213af392150e2fe6f81d33261bb5ce968a288eb698aad4f46eb711bd",
+                "sha256:123ac9c0c7dd71464f58f1b4ee0bbd81285d96cdda8bc3519281b8973e3a461e",
+                "sha256:2e57a28567d874afd803b6d8a7aeee7bfa0a34994f1b237837e6d7e525cea9a4",
+                "sha256:4315d2ec3ca393dd5bc0b0089d23101276778c304d42faff5dc4579cb6caef09",
+                "sha256:4544dcf77d74f3a84f03dd6278174575c44c67d7165d4c42c71db3fdc3860726",
+                "sha256:63c0bd965e73a4e8dde6a15635262ec38036d83327230552ada21298ee5fa3b7",
+                "sha256:afd5513c0ae414ec71d24f6f123614a80f3d27ca655a4fcf6cabe50994cc1891",
+                "sha256:b4f591aa4b3fa7f32fb51e2ee9fea1b495eb75b0b3c8d0ca52514ad675ae63f7",
+                "sha256:bcac356d62edd330080aed082e78d4b580ff260a677508718f88016333e2c9c5",
+                "sha256:e7514d7a48c063226b7d06617cbb12a14278d4323a065a8d46a7962686ce2e95",
+                "sha256:f07909cd9fc08c52d294b1570bba92186181ca01fe3dc9ffba68955273dd7362"
+            ],
+            "version": "==0.14.0"
+        },
         "watchdog": {
             "hashes": [
                 "sha256:016b01495b9c55b5d4126ed8ae75d93ea0d99377084107c33162df52887cee18",
@@ -1123,6 +1190,14 @@
             "index": "pypi",
             "version": "==1.0.2"
         },
+        "watchgod": {
+            "hashes": [
+                "sha256:59700dab7445aa8e6067a5b94f37bae90fc367554549b1ed2e9d0f4f38a90d2a",
+                "sha256:5fb60afa9558b79736395db1cb60ad3ed59df5c2f507a3ff729220cf1251ffdc",
+                "sha256:e9cca0ab9c63f17fc85df9fd8bd18156ff00aff04ebe5976cee473f4968c6858"
+            ],
+            "version": "==0.6"
+        },
         "wcwidth": {
             "hashes": [
                 "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
@@ -1130,6 +1205,34 @@
             ],
             "version": "==0.2.5"
         },
+        "websockets": {
+            "hashes": [
+                "sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5",
+                "sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5",
+                "sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308",
+                "sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb",
+                "sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a",
+                "sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c",
+                "sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170",
+                "sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422",
+                "sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8",
+                "sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485",
+                "sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f",
+                "sha256:745a1c8ca62f7d27de42b517ca7c6f716d1eb96c5aa73cf6407936c0167cbb3c",
+                "sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8",
+                "sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc",
+                "sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779",
+                "sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989",
+                "sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1",
+                "sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092",
+                "sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824",
+                "sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d",
+                "sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55",
+                "sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36",
+                "sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b"
+            ],
+            "version": "==8.1"
+        },
         "whitenoise": {
             "hashes": [
                 "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7",
@@ -1147,14 +1250,6 @@
             "index": "pypi",
             "version": "==2.7.4"
         },
-        "zipp": {
-            "hashes": [
-                "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
-                "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.0"
-        },
         "zope.interface": {
             "hashes": [
                 "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
@@ -1406,22 +1501,6 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.2.0"
         },
-        "importlib-metadata": {
-            "hashes": [
-                "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771",
-                "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==3.4.0"
-        },
-        "importlib-resources": {
-            "hashes": [
-                "sha256:885b8eae589179f661c909d699a546cf10d83692553e34dca1bf5eb06f7f6217",
-                "sha256:bfdad047bce441405a49cf8eb48ddce5e56c696e185f59147a8b79e75e9e6380"
-            ],
-            "markers": "python_version < '3.7'",
-            "version": "==5.1.0"
-        },
         "iniconfig": {
             "hashes": [
                 "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
@@ -1718,15 +1797,6 @@
             "index": "pypi",
             "version": "==3.21.2"
         },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
-                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
-                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
-            ],
-            "markers": "python_version < '3.8'",
-            "version": "==3.7.4.3"
-        },
         "urllib3": {
             "hashes": [
                 "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
@@ -1742,14 +1812,6 @@
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.4.0"
-        },
-        "zipp": {
-            "hashes": [
-                "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
-                "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
-            ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.4.0"
         }
     }
 }
diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml
index a353a18ec..6412fa30f 100644
--- a/ansible/tasks/main.yml
+++ b/ansible/tasks/main.yml
@@ -418,7 +418,7 @@
     path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service"
     section: "Service"
     option: "ExecStart"
-    value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}"
+    value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.asgi:application -w 2 -k uvicorn.workers.UvicornWorker -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}"
 
 - name: copy systemd services
   copy:
diff --git a/docker/gunicorn.conf.py b/docker/gunicorn.conf.py
index edfb362d9..36bde9cf7 100644
--- a/docker/gunicorn.conf.py
+++ b/docker/gunicorn.conf.py
@@ -1,7 +1,7 @@
 bind = '0.0.0.0:8000'
 backlog = 2048
 workers = 3
-worker_class = 'sync'
+worker_class = 'uvicorn.workers.UvicornWorker'
 worker_connections = 1000
 timeout = 20
 keepalive = 2
diff --git a/docker/supervisord.conf b/docker/supervisord.conf
index a643b1136..fca66c83c 100644
--- a/docker/supervisord.conf
+++ b/docker/supervisord.conf
@@ -7,8 +7,8 @@ logfile_backups=10           ; # of main logfile backups; 0 means none, default
 loglevel=info                ; log level; default info; others: debug,warn,trace
 user=root
 
-[program:daphne]
-command=daphne -b 0.0.0.0 -p 8000 paperless.asgi:application
+[program:gunicorn]
+command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.asgi:application
 user=paperless
 
 stdout_logfile=/dev/stdout
diff --git a/requirements.txt b/requirements.txt
index e1a382426..c0228d378 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -20,10 +20,11 @@ cffi==1.14.4
 channels-redis==3.2.0
 channels==3.0.3
 chardet==4.0.0; python_version >= '3.1'
+click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 constantly==15.1.0
 cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-daphne==3.0.1
+daphne==3.0.1; python_version >= '3.6'
 dateparser==0.7.6
 django-cors-headers==3.7.0
 django-extensions==3.1.0
@@ -35,13 +36,14 @@ djangorestframework==3.12.2
 filelock==3.0.12
 fuzzywuzzy==0.18.0
 gunicorn==20.0.4
+h11==0.12.0; python_version >= '3.6'
 hiredis==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+httptools==0.1.1
 humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 hyperlink==21.0.0
 idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 imap-tools==0.36.0
 img2pdf==0.4.0
-importlib-metadata==3.4.0; python_version < '3.8'
 incremental==17.5.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
@@ -62,19 +64,20 @@ pyasn1-modules==0.2.8
 pyasn1==0.4.8
 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pyhamcrest==2.0.2; python_version >= '3.5'
-pyopenssl==20.0.1
+pyopenssl==20.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 python-dateutil==2.8.1
 python-dotenv==0.15.0
 python-gnupg==0.4.6
 python-levenshtein==0.12.1
 python-magic==0.4.18
 pytz==2020.5
+pyyaml==5.4.1
 redis==3.5.3
 regex==2020.11.13
 reportlab==3.5.60
 requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 scikit-learn==0.24.0
-scipy==1.5.4; python_version >= '3.6'
+scipy==1.6.0; python_version >= '3.7'
 service-identity==18.1.0
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sortedcontainers==2.3.0
@@ -84,12 +87,14 @@ tika==1.24
 tqdm==4.56.0
 twisted[tls]==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 txaio==20.12.1; python_version >= '3.6'
-typing-extensions==3.7.4.3; python_version < '3.8'
 tzlocal==2.1
 urllib3==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+uvicorn[standard]==0.13.3
+uvloop==0.14.0
 watchdog==1.0.2
+watchgod==0.6
 wcwidth==0.2.5
+websockets==8.1
 whitenoise==5.2.0
 whoosh==2.7.4
-zipp==3.4.0; python_version >= '3.6'
 zope.interface==5.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
diff --git a/scripts/paperless-webserver.service b/scripts/paperless-webserver.service
index a43110be2..77d2def34 100644
--- a/scripts/paperless-webserver.service
+++ b/scripts/paperless-webserver.service
@@ -8,7 +8,7 @@ Requires=redis.service
 User=paperless
 Group=paperless
 WorkingDirectory=/opt/paperless/src
-ExecStart=/opt/paperless/.local/bin/gunicorn paperless.wsgi -w 2 -b 0.0.0.0:8000
+ExecStart=/opt/paperless/.local/bin/gunicorn paperless.asgi:application -w 2 -k uvicorn.workers.UvicornWorker -b 0.0.0.0:8000
 
 [Install]
 WantedBy=multi-user.target

From d2a8bc5c4c9e3642a2bb7b4890d290313f15ed85 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 20:08:09 +0100
Subject: [PATCH 46/54] fix redis configuration

---
 src/paperless/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index b36ab6cf1..678ce5a21 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -160,7 +160,7 @@ CHANNEL_LAYERS = {
     "default": {
         "BACKEND": "channels_redis.core.RedisChannelLayer",
         "CONFIG": {
-            "hosts": ["redis://localhost:6379"],
+            "hosts": [os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")],
         },
     },
 }

From 29e1b51164ac531c965c921b0b6ed48e777e74bb Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 20:27:06 +0100
Subject: [PATCH 47/54] add python 3.9 for tests

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 03b789bb7..e92afcc06 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8']
+        python-version: ['3.6', '3.7', '3.8', '3.9']
       fail-fast: false
     steps:
       -

From 062008269b8d8cd2a79d1fcd9b1dab5365ab9280 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 20:27:40 +0100
Subject: [PATCH 48/54] prevent updates to scipy, since 1.6 is incompatible
 with python 3.6

---
 Pipfile          |   2 +
 Pipfile.lock     | 110 +++++++++++++++++++++++++++++++++++++----------
 requirements.txt |   5 ++-
 3 files changed, 93 insertions(+), 24 deletions(-)

diff --git a/Pipfile b/Pipfile
index c97ec297b..95d845d1f 100644
--- a/Pipfile
+++ b/Pipfile
@@ -35,6 +35,8 @@ psycopg2-binary = "*"
 redis = "*"
 # Pinned because aarch64 wheels and updates cause warnings when loading the classifier model.
 scikit-learn="==0.24.0"
+# Prevent scipy updates because 1.6 is incompatible with python 3.6
+scipy="~=1.5.4"
 whitenoise = "~=5.2.0"
 watchdog = "*"
 whoosh="~=2.7.4"
diff --git a/Pipfile.lock b/Pipfile.lock
index 5ae121105..b8cb29b29 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "33586bca793586bd4e413ae634780628b6df65a0d67b581c86ab4d2a16a817e0"
+            "sha256": "d80d2539a4528a8fd9e848875c2e2d5bcdb3e98154f45b612706094b84ecaaea"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -416,6 +416,14 @@
             ],
             "version": "==0.4.0"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771",
+                "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.4.0"
+        },
         "incremental": {
             "hashes": [
                 "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f",
@@ -1005,29 +1013,35 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:155225621df90fcd151e25d51c50217e412de717475999ebb76e17e310176981",
-                "sha256:1bc5b446600c4ff7ab36bade47180673141322f0febaa555f1c433fe04f2a0e3",
-                "sha256:2f1c2ebca6fd867160e70102200b1bd07b3b2d31a3e6af3c58d688c15d0d07b7",
-                "sha256:313785c4dab65060f9648112d025f6d2fec69a8a889c714328882d678a95f053",
-                "sha256:31ab217b5c27ab429d07428a76002b33662f98986095bbce5d55e0788f7e8b15",
-                "sha256:3d4303e3e21d07d9557b26a1707bb9fc065510ee8501c9bf22a0157249a82fd0",
-                "sha256:4f1d9cc977ac6a4a63c124045c1e8bf67ec37098f67c699887a93736961a00ae",
-                "sha256:58731bbe0103e96b89b2f41516699db9b63066e4317e31b8402891571f6d358f",
-                "sha256:8629135ee00cc2182ac8be8e75643b9f02235942443732c2ed69ab48edcb6614",
-                "sha256:876badc33eec20709d4e042a09834f5953ebdac4088d45a4f3a1f18b56885718",
-                "sha256:8840a9adb4ede3751f49761653d3ebf664f25195fdd42ada394ffea8903dd51d",
-                "sha256:aef3a2dbc436bbe8f6e0b635f0b5fe5ed024b522eee4637dbbe0b974129ca734",
-                "sha256:b8af26839ae343655f3ca377a5d5e5466f1d3b3ac7432a43449154fe958ae0e0",
-                "sha256:c0911f3180de343643f369dc5cfedad6ba9f939c2d516bddea4a6871eb000722",
-                "sha256:cb6dc9f82dfd95f6b9032a8d7ea70efeeb15d5b5fd6ed4e8537bb3c673580566",
-                "sha256:cdbc47628184a0ebeb5c08f1892614e1bd4a51f6e0d609c6eed253823a960f5b",
-                "sha256:d902d3a5ad7f28874c0a82db95246d24ca07ad932741df668595fe00a4819870",
-                "sha256:eab389aba0ad8b5e6b5abdc3337ade46823df75f80a8edd4c67833567577cb3d",
-                "sha256:eb7928275f3560d47e5538e15e9f32b3d64cd30ea8f85f3e82987425476f53f6",
-                "sha256:f68d5761a2d2376e2b194c8e9192bbf7c51306ca176f1a0889990a52ef0d551f"
+                "sha256:168c45c0c32e23f613db7c9e4e780bc61982d71dcd406ead746c7c7c2f2004ce",
+                "sha256:213bc59191da2f479984ad4ec39406bf949a99aba70e9237b916ce7547b6ef42",
+                "sha256:25b241034215247481f53355e05f9e25462682b13bd9191359075682adcd9554",
+                "sha256:2c872de0c69ed20fb1a9b9cf6f77298b04a26f0b8720a5457be08be254366c6e",
+                "sha256:3397c129b479846d7eaa18f999369a24322d008fac0782e7828fa567358c36ce",
+                "sha256:368c0f69f93186309e1b4beb8e26d51dd6f5010b79264c0f1e9ca00cd92ea8c9",
+                "sha256:3d5db5d815370c28d938cf9b0809dade4acf7aba57eaf7ef733bfedc9b2474c4",
+                "sha256:4598cf03136067000855d6b44d7a1f4f46994164bcd450fb2c3d481afc25dd06",
+                "sha256:4a453d5e5689de62e5d38edf40af3f17560bfd63c9c5bd228c18c1f99afa155b",
+                "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47",
+                "sha256:634568a3018bc16a83cda28d4f7aed0d803dd5618facb36e977e53b2df868443",
+                "sha256:65923bc3809524e46fb7eb4d6346552cbb6a1ffc41be748535aa502a2e3d3389",
+                "sha256:6b0ceb23560f46dd236a8ad4378fc40bad1783e997604ba845e131d6c680963e",
+                "sha256:8c8d6ca19c8497344b810b0b0344f8375af5f6bb9c98bd42e33f747417ab3f57",
+                "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62",
+                "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d",
+                "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437",
+                "sha256:b5e9d3e4474644915809d6aa1416ff20430a3ed9ae723a5d295da5ddb24985e2",
+                "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2",
+                "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54",
+                "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474",
+                "sha256:e360cb2299028d0b0d0f65a5c5e51fc16a335f1603aa2357c25766c8dab56938",
+                "sha256:e98d49a5717369d8241d6cf33ecb0ca72deee392414118198a8e5b4c35c56340",
+                "sha256:ed572470af2438b526ea574ff8f05e7f39b44ac37f712105e57fc4d53a6fb660",
+                "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012",
+                "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.0"
+            "index": "pypi",
+            "version": "==1.5.4"
         },
         "service-identity": {
             "hashes": [
@@ -1125,6 +1139,15 @@
             "markers": "python_version >= '3.6'",
             "version": "==20.12.1"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.7.4.3"
+        },
         "tzlocal": {
             "hashes": [
                 "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44",
@@ -1250,6 +1273,14 @@
             "index": "pypi",
             "version": "==2.7.4"
         },
+        "zipp": {
+            "hashes": [
+                "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
+                "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.4.0"
+        },
         "zope.interface": {
             "hashes": [
                 "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1",
@@ -1501,6 +1532,22 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.2.0"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771",
+                "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.4.0"
+        },
+        "importlib-resources": {
+            "hashes": [
+                "sha256:885b8eae589179f661c909d699a546cf10d83692553e34dca1bf5eb06f7f6217",
+                "sha256:bfdad047bce441405a49cf8eb48ddce5e56c696e185f59147a8b79e75e9e6380"
+            ],
+            "markers": "python_version < '3.7'",
+            "version": "==5.1.0"
+        },
         "iniconfig": {
             "hashes": [
                 "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
@@ -1797,6 +1844,15 @@
             "index": "pypi",
             "version": "==3.21.2"
         },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
+                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
+                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
+            ],
+            "markers": "python_version < '3.8'",
+            "version": "==3.7.4.3"
+        },
         "urllib3": {
             "hashes": [
                 "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80",
@@ -1812,6 +1868,14 @@
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.4.0"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108",
+                "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.4.0"
         }
     }
 }
diff --git a/requirements.txt b/requirements.txt
index c0228d378..0c6deff1d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -44,6 +44,7 @@ hyperlink==21.0.0
 idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 imap-tools==0.36.0
 img2pdf==0.4.0
+importlib-metadata==3.4.0; python_version < '3.8'
 incremental==17.5.0
 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 inotifyrecursive==0.3.5
@@ -77,7 +78,7 @@ regex==2020.11.13
 reportlab==3.5.60
 requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 scikit-learn==0.24.0
-scipy==1.6.0; python_version >= '3.7'
+scipy==1.5.4
 service-identity==18.1.0
 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sortedcontainers==2.3.0
@@ -87,6 +88,7 @@ tika==1.24
 tqdm==4.56.0
 twisted[tls]==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 txaio==20.12.1; python_version >= '3.6'
+typing-extensions==3.7.4.3; python_version < '3.8'
 tzlocal==2.1
 urllib3==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
 uvicorn[standard]==0.13.3
@@ -97,4 +99,5 @@ wcwidth==0.2.5
 websockets==8.1
 whitenoise==5.2.0
 whoosh==2.7.4
+zipp==3.4.0; python_version >= '3.6'
 zope.interface==5.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'

From 41fe607157710bcdb54ad3e6802cc3ab293c6916 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Wed, 27 Jan 2021 20:45:14 +0100
Subject: [PATCH 49/54] documentation

---
 docs/setup.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/setup.rst b/docs/setup.rst
index a3a0bd1e8..60a8b79d4 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -244,7 +244,7 @@ writing. Windows is not and will never be supported.
 
 1.  Install dependencies. Paperless requires the following packages.
 
-    *   ``python3`` 3.6, 3.7, 3.8 (3.9 is untested).
+    *   ``python3`` 3.6, 3.7, 3.8, 3.9
     *   ``python3-pip``, optionally ``pipenv`` for package installation
     *   ``python3-dev``
 

From fafe259e537b30301b604b6ec3ff8c6aef5cad36 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 28 Jan 2021 10:54:56 +0100
Subject: [PATCH 50/54] added a status summary line

---
 src-ui/messages.xlf                           | 35 +++++++++++++++----
 .../upload-file-widget.component.html         |  9 ++---
 .../upload-file-widget.component.ts           | 27 +++++++++++++-
 .../app/services/consumer-status.service.ts   |  6 +++-
 4 files changed, 64 insertions(+), 13 deletions(-)

diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf
index 84ca4888b..93f970d86 100644
--- a/src-ui/messages.xlf
+++ b/src-ui/messages.xlf
@@ -1414,25 +1414,46 @@
           <context context-type="linenumber">4</context>
         </context-group>
       </trans-unit>
+      <trans-unit id="6443586946875325554" datatype="html">
+        <source>Processing: <x id="PH" equiv-text="countUploadingAndProcessing"/></source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
+          <context context-type="linenumber">32</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="9182918211699394982" datatype="html">
+        <source>Failed: <x id="PH" equiv-text="countFailed"/></source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
+          <context context-type="linenumber">35</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="534116346205124059" datatype="html">
+        <source>Added: <x id="PH" equiv-text="countSuccess"/></source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
+          <context context-type="linenumber">38</context>
+        </context-group>
+      </trans-unit>
       <trans-unit id="3852289441366561594" datatype="html">
         <source>Connecting...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">93</context>
+          <context context-type="linenumber">118</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1245343823699368872" datatype="html">
         <source>Uploading...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">98</context>
+          <context context-type="linenumber">123</context>
         </context-group>
       </trans-unit>
       <trans-unit id="3994065460580948013" datatype="html">
         <source>Waiting for consumer...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts</context>
-          <context context-type="linenumber">101</context>
+          <context context-type="linenumber">126</context>
         </context-group>
       </trans-unit>
       <trans-unit id="e022072b3e4dd77e3f09960817ef3359a49963b3" datatype="html">
@@ -1463,18 +1484,18 @@
           <context context-type="linenumber">4</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="7cf3abd55bf1d2095435a050325927f083e6034e" datatype="html">
-        <source><x id="INTERPOLATION" equiv-text="{{getStatusesHidden().length}}"/> more hidden</source>
+      <trans-unit id="1fc4e0a1e93fdda0ed3c9e590971d283afb68265" datatype="html">
+        <source><x id="INTERPOLATION" equiv-text="{{getStatusHidden().length}}"/> more hidden</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
-          <context context-type="linenumber">23</context>
+          <context context-type="linenumber">24</context>
         </context-group>
       </trans-unit>
       <trans-unit id="710254a196a2649674438edf8a15b7ab1f48271b" datatype="html">
         <source>Open document</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html</context>
-          <context context-type="linenumber">40</context>
+          <context context-type="linenumber">41</context>
         </context-group>
       </trans-unit>
       <trans-unit id="45854ddec74086b271e62be6a363f4fa5036f7fc" datatype="html">
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
index 5cabb3c85..3e2908b84 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html
@@ -12,17 +12,18 @@
     <form>
       <ngx-file-drop dropZoneLabel="Drop documents here or" browseBtnLabel="Browse files" (onFileDrop)="dropped($event)"
         (onFileOver)="fileOver($event)" (onFileLeave)="fileLeave($event)" dropZoneClassName="bg-light card"
-        multiple="true" contentClassName="justify-content-center d-flex align-items-center p-5" [showBrowseBtn]=true
+        multiple="true" contentClassName="justify-content-center d-flex align-items-center py-5 px-2" [showBrowseBtn]=true
         browseBtnClassName="btn btn-sm btn-outline-primary ml-2" i18n-dropZoneLabel i18n-browseBtnLabel>
       </ngx-file-drop>
     </form>
+    <p class="mt-3" *ngIf="getStatus().length > 0">{{getStatusSummary()}}</p>
     <div *ngFor="let status of getStatus()">
       <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>
     </div>
-    <div *ngIf="getStatusesHidden().length" class="alerts-hidden">
-      <p *ngIf="!alertsExpanded" class="mt-3 mb-0 text-center"><span i18n>{{getStatusesHidden().length}} more hidden</span> <button class="btn btn-sm btn-link py-0" (click)="alertsExpanded = !alertsExpanded" aria-controls="hiddenAlerts" [attr.aria-expanded]="alertsExpanded" i18n>Show all</button></p>
+    <div *ngIf="getStatusHidden().length" class="alerts-hidden">
+      <p *ngIf="!alertsExpanded" class="mt-3 mb-0 text-center"><span i18n>{{getStatusHidden().length}} more hidden</span> <button class="btn btn-sm btn-link py-0" (click)="alertsExpanded = !alertsExpanded" aria-controls="hiddenAlerts" [attr.aria-expanded]="alertsExpanded" i18n>Show all</button></p>
       <div #hiddenAlerts="ngbCollapse" [(ngbCollapse)]="!alertsExpanded">
-        <div *ngFor="let status of getStatusesHidden()">
+        <div *ngFor="let status of getStatusHidden()">
           <ng-container [ngTemplateOutlet]="consumerAlert" [ngTemplateOutletContext]="{ $implicit: status }"></ng-container>
         </div>
       </div>
diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
index a6870e0e7..5ac68d42a 100644
--- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts
@@ -23,7 +23,24 @@ export class UploadFileWidgetComponent implements OnInit {
     return this.consumerStatusService.getConsumerStatus().slice(0, MAX_ALERTS)
   }
 
-  getStatusesHidden() {
+  getStatusSummary() {
+    let strings = []
+    let countUploadingAndProcessing =  this.consumerStatusService.getConsumerStatusNotCompleted().length
+    let countFailed = this.getStatusFailed().length
+    let countSuccess = this.getStatusSuccess().length
+    if (countUploadingAndProcessing > 0) {
+      strings.push($localize`Processing: ${countUploadingAndProcessing}`)
+    }
+    if (countFailed > 0) {
+      strings.push($localize`Failed: ${countFailed}`)
+    }
+    if (countSuccess > 0) {
+      strings.push($localize`Added: ${countSuccess}`)
+    }
+    return strings.join($localize`:this string is used to separate processing, failed and added on the file upload widget:, `)
+  }
+
+  getStatusHidden() {
     if (this.consumerStatusService.getConsumerStatus().length < MAX_ALERTS) return []
     else return this.consumerStatusService.getConsumerStatus().slice(MAX_ALERTS)
   }
@@ -32,6 +49,14 @@ export class UploadFileWidgetComponent implements OnInit {
     return this.consumerStatusService.getConsumerStatus(FileStatusPhase.UPLOADING)
   }
 
+  getStatusFailed() {
+    return this.consumerStatusService.getConsumerStatus(FileStatusPhase.FAILED)
+  }
+
+  getStatusSuccess() {
+    return this.consumerStatusService.getConsumerStatus(FileStatusPhase.SUCCESS)
+  }
+
   getStatusCompleted() {
     return this.consumerStatusService.getConsumerStatusCompleted()
   }
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index c2169d51f..8151b1c18 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -90,13 +90,17 @@ export class ConsumerStatusService {
   }
 
   getConsumerStatus(phase?: FileStatusPhase) {
-    if (phase) {
+    if (phase != null) {
       return this.consumerStatus.filter(s => s.phase == phase)
     } else {
       return this.consumerStatus
     }
   }
 
+  getConsumerStatusNotCompleted() {
+    return this.consumerStatus.filter(s => s.phase < FileStatusPhase.SUCCESS)
+  }
+
   getConsumerStatusCompleted() {
     return this.consumerStatus.filter(s => s.phase == FileStatusPhase.FAILED || s.phase == FileStatusPhase.SUCCESS)
   }

From 47189342e4488f63486a7a9e6efb9feeeb39e1a0 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 28 Jan 2021 12:43:27 +0100
Subject: [PATCH 51/54] update dependencies

---
 Pipfile          |  2 ++
 Pipfile.lock     | 12 ++++++------
 requirements.txt |  2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/Pipfile b/Pipfile
index 95d845d1f..cbe5c5103 100644
--- a/Pipfile
+++ b/Pipfile
@@ -44,6 +44,8 @@ inotifyrecursive = "~=0.3.4"
 ocrmypdf = "~=11.4.5"
 tqdm = "*"
 tika = "*"
+# TODO: This will sadly also install daphne+dependencies,
+#  which an ASGI server we don't need. Adds about 15MB image size.
 channels = "~=3.0"
 channels-redis = "*"
 uvicorn = {extras = ["standard"], version = "*"}
diff --git a/Pipfile.lock b/Pipfile.lock
index b8cb29b29..04e8b7fe0 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -403,11 +403,11 @@
         },
         "imap-tools": {
             "hashes": [
-                "sha256:4ec9575f21fb5a3efb573e71107806367dd52ecab0bf8f5c3724d1de6762b77b",
-                "sha256:860221dc13eb5d640da240e9bcf7867275c6162b742255789fd84f59124dfdd9"
+                "sha256:0eaa9b990fae336601dd44f353fac2d35ea25ca3b1b682a83700511635fc30ae",
+                "sha256:1c809e286d439e41fbe796c522ad4e565fd47a4260253343fa1b1045b6bfe8b1"
             ],
             "index": "pypi",
-            "version": "==0.36.0"
+            "version": "==0.37.0"
         },
         "img2pdf": {
             "hashes": [
@@ -1838,11 +1838,11 @@
         },
         "tox": {
             "hashes": [
-                "sha256:0aa777ee466f2ef18e6f58428c793c32378779e0a321dbb8934848bc3e78998c",
-                "sha256:f501808381c01c6d7827c2f17328be59c0a715046e94605ddca15fb91e65827d"
+                "sha256:76df3db6eee929bb62bdbacca5bb6bc840669d98e86a015b7a57b7df0a6eaf8b",
+                "sha256:854e6e4a71c614b488f81cb88df3b92edcb1a9ec43d4102e6289e9669bbf7f18"
             ],
             "index": "pypi",
-            "version": "==3.21.2"
+            "version": "==3.21.3"
         },
         "typing-extensions": {
             "hashes": [
diff --git a/requirements.txt b/requirements.txt
index 0c6deff1d..d6f45fea4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -42,7 +42,7 @@ httptools==0.1.1
 humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 hyperlink==21.0.0
 idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-imap-tools==0.36.0
+imap-tools==0.37.0
 img2pdf==0.4.0
 importlib-metadata==3.4.0; python_version < '3.8'
 incremental==17.5.0

From 94be8781e9dd3ecfad762af6ec5e4b062a94a8ac Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 28 Jan 2021 19:28:48 +0100
Subject: [PATCH 52/54] test cases

---
 src/documents/tests/test_consumer.py | 119 ++++++++++++++++++---------
 1 file changed, 82 insertions(+), 37 deletions(-)

diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py
index 54d20ca8b..22e6afb61 100644
--- a/src/documents/tests/test_consumer.py
+++ b/src/documents/tests/test_consumer.py
@@ -212,6 +212,20 @@ def fake_magic_from_file(file, mime=False):
 @mock.patch("documents.consumer.magic.from_file", fake_magic_from_file)
 class TestConsumer(DirectoriesMixin, TestCase):
 
+    def _assert_first_last_send_progress(self, first_status="STARTING", last_status="SUCCESS", first_progress=0, first_progress_max=100, last_progress=100, last_progress_max=100):
+
+        self._send_progress.assert_called()
+
+        args, kwargs = self._send_progress.call_args_list[0]
+        self.assertEqual(args[0], first_progress)
+        self.assertEqual(args[1], first_progress_max)
+        self.assertEqual(args[2], first_status)
+
+        args, kwargs = self._send_progress.call_args_list[len(self._send_progress.call_args_list) - 1]
+        self.assertEqual(args[0], last_progress)
+        self.assertEqual(args[1], last_progress_max)
+        self.assertEqual(args[2], last_status)
+
     def make_dummy_parser(self, logging_group, progress_callback=None):
         return DummyParser(logging_group, self.dirs.scratch_dir, self.get_test_archive_file())
 
@@ -232,7 +246,7 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         # this prevents websocket message reports during testing.
         patcher = mock.patch("documents.consumer.Consumer._send_progress")
-        patcher.start()
+        self._send_progress = patcher.start()
         self.addCleanup(patcher.stop)
 
         self.consumer = Consumer()
@@ -278,6 +292,8 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         self.assertFalse(os.path.isfile(filename))
 
+        self._assert_first_last_send_progress()
+
     def testOverrideFilename(self):
         filename = self.get_test_file()
         override_filename = "Statement for November.pdf"
@@ -286,21 +302,26 @@ class TestConsumer(DirectoriesMixin, TestCase):
 
         self.assertEqual(document.title, "Statement for November")
 
+        self._assert_first_last_send_progress()
+
     def testOverrideTitle(self):
         document = self.consumer.try_consume_file(self.get_test_file(), override_title="Override Title")
         self.assertEqual(document.title, "Override Title")
+        self._assert_first_last_send_progress()
 
     def testOverrideCorrespondent(self):
         c = Correspondent.objects.create(name="test")
 
         document = self.consumer.try_consume_file(self.get_test_file(), override_correspondent_id=c.pk)
         self.assertEqual(document.correspondent.id, c.id)
+        self._assert_first_last_send_progress()
 
     def testOverrideDocumentType(self):
         dt = DocumentType.objects.create(name="test")
 
         document = self.consumer.try_consume_file(self.get_test_file(), override_document_type_id=dt.pk)
         self.assertEqual(document.document_type.id, dt.id)
+        self._assert_first_last_send_progress()
 
     def testOverrideTags(self):
         t1 = Tag.objects.create(name="t1")
@@ -311,37 +332,42 @@ class TestConsumer(DirectoriesMixin, TestCase):
         self.assertIn(t1, document.tags.all())
         self.assertNotIn(t2, document.tags.all())
         self.assertIn(t3, document.tags.all())
+        self._assert_first_last_send_progress()
 
     def testNotAFile(self):
-        try:
-            self.consumer.try_consume_file("non-existing-file")
-        except ConsumerError as e:
-            self.assertTrue(str(e).endswith('File not found.'))
-            return
 
-        self.fail("Should throw exception")
+        self.assertRaisesMessage(
+            ConsumerError,
+            "File not found",
+            self.consumer.try_consume_file,
+            "non-existing-file"
+        )
+
+        self._assert_first_last_send_progress(last_status="FAILED")
 
     def testDuplicates1(self):
         self.consumer.try_consume_file(self.get_test_file())
 
-        try:
-            self.consumer.try_consume_file(self.get_test_file())
-        except ConsumerError as e:
-            self.assertTrue(str(e).endswith("It is a duplicate."))
-            return
+        self.assertRaisesMessage(
+            ConsumerError,
+            "It is a duplicate",
+            self.consumer.try_consume_file,
+            self.get_test_file()
+        )
 
-        self.fail("Should throw exception")
+        self._assert_first_last_send_progress(last_status="FAILED")
 
     def testDuplicates2(self):
         self.consumer.try_consume_file(self.get_test_file())
 
-        try:
-            self.consumer.try_consume_file(self.get_test_archive_file())
-        except ConsumerError as e:
-            self.assertTrue(str(e).endswith("It is a duplicate."))
-            return
+        self.assertRaisesMessage(
+            ConsumerError,
+            "It is a duplicate",
+            self.consumer.try_consume_file,
+            self.get_test_archive_file()
+        )
 
-        self.fail("Should throw exception")
+        self._assert_first_last_send_progress(last_status="FAILED")
 
     def testDuplicates3(self):
         self.consumer.try_consume_file(self.get_test_archive_file())
@@ -351,13 +377,15 @@ class TestConsumer(DirectoriesMixin, TestCase):
     def testNoParsers(self, m):
         m.return_value = []
 
-        try:
-            self.consumer.try_consume_file(self.get_test_file())
-        except ConsumerError as e:
-            self.assertEqual(str(e), "sample.pdf: Unsupported mime type application/pdf")
-            return
+        self.assertRaisesMessage(
+            ConsumerError,
+            "sample.pdf: Unsupported mime type application/pdf",
+            self.consumer.try_consume_file,
+            self.get_test_file()
+        )
+
+        self._assert_first_last_send_progress(last_status="FAILED")
 
-        self.fail("Should throw exception")
 
     @mock.patch("documents.parsers.document_consumer_declaration.send")
     def testFaultyParser(self, m):
@@ -367,24 +395,28 @@ class TestConsumer(DirectoriesMixin, TestCase):
             "weight": 0
         })]
 
-        try:
-            self.consumer.try_consume_file(self.get_test_file())
-        except ConsumerError as e:
-            self.assertEqual(str(e), "sample.pdf: Error while consuming document sample.pdf: Does not compute.")
-            return
+        self.assertRaisesMessage(
+            ConsumerError,
+            "sample.pdf: Error while consuming document sample.pdf: Does not compute.",
+            self.consumer.try_consume_file,
+            self.get_test_file()
+        )
 
-        self.fail("Should throw exception.")
+        self._assert_first_last_send_progress(last_status="FAILED")
 
     @mock.patch("documents.consumer.Consumer._write")
     def testPostSaveError(self, m):
         filename = self.get_test_file()
         m.side_effect = OSError("NO.")
-        try:
-            self.consumer.try_consume_file(filename)
-        except ConsumerError as e:
-            self.assertEqual(str(e), "sample.pdf: The following error occured while consuming sample.pdf: NO.")
-        else:
-            self.fail("Should raise exception")
+
+        self.assertRaisesMessage(
+            ConsumerError,
+            "sample.pdf: The following error occured while consuming sample.pdf: NO.",
+            self.consumer.try_consume_file,
+            filename
+        )
+
+        self._assert_first_last_send_progress(last_status="FAILED")
 
         # file not deleted
         self.assertTrue(os.path.isfile(filename))
@@ -401,6 +433,8 @@ class TestConsumer(DirectoriesMixin, TestCase):
         self.assertEqual(document.title, "new docs")
         self.assertEqual(document.filename, "none/new docs.pdf")
 
+        self._assert_first_last_send_progress()
+
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
     @mock.patch("documents.signals.handlers.generate_unique_filename")
     def testFilenameHandlingUnstableFormat(self, m):
@@ -424,6 +458,8 @@ class TestConsumer(DirectoriesMixin, TestCase):
         self.assertIsNotNone(os.path.isfile(document.title))
         self.assertTrue(os.path.isfile(document.source_path))
 
+        self._assert_first_last_send_progress()
+
     @mock.patch("documents.consumer.DocumentClassifier")
     def testClassifyDocument(self, m):
         correspondent = Correspondent.objects.create(name="test")
@@ -443,19 +479,26 @@ class TestConsumer(DirectoriesMixin, TestCase):
         self.assertIn(t1, document.tags.all())
         self.assertNotIn(t2, document.tags.all())
 
+        self._assert_first_last_send_progress()
+
     @override_settings(CONSUMER_DELETE_DUPLICATES=True)
     def test_delete_duplicate(self):
         dst = self.get_test_file()
         self.assertTrue(os.path.isfile(dst))
         doc = self.consumer.try_consume_file(dst)
 
+        self._assert_first_last_send_progress()
+
         self.assertFalse(os.path.isfile(dst))
         self.assertIsNotNone(doc)
 
+        self._send_progress.reset_mock()
+
         dst = self.get_test_file()
         self.assertTrue(os.path.isfile(dst))
         self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst)
         self.assertFalse(os.path.isfile(dst))
+        self._assert_first_last_send_progress(last_status="FAILED")
 
     @override_settings(CONSUMER_DELETE_DUPLICATES=False)
     def test_no_delete_duplicate(self):
@@ -471,6 +514,8 @@ class TestConsumer(DirectoriesMixin, TestCase):
         self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst)
         self.assertTrue(os.path.isfile(dst))
 
+        self._assert_first_last_send_progress(last_status="FAILED")
+
 
 class PreConsumeTestCase(TestCase):
 

From 2faa425caf1ff90f0a1824e84d16d8050e7fd9d3 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 28 Jan 2021 22:06:02 +0100
Subject: [PATCH 53/54] localization for websockets

---
 src-ui/messages.xlf                           | 91 +++++++++++++++++++
 .../app/services/consumer-status.service.ts   | 22 ++++-
 src/documents/consumer.py                     | 50 ++++++----
 src/documents/parsers.py                      |  5 +
 src/locale/en-us/LC_MESSAGES/django.po        | 64 ++-----------
 5 files changed, 154 insertions(+), 78 deletions(-)

diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf
index 93f970d86..eab8ed80e 100644
--- a/src-ui/messages.xlf
+++ b/src-ui/messages.xlf
@@ -1617,6 +1617,97 @@
           <context context-type="linenumber">85</context>
         </context-group>
       </trans-unit>
+      <trans-unit id="2119857572761283468" datatype="html">
+        <source>Document already exists.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">14</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="148389968432135849" datatype="html">
+        <source>File not found.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">15</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="1520671543092565667" datatype="html">
+        <source>Pre-consume script does not exist.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">16</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="7742915911032564889" datatype="html">
+        <source>Error while executing pre-consume script.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">17</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="8995193730018060346" datatype="html">
+        <source>Post-consume script does not exist.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">18</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="256773668518189604" datatype="html">
+        <source>Error while executing post-consume script.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">19</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="6252258095055634191" datatype="html">
+        <source>Received new file.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">20</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="7337565919209746135" datatype="html">
+        <source>File type not supported.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">21</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="5002399167376099234" datatype="html">
+        <source>Processing document...</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">22</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="1085975194762600381" datatype="html">
+        <source>Generating thumbnail...</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">23</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="3280851677698431426" datatype="html">
+        <source>Retrieving date from document...</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">24</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="7162102384876037296" datatype="html">
+        <source>Saving document...</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">25</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="4550450765009165976" datatype="html">
+        <source>Finished.</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/services/consumer-status.service.ts</context>
+          <context context-type="linenumber">26</context>
+        </context-group>
+      </trans-unit>
       <trans-unit id="1519954996184640001" datatype="html">
         <source>Error</source>
         <context-group purpose="location">
diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts
index 8151b1c18..026c3c64f 100644
--- a/src-ui/src/app/services/consumer-status.service.ts
+++ b/src-ui/src/app/services/consumer-status.service.ts
@@ -10,6 +10,22 @@ export enum FileStatusPhase {
   FAILED = 4
 }
 
+export const FILE_STATUS_MESSAGES = {
+  "document_already_exists": $localize`Document already exists.`,
+  "file_not_found": $localize`File not found.`,
+  "pre_consume_script_not_found": $localize`Pre-consume script does not exist.`,
+  "pre_consume_script_error": $localize`Error while executing pre-consume script.`,
+  "post_consume_script_not_found": $localize`Post-consume script does not exist.`,
+  "post_consume_script_error": $localize`Error while executing post-consume script.`,
+  "new_file": $localize`Received new file.`,
+  "unsupported_type": $localize`File type not supported.`,
+  "parsing_document": $localize`Processing document...`,
+  "generating_thumbnail": $localize`Generating thumbnail...`,
+  "parse_date": $localize`Retrieving date from document...`,
+  "save_document": $localize`Saving document...`,
+  "finished": $localize`Finished.`
+}
+
 export class FileStatus {
 
   filename: string
@@ -116,7 +132,11 @@ export class ConsumerStatusService {
       let created = statusMessageGet.created
 
       status.updateProgress(FileStatusPhase.PROCESSING, statusMessage.current_progress, statusMessage.max_progress)
-      status.message = statusMessage.message
+      if (statusMessage.message && statusMessage.message in FILE_STATUS_MESSAGES) {
+        status.message = FILE_STATUS_MESSAGES[statusMessage.message]
+      } else if (statusMessage.message) {
+        status.message = statusMessage.message
+      }
       status.documentId = statusMessage.document_id
 
       if (created && statusMessage.status == 'STARTING') {
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index 05043201b..146b11014 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -25,17 +25,30 @@ from .signals import (
     document_consumption_started
 )
 
-from django.utils.translation import gettext as _
-
 
 class ConsumerError(Exception):
     pass
 
 
+MESSAGE_DOCUMENT_ALREADY_EXISTS = "document_already_exists"
+MESSAGE_FILE_NOT_FOUND = "file_not_found"
+MESSAGE_PRE_CONSUME_SCRIPT_NOT_FOUND = "pre_consume_script_not_found"
+MESSAGE_PRE_CONSUME_SCRIPT_ERROR = "pre_consume_script_error"
+MESSAGE_POST_CONSUME_SCRIPT_NOT_FOUND = "post_consume_script_not_found"
+MESSAGE_POST_CONSUME_SCRIPT_ERROR = "post_consume_script_error"
+MESSAGE_NEW_FILE = "new_file"
+MESSAGE_UNSUPPORTED_TYPE = "unsupported_type"
+MESSAGE_PARSING_DOCUMENT = "parsing_document"
+MESSAGE_GENERATING_THUMBNAIL = "generating_thumbnail"
+MESSAGE_PARSE_DATE = "parse_date"
+MESSAGE_SAVE_DOCUMENT = "save_document"
+MESSAGE_FINISHED = "finished"
+
+
 class Consumer(LoggingMixin):
 
     def _send_progress(self, current_progress, max_progress, status,
-                       message, document_id=None):
+                       message=None, document_id=None):
         payload = {
             'filename': os.path.basename(self.filename) if self.filename else None,  # NOQA: E501
             'task_id': self.task_id,
@@ -69,7 +82,7 @@ class Consumer(LoggingMixin):
     def pre_check_file_exists(self):
         if not os.path.isfile(self.path):
             self._fail(
-                _("File not found"),
+                MESSAGE_FILE_NOT_FOUND,
                 f"Cannot consume {self.path}: File not found."
             )
 
@@ -80,7 +93,7 @@ class Consumer(LoggingMixin):
             if settings.CONSUMER_DELETE_DUPLICATES:
                 os.unlink(self.path)
             self._fail(
-                _("Document already exists"),
+                MESSAGE_DOCUMENT_ALREADY_EXISTS,
                 f"Not consuming {self.filename}: It is a duplicate."
             )
 
@@ -96,7 +109,7 @@ class Consumer(LoggingMixin):
 
         if not os.path.isfile(settings.PRE_CONSUME_SCRIPT):
             self._fail(
-                _("Pre-consume script does not exist."),
+                MESSAGE_PRE_CONSUME_SCRIPT_NOT_FOUND,
                 f"Configured pre-consume script "
                 f"{settings.PRE_CONSUME_SCRIPT} does not exist.")
 
@@ -104,7 +117,7 @@ class Consumer(LoggingMixin):
             Popen((settings.PRE_CONSUME_SCRIPT, self.path)).wait()
         except Exception as e:
             self._fail(
-                _("Error while executing pre-consume script"),
+                MESSAGE_PRE_CONSUME_SCRIPT_ERROR,
                 f"Error while executing pre-consume script: {e}"
             )
 
@@ -114,7 +127,7 @@ class Consumer(LoggingMixin):
 
         if not os.path.isfile(settings.POST_CONSUME_SCRIPT):
             self._fail(
-                _("Post-consume script does not exist."),
+                MESSAGE_POST_CONSUME_SCRIPT_NOT_FOUND,
                 f"Configured post-consume script "
                 f"{settings.POST_CONSUME_SCRIPT} does not exist."
             )
@@ -134,7 +147,7 @@ class Consumer(LoggingMixin):
             )).wait()
         except Exception as e:
             self._fail(
-                _("Error while executing post-consume script"),
+                MESSAGE_POST_CONSUME_SCRIPT_ERROR,
                 f"Error while executing post-consume script: {e}"
             )
 
@@ -158,7 +171,7 @@ class Consumer(LoggingMixin):
         self.override_tag_ids = override_tag_ids
         self.task_id = task_id or str(uuid.uuid4())
 
-        self._send_progress(0, 100, 'STARTING', _('Received new file'))
+        self._send_progress(0, 100, 'STARTING', MESSAGE_NEW_FILE)
 
         # this is for grouping logging entries for this particular file
         # together.
@@ -182,8 +195,7 @@ class Consumer(LoggingMixin):
         parser_class = get_parser_class_for_mime_type(mime_type)
         if not parser_class:
             self._fail(
-                _("File type %(type)s not supported") %
-                {'type': mime_type},
+                MESSAGE_UNSUPPORTED_TYPE,
                 f"Unsupported mime type {mime_type}"
             )
         else:
@@ -199,10 +211,10 @@ class Consumer(LoggingMixin):
 
         self.run_pre_consume_script()
 
-        def progress_callback(current_progress, max_progress, message):
+        def progress_callback(current_progress, max_progress):
             # recalculate progress to be within 20 and 80
             p = int((current_progress / max_progress) * 50 + 20)
-            self._send_progress(p, 100, "WORKING", message)
+            self._send_progress(p, 100, "WORKING")
 
         # This doesn't parse the document yet, but gives us a parser.
 
@@ -219,13 +231,13 @@ class Consumer(LoggingMixin):
         archive_path = None
 
         try:
-            self._send_progress(20, 100, 'WORKING', _('Parsing document...'))
+            self._send_progress(20, 100, 'WORKING', MESSAGE_PARSING_DOCUMENT)
             self.log("debug", "Parsing {}...".format(self.filename))
             document_parser.parse(self.path, mime_type, self.filename)
 
             self.log("debug", f"Generating thumbnail for {self.filename}...")
             self._send_progress(70, 100, 'WORKING',
-                                _('Generating thumbnail...'))
+                                MESSAGE_GENERATING_THUMBNAIL)
             thumbnail = document_parser.get_optimised_thumbnail(
                 self.path, mime_type)
 
@@ -233,7 +245,7 @@ class Consumer(LoggingMixin):
             date = document_parser.get_date()
             if not date:
                 self._send_progress(90, 100, 'WORKING',
-                                    _('Getting date from document...'))
+                                    MESSAGE_PARSE_DATE)
                 date = parse_date(self.filename, text)
             archive_path = document_parser.get_archive_path()
 
@@ -258,7 +270,7 @@ class Consumer(LoggingMixin):
                 "warning",
                 f"Cannot classify documents: {e}.")
             classifier = None
-        self._send_progress(95, 100, 'WORKING', _('Saving document...'))
+        self._send_progress(95, 100, 'WORKING', MESSAGE_SAVE_DOCUMENT)
         # now that everything is done, we can start to store the document
         # in the system. This will be a transaction and reasonably fast.
         try:
@@ -327,7 +339,7 @@ class Consumer(LoggingMixin):
             "Document {} consumption finished".format(document)
         )
 
-        self._send_progress(100, 100, 'SUCCESS', _('Finished.'), document.id)
+        self._send_progress(100, 100, 'SUCCESS', MESSAGE_FINISHED, document.id)
 
         return document
 
diff --git a/src/documents/parsers.py b/src/documents/parsers.py
index ddad6897a..3f0879b3c 100644
--- a/src/documents/parsers.py
+++ b/src/documents/parsers.py
@@ -273,6 +273,11 @@ class DocumentParser(LoggingMixin):
         self.date = None
         self.progress_callback = progress_callback
 
+    def progress(self, current, max):
+        print(self.progress_callback)
+        if self.progress_callback:
+            self.progress_callback(current, max)
+
     def extract_metadata(self, document_path, mime_type):
         return []
 
diff --git a/src/locale/en-us/LC_MESSAGES/django.po b/src/locale/en-us/LC_MESSAGES/django.po
index 8513c577b..fdf3fd809 100644
--- a/src/locale/en-us/LC_MESSAGES/django.po
+++ b/src/locale/en-us/LC_MESSAGES/django.po
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-01-27 17:57+0100\n"
+"POT-Creation-Date: 2021-01-28 22:02+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -21,60 +21,6 @@ msgstr ""
 msgid "Documents"
 msgstr ""
 
-#: documents/consumer.py:72
-msgid "File not found"
-msgstr "Datei nicht gefunden"
-
-#: documents/consumer.py:83
-msgid "Document already exists"
-msgstr ""
-
-#: documents/consumer.py:99
-msgid "Pre-consume script does not exist."
-msgstr ""
-
-#: documents/consumer.py:107
-msgid "Error while executing pre-consume script"
-msgstr ""
-
-#: documents/consumer.py:117
-msgid "Post-consume script does not exist."
-msgstr ""
-
-#: documents/consumer.py:137
-msgid "Error while executing post-consume script"
-msgstr ""
-
-#: documents/consumer.py:161
-msgid "Received new file"
-msgstr ""
-
-#: documents/consumer.py:185 documents/serialisers.py:383
-#, fuzzy, python-format
-#| msgid "File type {type} not supported."
-msgid "File type %(type)s not supported"
-msgstr "Dateityp {type} wird nicht unterstützt"
-
-#: documents/consumer.py:222
-msgid "Parsing document..."
-msgstr ""
-
-#: documents/consumer.py:228
-msgid "Generating thumbnail..."
-msgstr ""
-
-#: documents/consumer.py:236
-msgid "Getting date from document..."
-msgstr ""
-
-#: documents/consumer.py:261
-msgid "Saving document..."
-msgstr ""
-
-#: documents/consumer.py:330
-msgid "Finished."
-msgstr ""
-
 #: documents/models.py:33
 msgid "Any word"
 msgstr ""
@@ -392,6 +338,11 @@ msgstr ""
 msgid "filter rules"
 msgstr ""
 
+#: documents/serialisers.py:383
+#, python-format
+msgid "File type %(type)s not supported"
+msgstr ""
+
 #: documents/templates/index.html:20
 msgid "Paperless-ng is loading..."
 msgstr ""
@@ -674,6 +625,3 @@ msgstr ""
 #: paperless_mail/models.py:205
 msgid "assign this correspondent"
 msgstr ""
-
-#~ msgid "Document is a duplicate"
-#~ msgstr "Dokument existiert bereits"

From dcd350a30c3797bcf50891a8660b6bb0b156fc8d Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 29 Jan 2021 00:26:17 +0100
Subject: [PATCH 54/54] not sure what happened with reportlab 3.5.60, maybe a
 bad release.

---
 Pipfile.lock     | 86 ++++++++++++++++++++++++------------------------
 requirements.txt |  2 +-
 2 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index 04e8b7fe0..ff9e96717 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -926,50 +926,50 @@
         },
         "reportlab": {
             "hashes": [
-                "sha256:10d0cd2ab669fbad8b766db57066270296396caf515731643e7c7e732159a432",
-                "sha256:21508065492fbb750aa6d07297a2ba4bcacb418c84898fe892e12bdceefc3b9a",
-                "sha256:21c433905569af36bc220490a33f8f78442fd71c1bea3d021f851623c8089a78",
-                "sha256:236c4142474a59e4564b0e7c32e4d7b9cfc0dc2236ffa721ae97a4dbc2667d10",
-                "sha256:287b241a9d4189e7bfffd68aea0efd701d703bccee6686b9779bf16e22e9a2ad",
-                "sha256:2bc48a3cad6dd3cd1b5cd09526dad50afbda14afadd6fb40635a5227246bbe26",
-                "sha256:2e4631c49ec72f6a84502cda1710e988fa3027250e17c26713e543af31fff58a",
-                "sha256:2fd3305a75502d11942a9629cf6af96660a508f19adc4abaac5549909d1db1b3",
-                "sha256:32be7f9a55b8dfd91924730644632d19352fd9e0bb77cd03eba8a34fc846c4f6",
-                "sha256:33dc663ac3713e9d2f5a96b20412e16fd00b688b1b5a8057436e3fce69c2a59d",
-                "sha256:3d83cffa1211a7ae4912bfe3ac36051e31371cfb63b716e458c520da9f645fd9",
-                "sha256:3f9d56f5778b54260ce79878761b1f98267677702233daa26edd9f06f3be85c0",
-                "sha256:49a599f1aae7d91649fb10dbe070614740a60e4d2b65882a002dc20a7eba9f6f",
-                "sha256:4d5d62d49f3632cf437ac28ad52a8274fe6392ea2c48d294d2e92f1258d1479e",
-                "sha256:552ea83656f18e52ee073aee04d7e3f4d0eb71147f6882ae4fc15681d8e6ab1c",
-                "sha256:605f4a49ee313eabae4db4694d073dbb0778a0eccbc6d090e58c5a4db862c2a8",
-                "sha256:6289057afa4023f58d7977d105d09503c2cb1031e248fff52b44e41914d176cf",
-                "sha256:669526b580d52c05efbd45a8c9f1adf9370b8818bc2d48adec661ea7037415d4",
-                "sha256:6fc5f21d1383b552dc5b3eb774aaaf14cb7f8f34f15ea45772f72867dfaeecc6",
-                "sha256:74c7499fdda682028bb1b01d7ca89d813175c0f8c18d3ae9924c6393987ea4ef",
-                "sha256:75c0b6cf47c199f1e69283b0678b47537cc15af31acbad783fffe2c9cfa6a626",
-                "sha256:7b3ec1255c940fabfdb05577a4f148cca54943215cecb24efd5840a8ecb618ce",
-                "sha256:7d39eaca932b1d51de84c387616462913fa988c02a4a462798098f62edcf1e21",
-                "sha256:81e2faddfde7a21154d1a08e28e3d9b820274afa282b7ed17fcf1e9d7b4aa7a1",
-                "sha256:8aca938409fc99a9e79514cb97b3478780b261ff74a058c75a3a12552b1d6b9a",
-                "sha256:8f963222cc0c302ed5a15766ea00ff470c80c47d691e828ef21032df95e8a2ff",
-                "sha256:9be1ec218966e2e9982e73109863d91e3e4b9e7649ae6be0bf98b78c52a5a330",
-                "sha256:aa24f56c504d41982eb212f79669c05d4452f7736618dd33ba4167da0b5578b7",
-                "sha256:aa9caa395823130b360b5f3184eb25c9f1ea4fb51ab0370c2e693dc139cd7d83",
-                "sha256:b2f2bf419229bda5d6c8a10219ad1d984d2a6cd246acf613559d4a5fbcee0aea",
-                "sha256:b73f85e09e4271aee192489f5a8a7c33a1a82f42702cd0886f8f630600a23ad8",
-                "sha256:c3fac0186c18d7d5285f508d79bd86f751526190728ef2a33107568533f9aff0",
-                "sha256:c7ea886aa8dc3d3adb0c32ee87522821dd8ead2bac6d41e0481c1d911049c3c9",
-                "sha256:c9a487b646ac367d15bf6749deffa04ecbe339ba067d29414f206a7516a0a775",
-                "sha256:d0ae3cd60b659da41bb4962facec2ed3c8cfb3cedc85c5993e6299938134fc94",
-                "sha256:d6b900b1dc0d3c7b7f69fde25c8e60cb978b774056383c4a03bb38c829342299",
-                "sha256:d6feecad778e2a3d5a5ea48b027063e9baf73c6454235c1c1556b59c114c90cb",
-                "sha256:de5d98f41ffbe567717c334a9866c0f289a02cef37a78323cb5f8a86eecb6a8c",
-                "sha256:e47daf974b68fdcd2ffb5442ea5f1dd849eeaf6f9d4915128bf5374467e7b63a",
-                "sha256:e52536cba231fb3dd0628abf98b23519778e1247d4d69553d879eea6159b648d",
-                "sha256:f3181284502207e78eb472d00e8f9e84f43a7fe502821d2d76f3dd7da1d5e081",
-                "sha256:fc900dbc8f020305e15781210e64f6e96c1f9ca0fa00c4a0ccc1174cc44c5a5b"
+                "sha256:009fa61710647cdc62eb373345248d8ebb93583a058990f7c4f9be46d90aa5b1",
+                "sha256:04a08d284da86882ec3a41a7c719833362ef891b09ee8e2fbb47cee352aa684a",
+                "sha256:07bff6742fba612da8d1b1f783c436338c6fdc6962828159827d5ca7d2b67935",
+                "sha256:09fb11ab1500e679fc1b01199d2fed24435499856e75043a9ac0d31dd48fd881",
+                "sha256:18a876449c9000c391dd3415ebc8454cd7bb9e488977b894886a2d7d018f16cd",
+                "sha256:18eec161411026dde49767bee4e5e8eeb8014879554811a62581dc7433628d5b",
+                "sha256:19353aead39fc115a4d6c598d6fb9fa26da7e69160a0443ebb49b02903e704e8",
+                "sha256:1b85c20e89c22ae902ca973df2afdd2d64d27dc4ffd2b29ebad8c805a213756b",
+                "sha256:1da3d7a35f918cee905facfa94bd00ae6091cadc06dca1b0b31b69ae02d41d1d",
+                "sha256:1e484ce83dae26cb40fcbd312d45b8ba921de7856a00339d867dd4ecf145a1e7",
+                "sha256:33f3cfdc492575f8af3225701301a7e62fc478358729820c9e0091aff5831378",
+                "sha256:3b0026c1129147befd4e5a8cf25da8dea1096fce371e7b2412e36d7254019c06",
+                "sha256:3d7713dddaa8081ed709a1fa2456a43f6a74b0f07d605da8441fd53fef334f69",
+                "sha256:3e2b4d69763103b9dc9b54c0952dc3cee05cedd06e28c0987fad7f84705b12c0",
+                "sha256:4ca5233a19a5ceca23546290f43addec2345789c7d65bb32f8b2668aa148351f",
+                "sha256:5214a289cf01ebbd65e49bae83709671dd9edb601891cf0ae8abf85f3c0b392f",
+                "sha256:52f8237654acbc78ea2fa6fb4a6a06e5b023b6da93f7889adfe2deba09473fad",
+                "sha256:5ed00894e0f8281c0b7c0494b4d3067c641fd90c8e5cf933089ec4cc9a48e491",
+                "sha256:6191961533d49c9d860964d42bada4d7ac3bb28502d984feb8034093f2012fa8",
+                "sha256:6f3ad2b1afe99c436563cd436d8693d4a12e2c4bd45f70c7705759ff7837fe53",
+                "sha256:739b743b7ca1ba4b4d64c321de6fccb49b562d0507ea06c817d9cc4faed5cd22",
+                "sha256:792efba0c0c6e4ee94f6dc95f305451733ee9230a1c7d51cb8e5301a549e0dfb",
+                "sha256:79d63ca40231ca3860859b39a92daa5219035ba9553da89a5e1b218550744121",
+                "sha256:83b28104edd58ad65748d2d0e60e0d97e3b91b3e90b4573ea6fe60de6811972c",
+                "sha256:85650446538cd2f606ca234634142a7ccd74cb6db7cfec250f76a4242e0f2431",
+                "sha256:8850eba6de6eb813036eb8dce353e40d60c8af48bbce107de82770b10d3aa525",
+                "sha256:9da445cb79e3f740756924c053edc952cde11a65ff5af8acfda3c0a1317136ef",
+                "sha256:9fabd5fbd24f5971085ffe53150d663f158f7d3050b25c95736e29ebf676d454",
+                "sha256:a0c377bc45e73c3f15f55d7de69fab270d174749d5b454ab0de502b15430ec2a",
+                "sha256:a1d3f7022a920d4a5e165d264581f1862e1c1b877ceeabb96fe98cec98125ae5",
+                "sha256:a315edef5c5610b0c75790142f49487e89ea34397fc247ae8aa890fe6d6dd057",
+                "sha256:a755cca2dcf023130b03bb671670301a992157d5c3151d838c0b68ef89894536",
+                "sha256:b1b20208ecdfffd7ca027955c4fe8972b28b30a4b3b80cf25099a08d3b20ed7c",
+                "sha256:b26d6f416891cef93411d6d478a25db275766081a5fb66368248293ef459f3be",
+                "sha256:b4ba4c30af7044ee987e61c88a5ffb76031ca0c53666bc85d823b7de55ddbc75",
+                "sha256:b71faf3b6e4d7058e1af1b8afedaf39a962db4a219affc8177009d8244ec10d4",
+                "sha256:cfa854bea525f8c913cb77e2bda724d94b965a0eb3bcfc4a645a9baa29bb86e2",
+                "sha256:dd9687359e466086b9f6fe6d8069034017f8b6ca3080944fae5709767ca6814e",
+                "sha256:de0c675fc2998a7eaa929c356ba49c84f53a892e9ab25e8ee7d8ebbbdcb2ac16",
+                "sha256:e2b4e33fea2ce9d3a14ea39191b169e41eb2ac995274f54ac8fd27519974bce8",
+                "sha256:f3d4a1a273dc141e03b72a553c11bc14dd7a27ec7654a071edcf83eb04f004bc",
+                "sha256:ff547cf4c1de7e104cad1a378431ff81efcb03e90e40871ee686107da5b91442"
             ],
-            "version": "==3.5.60"
+            "version": "==3.5.59"
         },
         "requests": {
             "hashes": [
diff --git a/requirements.txt b/requirements.txt
index d6f45fea4..c3f32002b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -75,7 +75,7 @@ pytz==2020.5
 pyyaml==5.4.1
 redis==3.5.3
 regex==2020.11.13
-reportlab==3.5.60
+reportlab==3.5.59
 requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 scikit-learn==0.24.0
 scipy==1.5.4