diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f27d258b8..a1e9640d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ['3.6', '3.7', '3.8'] + python-version: ['3.6', '3.7', '3.8', '3.9'] fail-fast: false steps: - @@ -226,7 +226,7 @@ jobs: # build and push image to docker hub. build-docker-image: - if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-')) + if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/ng-')) runs-on: ubuntu-latest needs: [frontend, tests] steps: diff --git a/Pipfile b/Pipfile index 15da2503a..cbe5c5103 100644 --- a/Pipfile +++ b/Pipfile @@ -33,7 +33,10 @@ python-Levenshtein = "*" python-magic = "*" psycopg2-binary = "*" redis = "*" -scikit-learn="~=0.24.0" +# Pinned because aarch64 wheels and updates cause warnings when loading the classifier model. +scikit-learn="==0.24.0" +# Prevent scipy updates because 1.6 is incompatible with python 3.6 +scipy="~=1.5.4" whitenoise = "~=5.2.0" watchdog = "*" whoosh="~=2.7.4" @@ -41,6 +44,11 @@ inotifyrecursive = "~=0.3.4" ocrmypdf = "~=11.4.5" tqdm = "*" tika = "*" +# TODO: This will sadly also install daphne+dependencies, +# which an ASGI server we don't need. Adds about 15MB image size. +channels = "~=3.0" +channels-redis = "*" +uvicorn = {extras = ["standard"], version = "*"} [dev-packages] coveralls = "*" diff --git a/Pipfile.lock b/Pipfile.lock index dab52dbbd..ff9e96717 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3c85a487240f18b3feb44f8899395696cb79630f320f0df9ef5ee37b914c89f2" + "sha256": "d80d2539a4528a8fd9e848875c2e2d5bcdb3e98154f45b612706094b84ecaaea" }, "pipfile-spec": 6, "requires": {}, @@ -19,6 +19,13 @@ ] }, "default": { + "aioredis": { + "hashes": [ + "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a", + "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3" + ], + "version": "==1.3.1" + }, "arrow": { "hashes": [ "sha256:e098abbd9af3665aea81bdd6c869e93af4feb078e98468dd351c383af187aac5", @@ -35,6 +42,38 @@ "markers": "python_version >= '3.5'", "version": "==3.3.1" }, + "async-timeout": { + "hashes": [ + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" + ], + "markers": "python_full_version >= '3.5.3'", + "version": "==3.0.1" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.3.0" + }, + "autobahn": { + "hashes": [ + "sha256:410a93e0e29882c8b5d5ab05d220b07609b886ef5f23c0b8d39153254ffd6895", + "sha256:52ee4236ff9a1fcbbd9500439dcf3284284b37f8a6b31ecc8a36e00cf9f95049" + ], + "markers": "python_version >= '3.6'", + "version": "==20.12.3" + }, + "automat": { + "hashes": [ + "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33", + "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111", + "sha256:d6d976cf8da698fc85fa7def46e2544493f78cb7ee72d2f4acd1a5c759a3060e" + ], + "version": "==20.2.0" + }, "blessed": { "hashes": [ "sha256:0a74a8d3f0366db600d061273df77d44f0db07daade7bb7a4d49c8bc22ed9f74", @@ -92,6 +131,22 @@ ], "version": "==1.14.4" }, + "channels": { + "hashes": [ + "sha256:056b72e51080a517a0f33a0a30003e03833b551d75394d6636c885d4edb8188f", + "sha256:3f15bdd2138bb4796e76ea588a0a344b12a7964ea9b2e456f992fddb988a4317" + ], + "index": "pypi", + "version": "==3.0.3" + }, + "channels-redis": { + "hashes": [ + "sha256:18d63f6462a58011740dc8eeb57ea4b31ec220eb551cb71b27de9c6779a549de", + "sha256:2fb31a63b05373f6402da2e6a91a22b9e66eb8b56626c6bfc93e156c734c5ae6" + ], + "index": "pypi", + "version": "==3.2.0" + }, "chardet": { "hashes": [ "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", @@ -100,6 +155,15 @@ "markers": "python_version >= '3.1'", "version": "==4.0.0" }, + "click": { + "hashes": [ + "sha256:a3747c864f8e400a3664f5f4fd6dae11b4605bf6b727dae7b6f22ba9bd0a194a", + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, "coloredlogs": { "hashes": [ "sha256:5e78691e2673a8e294499e1832bb13efcfb44a86b92e18109fa18951093218ab", @@ -108,6 +172,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==15.0" }, + "constantly": { + "hashes": [ + "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35", + "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d" + ], + "version": "==15.1.0" + }, "cryptography": { "hashes": [ "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d", @@ -129,6 +200,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==3.3.1" }, + "daphne": { + "hashes": [ + "sha256:0052c9887600c57054a5867d4b0240159fa009faa3bcf6a1627271d9cdcb005a", + "sha256:c22b692707f514de9013651ecb687f2abe4f35cf6fe292ece634e9f1737bc7e3" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, "dateparser": { "hashes": [ "sha256:7552c994f893b5cb8fcf103b4cd2ff7f57aab9bfd2619fdf0cf571c0740fd90b", @@ -147,11 +226,11 @@ }, "django-cors-headers": { "hashes": [ - "sha256:5665fc1b1aabf1b678885cf6f8f8bd7da36ef0a978375e767d491b48d3055d8f", - "sha256:ba898dd478cd4be3a38ebc3d8729fa4d044679f8c91b2684edee41129d7e968a" + "sha256:1ac2b1213de75a251e2ba04448da15f99bcfcbe164288ae6b5ff929dc49b372f", + "sha256:96069c4aaacace786a34ee7894ff680780ec2644e4268b31181044410fecd12e" ], "index": "pypi", - "version": "==3.6.0" + "version": "==3.7.0" }, "django-extensions": { "hashes": [ @@ -217,6 +296,87 @@ "index": "pypi", "version": "==20.0.4" }, + "h11": { + "hashes": [ + "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6", + "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042" + ], + "markers": "python_version >= '3.6'", + "version": "==0.12.0" + }, + "hiredis": { + "hashes": [ + "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680", + "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0", + "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0", + "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01", + "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a", + "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b", + "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6", + "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73", + "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee", + "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55", + "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12", + "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b", + "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323", + "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c", + "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655", + "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5", + "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75", + "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb", + "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23", + "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1", + "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f", + "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872", + "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058", + "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454", + "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882", + "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2", + "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132", + "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6", + "sha256:9f4e67f87e072de981570eaf7cb41444bbac7e92b05c8651dbab6eb1fb8d5a14", + "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c", + "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363", + "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3", + "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4", + "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919", + "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349", + "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae", + "sha256:b39989b49e8aca9d224324d2650029eda410a4faf43f6afb0eb4f9acb7be6097", + "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da", + "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f", + "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed", + "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628", + "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64", + "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86", + "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf", + "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c", + "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded", + "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390", + "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.0" + }, + "httptools": { + "hashes": [ + "sha256:0a4b1b2012b28e68306575ad14ad5e9120b34fccd02a81eb08838d7e3bbb48be", + "sha256:3592e854424ec94bd17dc3e0c96a64e459ec4147e6d53c0a42d0ebcef9cb9c5d", + "sha256:41b573cf33f64a8f8f3400d0a7faf48e1888582b6f6e02b82b9bd4f0bf7497ce", + "sha256:56b6393c6ac7abe632f2294da53f30d279130a92e8ae39d8d14ee2e1b05ad1f2", + "sha256:7f6c82262d3bdde886a29ae0d65d4fae6b3ac6fba763891ddb72e72e1dbe7075", + "sha256:86c6acd66765a934e8730bf0e9dfaac6fdcf2a4334212bd4a0a1c78f16475ca6", + "sha256:96da81e1992be8ac2fd5597bf0283d832287e20cb3cfde8996d2b00356d4e17f", + "sha256:96eb359252aeed57ea5c7b3d79839aaa0382c9d3149f7d24dd7172b1bcecb009", + "sha256:a2719e1d7a84bb131c4f1e0cb79705034b48de6ae486eb5297a139d6a3296dce", + "sha256:ac0aa11e99454b6a66989aa2d44bca41d4e0f968e395a0a8f164b401fefe359a", + "sha256:bc3114b9edbca5a1eb7ae7db698c669eb53eb8afbbebdde116c174925260849c", + "sha256:cb46a65a0ed99c38dfcbf9f5be8be6cf9cb497e527505fefac7cbd38a467f3c6", + "sha256:fa3cd71e31436911a44620473e873a256851e1f53dee56669dae403ba41756a4", + "sha256:fea04e126014169384dee76a153d4573d90d0cbd1d12185da089f73c78390437" + ], + "version": "==0.1.1" + }, "humanfriendly": { "hashes": [ "sha256:066562956639ab21ff2676d1fda0b5987e985c534fc76700a19bd54bcb81121d", @@ -225,6 +385,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==9.1" }, + "hyperlink": { + "hashes": [ + "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b", + "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4" + ], + "version": "==21.0.0" + }, "idna": { "hashes": [ "sha256:4a57a6379512ade94fa99e2fa46d3cd0f2f553040548d0e2958c6ed90ee48226", @@ -236,11 +403,11 @@ }, "imap-tools": { "hashes": [ - "sha256:7d2d25b35117a3750c3b561dd93cc2fcb24cdc457830a049796c639f4371e317", - "sha256:80088839cd1959f20c44206cdad4463ca1e7647ff67cf5b0e31e810fb6aaa6c4" + "sha256:0eaa9b990fae336601dd44f353fac2d35ea25ca3b1b682a83700511635fc30ae", + "sha256:1c809e286d439e41fbe796c522ad4e565fd47a4260253343fa1b1045b6bfe8b1" ], "index": "pypi", - "version": "==0.34.0" + "version": "==0.37.0" }, "img2pdf": { "hashes": [ @@ -257,6 +424,13 @@ "markers": "python_version < '3.8'", "version": "==3.4.0" }, + "incremental": { + "hashes": [ + "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", + "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3" + ], + "version": "==17.5.0" + }, "inotify-simple": { "hashes": [ "sha256:8440ffe49c4ae81a8df57c1ae1eb4b6bfa7acb830099bfb3e305b383005cc128", @@ -335,6 +509,41 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==4.6.2" }, + "msgpack": { + "hashes": [ + "sha256:0cb94ee48675a45d3b86e61d13c1e6f1696f0183f0715544976356ff86f741d9", + "sha256:1026dcc10537d27dd2d26c327e552f05ce148977e9d7b9f1718748281b38c841", + "sha256:26a1759f1a88df5f1d0b393eb582ec022326994e311ba9c5818adc5374736439", + "sha256:2a5866bdc88d77f6e1370f82f2371c9bc6fc92fe898fa2dec0c5d4f5435a2694", + "sha256:31c17bbf2ae5e29e48d794c693b7ca7a0c73bd4280976d408c53df421e838d2a", + "sha256:497d2c12426adcd27ab83144057a705efb6acc7e85957a51d43cdcf7f258900f", + "sha256:5a9ee2540c78659a1dd0b110f73773533ee3108d4e1219b5a15a8d635b7aca0e", + "sha256:7307e86f7ce75b49e65b55660b10b258e9e7b5e0f80d31d7a86a278d8204d1b4", + "sha256:8521e5be9e3b93d4d5e07cb80b7e32353264d143c1f072309e1863174c6aadb1", + "sha256:87869ba567fe371c4555d2e11e4948778ab6b59d6cc9d8460d543e4cfbbddd1c", + "sha256:8ffb24a3b7518e843cd83538cf859e026d24ec41ac5721c18ed0c55101f9775b", + "sha256:92be4b12de4806d3c36810b0fe2aeedd8d493db39e2eb90742b9c09299eb5759", + "sha256:9ea52fff0473f9f3000987f313310208c879493491ef3ccf66268eff8d5a0326", + "sha256:a4355d2193106c7aa77c98fc955252a737d8550320ecdb2e9ac701e15e2943bc", + "sha256:a99b144475230982aee16b3d249170f1cccebf27fb0a08e9f603b69637a62192", + "sha256:ac25f3e0513f6673e8b405c3a80500eb7be1cf8f57584be524c4fa78fe8e0c83", + "sha256:b28c0876cce1466d7c2195d7658cf50e4730667196e2f1355c4209444717ee06", + "sha256:b55f7db883530b74c857e50e149126b91bb75d35c08b28db12dcb0346f15e46e", + "sha256:b6d9e2dae081aa35c44af9c4298de4ee72991305503442a5c74656d82b581fe9", + "sha256:c747c0cc08bd6d72a586310bda6ea72eeb28e7505990f342552315b229a19b33", + "sha256:c82dc0ba34d620fb94d12a7725e9362958bb1be3938688a061f53ed86bee005a", + "sha256:d6c64601af8f3893d17ec233237030e3110f11b8a962cb66720bf70c0141aa54", + "sha256:d8167b84af26654c1124857d71650404336f4eb5cc06900667a493fc619ddd9f", + "sha256:de6bd7990a2c2dabe926b7e62a92886ccbf809425c347ae7de277067f97c2887", + "sha256:e36a812ef4705a291cdb4a2fd352f013134f26c6ff63477f20235138d1d21009", + "sha256:e89ec55871ed5473a041c0495b7b4e6099f6263438e0bd04ccd8418f92d5d7f2", + "sha256:f3e6aaf217ac1c7ce1563cf52a2f4f5d5b1f64e8729d794165db71da57257f0c", + "sha256:f484cd2dca68502de3704f056fa9b318c94b1539ed17a4c784266df5d6978c87", + "sha256:fae04496f5bc150eefad4e9571d1a76c55d021325dcd484ce45065ebbdd00984", + "sha256:fe07bc6735d08e492a327f496b7850e98cb4d112c56df69b0c844dbebcbb47f6" + ], + "version": "==1.0.2" + }, "numpy": { "hashes": [ "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94", @@ -524,6 +733,42 @@ "index": "pypi", "version": "==2.8.6" }, + "pyasn1": { + "hashes": [ + "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", + "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", + "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", + "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", + "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", + "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", + "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", + "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12", + "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", + "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", + "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3" + ], + "version": "==0.4.8" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8", + "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199", + "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811", + "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed", + "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4", + "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", + "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74", + "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb", + "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45", + "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd", + "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0", + "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d", + "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405" + ], + "version": "==0.2.8" + }, "pycparser": { "hashes": [ "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", @@ -532,6 +777,22 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.20" }, + "pyhamcrest": { + "hashes": [ + "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316", + "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29" + ], + "markers": "python_version >= '3.5'", + "version": "==2.0.2" + }, + "pyopenssl": { + "hashes": [ + "sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51", + "sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==20.0.1" + }, "python-dateutil": { "hashes": [ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", @@ -558,13 +819,12 @@ }, "python-levenshtein": { "hashes": [ - "sha256:033a11de5e3d19ea25c9302d11224e1a1898fe5abd23c61c7c360c25195e3eb1", - "sha256:15e26882728c29ccdf74cfc6ac4b49fc22c08b44d152348cb0eb1ec4f3dbf9df", - "sha256:3df5e5eb144570ecf5ad38864a2393068798328c7f05e7b167a49391d36a2db1", - "sha256:7f049b3ddc4b525bd469febafb98bf5202f789b722e0e4ccbec2ffbe8c07d7b4" + "sha256:108edd3c271f1afda8b21a8d9da81886414dfb6940a085fa7903c592e0f8f54b", + "sha256:1ff19b712c5974080b003fd26ef365cd93dfc1a5e690be621f79f3e63e00a7cc", + "sha256:554e273a88060d177e7b3c1e6ea9158dde11563bfae8f7f661f73f47e5ff0911" ], "index": "pypi", - "version": "==0.12.0" + "version": "==0.12.1" }, "python-magic": { "hashes": [ @@ -581,6 +841,32 @@ ], "version": "==2020.5" }, + "pyyaml": { + "hashes": [ + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc" + ], + "version": "==5.4.1" + }, "redis": { "hashes": [ "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", @@ -754,9 +1040,16 @@ "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012", "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c" ], - "markers": "python_version >= '3.6'", + "index": "pypi", "version": "==1.5.4" }, + "service-identity": { + "hashes": [ + "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36", + "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d" + ], + "version": "==18.1.0" + }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", @@ -804,6 +1097,48 @@ "index": "pypi", "version": "==4.56.0" }, + "twisted": { + "extras": [ + "tls" + ], + "hashes": [ + "sha256:0150dae5adc962d15e00054cc6926f1e64763fb8dd26e1632593ac06e592104b", + "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f", + "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042", + "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c", + "sha256:15e52271f08f62e2230ff093e0278aa01c9dac057c4557cadadd2429eed86a3e", + "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292", + "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22", + "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec", + "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478", + "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2", + "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29", + "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114", + "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797", + "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa", + "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15", + "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd", + "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274", + "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad", + "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7", + "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a", + "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10", + "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780", + "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504", + "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467", + "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==20.3.0" + }, + "txaio": { + "hashes": [ + "sha256:1488d31d564a116538cc1265ac3f7979fb6223bb5a9e9f1479436ee2c17d8549", + "sha256:a8676d6c68aea1f0e2548c4afdb8e6253873af3bc2659bb5bcd9f39dff7ff90f" + ], + "markers": "python_version >= '3.6'", + "version": "==20.12.1" + }, "typing-extensions": { "hashes": [ "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", @@ -822,11 +1157,38 @@ }, "urllib3": { "hashes": [ - "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", - "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", + "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.2" + "version": "==1.26.3" + }, + "uvicorn": { + "extras": [ + "standard" + ], + "hashes": [ + "sha256:1079c50a06f6338095b4f203e7861dbff318dde5f22f3a324fc6e94c7654164c", + "sha256:ef1e0bb5f7941c6fe324e06443ddac0331e1632a776175f87891c7bd02694355" + ], + "index": "pypi", + "version": "==0.13.3" + }, + "uvloop": { + "hashes": [ + "sha256:08b109f0213af392150e2fe6f81d33261bb5ce968a288eb698aad4f46eb711bd", + "sha256:123ac9c0c7dd71464f58f1b4ee0bbd81285d96cdda8bc3519281b8973e3a461e", + "sha256:2e57a28567d874afd803b6d8a7aeee7bfa0a34994f1b237837e6d7e525cea9a4", + "sha256:4315d2ec3ca393dd5bc0b0089d23101276778c304d42faff5dc4579cb6caef09", + "sha256:4544dcf77d74f3a84f03dd6278174575c44c67d7165d4c42c71db3fdc3860726", + "sha256:63c0bd965e73a4e8dde6a15635262ec38036d83327230552ada21298ee5fa3b7", + "sha256:afd5513c0ae414ec71d24f6f123614a80f3d27ca655a4fcf6cabe50994cc1891", + "sha256:b4f591aa4b3fa7f32fb51e2ee9fea1b495eb75b0b3c8d0ca52514ad675ae63f7", + "sha256:bcac356d62edd330080aed082e78d4b580ff260a677508718f88016333e2c9c5", + "sha256:e7514d7a48c063226b7d06617cbb12a14278d4323a065a8d46a7962686ce2e95", + "sha256:f07909cd9fc08c52d294b1570bba92186181ca01fe3dc9ffba68955273dd7362" + ], + "version": "==0.14.0" }, "watchdog": { "hashes": [ @@ -851,6 +1213,14 @@ "index": "pypi", "version": "==1.0.2" }, + "watchgod": { + "hashes": [ + "sha256:59700dab7445aa8e6067a5b94f37bae90fc367554549b1ed2e9d0f4f38a90d2a", + "sha256:5fb60afa9558b79736395db1cb60ad3ed59df5c2f507a3ff729220cf1251ffdc", + "sha256:e9cca0ab9c63f17fc85df9fd8bd18156ff00aff04ebe5976cee473f4968c6858" + ], + "version": "==0.6" + }, "wcwidth": { "hashes": [ "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", @@ -858,6 +1228,34 @@ ], "version": "==0.2.5" }, + "websockets": { + "hashes": [ + "sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5", + "sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5", + "sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308", + "sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb", + "sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a", + "sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c", + "sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170", + "sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422", + "sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8", + "sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485", + "sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f", + "sha256:745a1c8ca62f7d27de42b517ca7c6f716d1eb96c5aa73cf6407936c0167cbb3c", + "sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8", + "sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc", + "sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779", + "sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989", + "sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1", + "sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092", + "sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824", + "sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d", + "sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55", + "sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36", + "sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b" + ], + "version": "==8.1" + }, "whitenoise": { "hashes": [ "sha256:05ce0be39ad85740a78750c86a93485c40f08ad8c62a6006de0233765996e5c7", @@ -882,6 +1280,66 @@ ], "markers": "python_version >= '3.6'", "version": "==3.4.0" + }, + "zope.interface": { + "hashes": [ + "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1", + "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d", + "sha256:09fc3922f235703c0b76f8234867685eee68a24a49fffa2220975f6142db45f1", + "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123", + "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232", + "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549", + "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102", + "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5", + "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45", + "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00", + "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc", + "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7", + "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104", + "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034", + "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3", + "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3", + "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4", + "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86", + "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96", + "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546", + "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb", + "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3", + "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b", + "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b", + "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec", + "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae", + "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e", + "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386", + "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2", + "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a", + "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d", + "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a", + "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24", + "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d", + "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b", + "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50", + "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523", + "sha256:974f5957e66a7524ea81df7b2686a456bfaf0408dbb7353ddfbedb594eadfef6", + "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a", + "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095", + "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a", + "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520", + "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65", + "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11", + "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c", + "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7", + "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332", + "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e", + "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c", + "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7", + "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20", + "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc", + "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd", + "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==5.2.0" } }, "develop": { @@ -940,60 +1398,60 @@ }, "coverage": { "hashes": [ - "sha256:08b3ba72bd981531fd557f67beee376d6700fba183b167857038997ba30dd297", - "sha256:262066798d786ad67a13c7ba869e3ce0e39609f99f6d6c80160ad602c4808e32", - "sha256:2757fa64e11ec12220968f65d086b7a29b6583d16e9a544c889b22ba98555ef1", - "sha256:3102bb2c206700a7d28181dbe04d66b30780cde1d1c02c5f3c165cf3d2489497", - "sha256:3498b27d8236057def41de3585f317abae235dd3a11d33e01736ffedb2ef8606", - "sha256:378ac77af41350a8c6b8801a66021b52da8a05fd77e578b7380e876c0ce4f528", - "sha256:38f16b1317b8dd82df67ed5daa5f5e7c959e46579840d77a67a4ceb9cef0a50b", - "sha256:3911c2ef96e5ddc748a3c8b4702c61986628bb719b8378bf1e4a6184bbd48fe4", - "sha256:3a3c3f8863255f3c31db3889f8055989527173ef6192a283eb6f4db3c579d830", - "sha256:3b14b1da110ea50c8bcbadc3b82c3933974dbeea1832e814aab93ca1163cd4c1", - "sha256:535dc1e6e68fad5355f9984d5637c33badbdc987b0c0d303ee95a6c979c9516f", - "sha256:6f61319e33222591f885c598e3e24f6a4be3533c1d70c19e0dc59e83a71ce27d", - "sha256:723d22d324e7997a651478e9c5a3120a0ecbc9a7e94071f7e1954562a8806cf3", - "sha256:76b2775dda7e78680d688daabcb485dc87cf5e3184a0b3e012e1d40e38527cc8", - "sha256:782a5c7df9f91979a7a21792e09b34a658058896628217ae6362088b123c8500", - "sha256:7e4d159021c2029b958b2363abec4a11db0ce8cd43abb0d9ce44284cb97217e7", - "sha256:8dacc4073c359f40fcf73aede8428c35f84639baad7e1b46fce5ab7a8a7be4bb", - "sha256:8f33d1156241c43755137288dea619105477961cfa7e47f48dbf96bc2c30720b", - "sha256:8ffd4b204d7de77b5dd558cdff986a8274796a1e57813ed005b33fd97e29f059", - "sha256:93a280c9eb736a0dcca19296f3c30c720cb41a71b1f9e617f341f0a8e791a69b", - "sha256:9a4f66259bdd6964d8cf26142733c81fb562252db74ea367d9beb4f815478e72", - "sha256:9a9d4ff06804920388aab69c5ea8a77525cf165356db70131616acd269e19b36", - "sha256:a2070c5affdb3a5e751f24208c5c4f3d5f008fa04d28731416e023c93b275277", - "sha256:a4857f7e2bc6921dbd487c5c88b84f5633de3e7d416c4dc0bb70256775551a6c", - "sha256:a607ae05b6c96057ba86c811d9c43423f35e03874ffb03fbdcd45e0637e8b631", - "sha256:a66ca3bdf21c653e47f726ca57f46ba7fc1f260ad99ba783acc3e58e3ebdb9ff", - "sha256:ab110c48bc3d97b4d19af41865e14531f300b482da21783fdaacd159251890e8", - "sha256:b239711e774c8eb910e9b1ac719f02f5ae4bf35fa0420f438cdc3a7e4e7dd6ec", - "sha256:be0416074d7f253865bb67630cf7210cbc14eb05f4099cc0f82430135aaa7a3b", - "sha256:c46643970dff9f5c976c6512fd35768c4a3819f01f61169d8cdac3f9290903b7", - "sha256:c5ec71fd4a43b6d84ddb88c1df94572479d9a26ef3f150cef3dacefecf888105", - "sha256:c6e5174f8ca585755988bc278c8bb5d02d9dc2e971591ef4a1baabdf2d99589b", - "sha256:c89b558f8a9a5a6f2cfc923c304d49f0ce629c3bd85cb442ca258ec20366394c", - "sha256:cc44e3545d908ecf3e5773266c487ad1877be718d9dc65fc7eb6e7d14960985b", - "sha256:cc6f8246e74dd210d7e2b56c76ceaba1cc52b025cd75dbe96eb48791e0250e98", - "sha256:cd556c79ad665faeae28020a0ab3bda6cd47d94bec48e36970719b0b86e4dcf4", - "sha256:ce6f3a147b4b1a8b09aae48517ae91139b1b010c5f36423fa2b866a8b23df879", - "sha256:ceb499d2b3d1d7b7ba23abe8bf26df5f06ba8c71127f188333dddcf356b4b63f", - "sha256:cef06fb382557f66d81d804230c11ab292d94b840b3cb7bf4450778377b592f4", - "sha256:e448f56cfeae7b1b3b5bcd99bb377cde7c4eb1970a525c770720a352bc4c8044", - "sha256:e52d3d95df81c8f6b2a1685aabffadf2d2d9ad97203a40f8d61e51b70f191e4e", - "sha256:eb33c4c858d06bd8d79713c7628d3f2b50fb1c62071e2e88cb44876be03eabe1", - "sha256:ee2f1d1c223c3d2c24e3afbb2dd38be3f03b1a8d6a83ee3d9eb8c36a52bee899", - "sha256:f2c6888eada180814b8583c3e793f3f343a692fc802546eed45f40a001b1169f", - "sha256:f51dbba78d68a44e99d484ca8c8f604f17e957c1ca09c3ebc2c7e3bbd9ba0448", - "sha256:f54de00baf200b4539a5a092a759f000b5f45fd226d6d25a76b0dff71177a714", - "sha256:fa10fee7e32213f5c7b0d6428ea92e3a3fdd6d725590238a3f92c0de1c78b9d2", - "sha256:fabeeb121735d47d8eab8671b6b031ce08514c86b7ad8f7d5490a7b6dcd6267d", - "sha256:fac3c432851038b3e6afe086f777732bcf7f6ebbfd90951fa04ee53db6d0bcdd", - "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7", - "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae" + "sha256:03ed2a641e412e42cc35c244508cf186015c217f0e4d496bf6d7078ebe837ae7", + "sha256:04b14e45d6a8e159c9767ae57ecb34563ad93440fc1b26516a89ceb5b33c1ad5", + "sha256:0cdde51bfcf6b6bd862ee9be324521ec619b20590787d1655d005c3fb175005f", + "sha256:0f48fc7dc82ee14aeaedb986e175a429d24129b7eada1b7e94a864e4f0644dde", + "sha256:107d327071061fd4f4a2587d14c389a27e4e5c93c7cba5f1f59987181903902f", + "sha256:1375bb8b88cb050a2d4e0da901001347a44302aeadb8ceb4b6e5aa373b8ea68f", + "sha256:14a9f1887591684fb59fdba8feef7123a0da2424b0652e1b58dd5b9a7bb1188c", + "sha256:16baa799ec09cc0dcb43a10680573269d407c159325972dd7114ee7649e56c66", + "sha256:1b811662ecf72eb2d08872731636aee6559cae21862c36f74703be727b45df90", + "sha256:1ccae21a076d3d5f471700f6d30eb486da1626c380b23c70ae32ab823e453337", + "sha256:2f2cf7a42d4b7654c9a67b9d091ec24374f7c58794858bff632a2039cb15984d", + "sha256:322549b880b2d746a7672bf6ff9ed3f895e9c9f108b714e7360292aa5c5d7cf4", + "sha256:32ab83016c24c5cf3db2943286b85b0a172dae08c58d0f53875235219b676409", + "sha256:3fe50f1cac369b02d34ad904dfe0771acc483f82a1b54c5e93632916ba847b37", + "sha256:4a780807e80479f281d47ee4af2eb2df3e4ccf4723484f77da0bb49d027e40a1", + "sha256:4a8eb7785bd23565b542b01fb39115a975fefb4a82f23d407503eee2c0106247", + "sha256:4b40b794775df10d7e3ea677108dd581bfec796235750c617d461874178a67f6", + "sha256:5bee3970617b3d74759b2d2df2f6a327d372f9732f9ccbf03fa591b5f7581e39", + "sha256:60a3307a84ec60578accd35d7f0c71a3a971430ed7eca6567399d2b50ef37b8c", + "sha256:6625e52b6f346a283c3d563d1fd8bae8956daafc64bb5bbd2b8f8a07608e3994", + "sha256:66a5aae8233d766a877c5ef293ec5ab9520929c2578fd2069308a98b7374ea8c", + "sha256:68fb816a5dd901c6aff352ce49e2a0ffadacdf9b6fae282a69e7a16a02dad5fb", + "sha256:6b588b5cf51dc0fd1c9e19f622457cc74b7d26fe295432e434525f1c0fae02bc", + "sha256:6c4d7165a4e8f41eca6b990c12ee7f44fef3932fac48ca32cecb3a1b2223c21f", + "sha256:6d2e262e5e8da6fa56e774fb8e2643417351427604c2b177f8e8c5f75fc928ca", + "sha256:6d9c88b787638a451f41f97446a1c9fd416e669b4d9717ae4615bd29de1ac135", + "sha256:755c56beeacac6a24c8e1074f89f34f4373abce8b662470d3aa719ae304931f3", + "sha256:7e40d3f8eb472c1509b12ac2a7e24158ec352fc8567b77ab02c0db053927e339", + "sha256:812eaf4939ef2284d29653bcfee9665f11f013724f07258928f849a2306ea9f9", + "sha256:84df004223fd0550d0ea7a37882e5c889f3c6d45535c639ce9802293b39cd5c9", + "sha256:859f0add98707b182b4867359e12bde806b82483fb12a9ae868a77880fc3b7af", + "sha256:87c4b38288f71acd2106f5d94f575bc2136ea2887fdb5dfe18003c881fa6b370", + "sha256:89fc12c6371bf963809abc46cced4a01ca4f99cba17be5e7d416ed7ef1245d19", + "sha256:9564ac7eb1652c3701ac691ca72934dd3009997c81266807aef924012df2f4b3", + "sha256:9754a5c265f991317de2bac0c70a746efc2b695cf4d49f5d2cddeac36544fb44", + "sha256:a565f48c4aae72d1d3d3f8e8fb7218f5609c964e9c6f68604608e5958b9c60c3", + "sha256:a636160680c6e526b84f85d304e2f0bb4e94f8284dd765a1911de9a40450b10a", + "sha256:a839e25f07e428a87d17d857d9935dd743130e77ff46524abb992b962eb2076c", + "sha256:ae9702c099546e72000d76758b5efec2dd937ba5d746ec8d0563d2fca0f9bc2e", + "sha256:b62046592b44263fa7570f1117d372ae3f310222af1fc1407416f037fb3af21b", + "sha256:b7f7421841f8db443855d2854e25914a79a1ff48ae92f70d0a5c2f8907ab98c9", + "sha256:ba7ca81b6d60a9f7a0b4b4e175dcc38e8fef4992673d9d6e6879fd6de00dd9b8", + "sha256:bb32ca14b4d04e172c541c69eec5f385f9a075b38fb22d765d8b0ce3af3a0c22", + "sha256:c0ff1c1b4d13e2240821ef23c1efb1f009207cb3f56e16986f713c2b0e7cd37f", + "sha256:c669b440ce46ae3abe9b2d44a913b5fd86bb19eb14a8701e88e3918902ecd345", + "sha256:c67734cff78383a1f23ceba3b3239c7deefc62ac2b05fa6a47bcd565771e5880", + "sha256:c6809ebcbf6c1049002b9ac09c127ae43929042ec1f1dbd8bb1615f7cd9f70a0", + "sha256:cd601187476c6bed26a0398353212684c427e10a903aeafa6da40c63309d438b", + "sha256:ebfa374067af240d079ef97b8064478f3bf71038b78b017eb6ec93ede1b6bcec", + "sha256:fbb17c0d0822684b7d6c09915677a32319f16ff1115df5ec05bdcaaee40b35f3", + "sha256:fff1f3a586246110f34dc762098b5afd2de88de507559e63553d7da643053786" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==5.3.1" + "version": "==5.4" }, "coveralls": { "hashes": [ @@ -1027,11 +1485,11 @@ }, "execnet": { "hashes": [ - "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50", - "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547" + "sha256:7a13113028b1e1cc4c6492b28098b3c6576c9dccc7973bfe47b342afadafb2ac", + "sha256:b73c5565e517f24b62dea8a5ceac178c661c4309d3aa0c3e420856c072c411b4" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.7.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.8.0" }, "factory-boy": { "hashes": [ @@ -1043,11 +1501,11 @@ }, "faker": { "hashes": [ - "sha256:47ac7d62d5bad8c16422a91f121430ab7656d40ca8fea9c84bcdbdf92e739b03", - "sha256:6bc44606d44f711e1d89ad9a5b42394cc6f7eedaffc765ddb5b2d22084c15733" + "sha256:0783729c61501d52efea2967aff6e6fcb8370f0f6b5a558f2a81233642ae529a", + "sha256:6b2995ffff6c2b02bc5daad96f8c24c021e5bd491d9d53d31bcbd66f348181d4" ], "markers": "python_version >= '3.6'", - "version": "==5.5.0" + "version": "==5.8.0" }, "filelock": { "hashes": [ @@ -1084,11 +1542,11 @@ }, "importlib-resources": { "hashes": [ - "sha256:4743f090ed8946e713745ec0e660249ef9fb0b9843eacc5b5ff931d2fd5aa67f", - "sha256:ea17df80a0ff04b5dbd3d96dbeab1842acfd1c6c902eaeb8c8858abf2720161e" + "sha256:885b8eae589179f661c909d699a546cf10d83692553e34dca1bf5eb06f7f6217", + "sha256:bfdad047bce441405a49cf8eb48ddce5e56c696e185f59147a8b79e75e9e6380" ], "markers": "python_version < '3.7'", - "version": "==5.0.0" + "version": "==5.1.0" }, "iniconfig": { "hashes": [ @@ -1199,19 +1657,19 @@ }, "pytest": { "hashes": [ - "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8", - "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306" + "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9", + "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839" ], "index": "pypi", - "version": "==6.2.1" + "version": "==6.2.2" }, "pytest-cov": { "hashes": [ - "sha256:45ec2d5182f89a81fc3eb29e3d1ed3113b9e9a873bcddb2a71faaab066110191", - "sha256:47bd0ce14056fdd79f93e1713f88fad7bdcc583dcd7783da86ef2f085a0bb88e" + "sha256:359952d9d39b9f822d9d29324483e7ba04a3a17dd7d05aa6beb7ea01e359e5f7", + "sha256:bdb9fdb0b85a7cc825269a4c56b48ccaa5c7e365054b6038772c32ddcdc969da" ], "index": "pypi", - "version": "==2.10.1" + "version": "==2.11.1" }, "pytest-django": { "hashes": [ @@ -1287,10 +1745,10 @@ }, "snowballstemmer": { "hashes": [ - "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", - "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" + "sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2", + "sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914" ], - "version": "==2.0.0" + "version": "==2.1.0" }, "sphinx": { "hashes": [ @@ -1380,11 +1838,11 @@ }, "tox": { "hashes": [ - "sha256:5efda30ad73e662c3844ac51ce1381bf28f61063773e06996aa8b6277133a7c0", - "sha256:8cccede64802e78aa6c69f81051b25f0706639d1cbbb34d9366ce00c70ee054f" + "sha256:76df3db6eee929bb62bdbacca5bb6bc840669d98e86a015b7a57b7df0a6eaf8b", + "sha256:854e6e4a71c614b488f81cb88df3b92edcb1a9ec43d4102e6289e9669bbf7f18" ], "index": "pypi", - "version": "==3.21.0" + "version": "==3.21.3" }, "typing-extensions": { "hashes": [ @@ -1397,19 +1855,19 @@ }, "urllib3": { "hashes": [ - "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", - "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", + "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.2" + "version": "==1.26.3" }, "virtualenv": { "hashes": [ - "sha256:205a7577275dd0d9223c730dd498e21a8910600085c3dee97412b041fc4b853b", - "sha256:7992b8de87e544a4ab55afc2240bf8388c4e3b5765d03784dad384bfdf9097ee" + "sha256:219ee956e38b08e32d5639289aaa5bd190cfbe7dafcb8fa65407fca08e808f9c", + "sha256:227a8fed626f2f20a6cdb0870054989f82dd27b2560a911935ba905a2a5e0034" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.3.0" + "version": "==20.4.0" }, "zipp": { "hashes": [ diff --git a/ansible/tasks/main.yml b/ansible/tasks/main.yml index dba409549..db8edcc75 100644 --- a/ansible/tasks/main.yml +++ b/ansible/tasks/main.yml @@ -498,7 +498,7 @@ path: "{{ paperlessng_directory }}/scripts/paperless-webserver.service" section: "Service" option: "ExecStart" - value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.wsgi -w 2 -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}" + value: "{{ paperlessng_virtualenv }}/bin/gunicorn paperless.asgi:application -w 2 -k uvicorn.workers.UvicornWorker -b {{ paperlessng_listen_address }}:{{ paperlessng_listen_port }}" - name: copy systemd services copy: diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 9b97b6825..fca66c83c 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -8,7 +8,7 @@ loglevel=info ; log level; default info; others: debug,warn,trace user=root [program:gunicorn] -command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.wsgi +command=gunicorn -c /usr/src/paperless/gunicorn.conf.py paperless.asgi:application user=paperless stdout_logfile=/dev/stdout diff --git a/docs/setup.rst b/docs/setup.rst index 268352d4f..eda03550e 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -254,7 +254,7 @@ writing. Windows is not and will never be supported. 1. Install dependencies. Paperless requires the following packages. - * ``python3`` 3.6, 3.7, 3.8 (3.9 is untested). + * ``python3`` 3.6, 3.7, 3.8, 3.9 * ``python3-pip`` * ``python3-dev`` diff --git a/gunicorn.conf.py b/gunicorn.conf.py index edfb362d9..36bde9cf7 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -1,7 +1,7 @@ bind = '0.0.0.0:8000' backlog = 2048 workers = 3 -worker_class = 'sync' +worker_class = 'uvicorn.workers.UvicornWorker' worker_connections = 1000 timeout = 20 keepalive = 2 diff --git a/requirements.txt b/requirements.txt index 3b7a158c1..c3f32002b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,16 +7,26 @@ -i https://pypi.python.org/simple --extra-index-url https://www.piwheels.org/simple +aioredis==1.3.1 arrow==0.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' asgiref==3.3.1; python_version >= '3.5' +async-timeout==3.0.1; python_full_version >= '3.5.3' +attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +autobahn==20.12.3; python_version >= '3.6' +automat==20.2.0 blessed==1.17.12 certifi==2020.12.5 cffi==1.14.4 +channels-redis==3.2.0 +channels==3.0.3 chardet==4.0.0; python_version >= '3.1' +click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' coloredlogs==15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +constantly==15.1.0 cryptography==3.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +daphne==3.0.1; python_version >= '3.6' dateparser==0.7.6 -django-cors-headers==3.6.0 +django-cors-headers==3.7.0 django-extensions==3.1.0 django-filter==2.4.0 django-picklefield==3.0.1; python_version >= '3' @@ -26,16 +36,22 @@ djangorestframework==3.12.2 filelock==3.0.12 fuzzywuzzy==0.18.0 gunicorn==20.0.4 +h11==0.12.0; python_version >= '3.6' +hiredis==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +httptools==0.1.1 humanfriendly==9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +hyperlink==21.0.0 idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -imap-tools==0.34.0 +imap-tools==0.37.0 img2pdf==0.4.0 importlib-metadata==3.4.0; python_version < '3.8' +incremental==17.5.0 inotify-simple==1.3.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' inotifyrecursive==0.3.5 joblib==1.0.0; python_version >= '3.6' langdetect==1.0.8 lxml==4.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +msgpack==1.0.2 numpy==1.19.5; python_version >= '3.6' ocrmypdf==11.4.5 pathvalidate==2.3.2 @@ -45,30 +61,43 @@ pikepdf==2.2.5 pillow==8.1.0 pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' psycopg2-binary==2.8.6 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +pyhamcrest==2.0.2; python_version >= '3.5' +pyopenssl==20.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' python-dateutil==2.8.1 python-dotenv==0.15.0 python-gnupg==0.4.6 -python-levenshtein==0.12.0 +python-levenshtein==0.12.1 python-magic==0.4.18 pytz==2020.5 +pyyaml==5.4.1 redis==3.5.3 regex==2020.11.13 reportlab==3.5.59 requests==2.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' scikit-learn==0.24.0 -scipy==1.5.4; python_version >= '3.6' +scipy==1.5.4 +service-identity==18.1.0 six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' sortedcontainers==2.3.0 sqlparse==0.4.1; python_version >= '3.5' threadpoolctl==2.1.0; python_version >= '3.5' tika==1.24 tqdm==4.56.0 +twisted[tls]==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +txaio==20.12.1; python_version >= '3.6' typing-extensions==3.7.4.3; python_version < '3.8' tzlocal==2.1 -urllib3==1.26.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' +urllib3==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' +uvicorn[standard]==0.13.3 +uvloop==0.14.0 watchdog==1.0.2 +watchgod==0.6 wcwidth==0.2.5 +websockets==8.1 whitenoise==5.2.0 whoosh==2.7.4 zipp==3.4.0; python_version >= '3.6' +zope.interface==5.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' diff --git a/scripts/paperless-webserver.service b/scripts/paperless-webserver.service index 00bc4fa37..7b7fa329a 100644 --- a/scripts/paperless-webserver.service +++ b/scripts/paperless-webserver.service @@ -8,7 +8,7 @@ Requires=redis.service User=paperless Group=paperless WorkingDirectory=/opt/paperless/src -ExecStart=/opt/paperless/.local/bin/gunicorn -c /opt/paperless/gunicorn.conf.py paperless.wsgi +ExecStart=/opt/paperless/.local/bin/gunicorn -c /opt/paperless/gunicorn.conf.py paperless..asgi:application [Install] WantedBy=multi-user.target diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 0ad552100..ff928863d 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -2,25 +2,67 @@ + + Document added + + src/app/app.component.ts + 51 + + + + Document was added to paperless. + + src/app/app.component.ts + 51 + + + + Open document + + src/app/app.component.ts + 51 + + + + Could not add : + + src/app/app.component.ts + 59 + + + + New document detected + + src/app/app.component.ts + 65 + + + + Document is being processed by paperless. + + src/app/app.component.ts + 65 + + Documents src/app/components/document-list/document-list.component.ts - 43 + 49 View "" saved successfully. src/app/components/document-list/document-list.component.ts - 94 + 109 View "" created successfully. src/app/components/document-list/document-list.component.ts - 115 + 130 @@ -482,35 +524,35 @@ Saved view "" deleted. src/app/components/manage/settings/settings.component.ts - 63 + 67 Settings saved successfully. src/app/components/manage/settings/settings.component.ts - 79 + 87 Use system language src/app/components/manage/settings/settings.component.ts - 83 + 91 Use date format of display language src/app/components/manage/settings/settings.component.ts - 87 + 95 Error while storing settings on server: src/app/components/manage/settings/settings.component.ts - 103 + 111 @@ -531,7 +573,7 @@ Saved views src/app/components/manage/settings/settings.component.html - 114 + 128 @@ -639,60 +681,109 @@ 98 + + Notifications + + src/app/components/manage/settings/settings.component.html + 102 + + + + Consumer status + + src/app/components/manage/settings/settings.component.html + 106 + + + + Show notifications when new documents are detected + + src/app/components/manage/settings/settings.component.html + 109 + + + + Show notifications when document consumption completes successfully + + src/app/components/manage/settings/settings.component.html + 110 + + + + Show notifications when document consumption fails + + src/app/components/manage/settings/settings.component.html + 111 + + + + Suppress notifications on dashboard + + src/app/components/manage/settings/settings.component.html + 112 + + + + This will suppress all consumer related status messages on the dashboard. + + src/app/components/manage/settings/settings.component.html + 112 + + Bulk editing src/app/components/manage/settings/settings.component.html - 102 + 116 Show confirmation dialogs src/app/components/manage/settings/settings.component.html - 106 + 120 Deleting documents will always ask for confirmation. src/app/components/manage/settings/settings.component.html - 106 + 120 Apply on close src/app/components/manage/settings/settings.component.html - 107 + 121 Appears on src/app/components/manage/settings/settings.component.html - 126 + 140 Show on dashboard src/app/components/manage/settings/settings.component.html - 129 + 143 Show in sidebar src/app/components/manage/settings/settings.component.html - 133 + 147 No saved views defined. src/app/components/manage/settings/settings.component.html - 143 + 157 @@ -1330,25 +1421,46 @@ 4 - - The document has been uploaded and will be processed by the consumer shortly. + + Processing: src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts - 63 + 32 - - There was an error while uploading the document: + + Failed: src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts - 71 + 35 - - An error has occurred while uploading the document. Sorry! + + Added: src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts - 75 + 38 + + + + Connecting... + + src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts + 118 + + + + Uploading... + + src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts + 123 + + + + Waiting for consumer... + + src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts + 126 @@ -1362,21 +1474,35 @@ Drop documents here or src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html - 5 + 13 Browse files src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html - 5 + 13 - - {VAR_PLURAL, plural, =1 {Uploading file...} =other {Uploading files...}} + + Dismiss completed src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html - 13 + 4 + + + + more hidden + + src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html + 24 + + + + Open document + + src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html + 41 @@ -1474,42 +1600,133 @@ English (US) src/app/services/settings.service.ts - 74 + 82 German src/app/services/settings.service.ts - 75 + 83 Dutch src/app/services/settings.service.ts - 76 + 84 French src/app/services/settings.service.ts - 77 + 85 + + + + Document already exists. + + src/app/services/consumer-status.service.ts + 14 + + + + File not found. + + src/app/services/consumer-status.service.ts + 15 + + + + Pre-consume script does not exist. + + src/app/services/consumer-status.service.ts + 16 + + + + Error while executing pre-consume script. + + src/app/services/consumer-status.service.ts + 17 + + + + Post-consume script does not exist. + + src/app/services/consumer-status.service.ts + 18 + + + + Error while executing post-consume script. + + src/app/services/consumer-status.service.ts + 19 + + + + Received new file. + + src/app/services/consumer-status.service.ts + 20 + + + + File type not supported. + + src/app/services/consumer-status.service.ts + 21 + + + + Processing document... + + src/app/services/consumer-status.service.ts + 22 + + + + Generating thumbnail... + + src/app/services/consumer-status.service.ts + 23 + + + + Retrieving date from document... + + src/app/services/consumer-status.service.ts + 24 + + + + Saving document... + + src/app/services/consumer-status.service.ts + 25 + + + + Finished. + + src/app/services/consumer-status.service.ts + 26 Error src/app/services/toast.service.ts - 31 + 35 Information src/app/services/toast.service.ts - 35 + 39 diff --git a/src-ui/src/app/app.component.ts b/src-ui/src/app/app.component.ts index 73c5fc861..836a6f66a 100644 --- a/src-ui/src/app/app.component.ts +++ b/src-ui/src/app/app.component.ts @@ -1,17 +1,70 @@ -import { Component } from '@angular/core'; -import { SettingsService } from './services/settings.service'; +import { SettingsService, SETTINGS_KEYS } from './services/settings.service'; +import { Component, OnDestroy, OnInit } from '@angular/core'; +import { Router } from '@angular/router'; +import { Subscription } from 'rxjs'; +import { ConsumerStatusService } from './services/consumer-status.service'; +import { ToastService } from './services/toast.service'; @Component({ selector: 'app-root', templateUrl: './app.component.html', styleUrls: ['./app.component.scss'] }) -export class AppComponent { +export class AppComponent implements OnInit, OnDestroy { - constructor (private settings: SettingsService) { + newDocumentSubscription: Subscription; + successSubscription: Subscription; + failedSubscription: Subscription; + + constructor (private settings: SettingsService, private consumerStatusService: ConsumerStatusService, private toastService: ToastService, private router: Router) { let anyWindow = (window as any) anyWindow.pdfWorkerSrc = '/assets/js/pdf.worker.min.js'; this.settings.updateDarkModeSettings() } + ngOnDestroy(): void { + this.consumerStatusService.disconnect() + if (this.successSubscription) { + this.successSubscription.unsubscribe() + } + if (this.failedSubscription) { + this.failedSubscription.unsubscribe() + } + if (this.newDocumentSubscription) { + this.newDocumentSubscription.unsubscribe() + } + } + + private showNotification(key) { + if (this.router.url == '/dashboard' && this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD)) { + return false + } + return this.settings.get(key) + } + + ngOnInit(): void { + this.consumerStatusService.connect() + + + this.successSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => { + if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)) { + this.toastService.show({title: $localize`Document added`, delay: 10000, content: $localize`Document ${status.filename} was added to paperless.`, actionName: $localize`Open document`, action: () => { + this.router.navigate(['documents', status.documentId]) + }}) + } + }) + + this.failedSubscription = this.consumerStatusService.onDocumentConsumptionFailed().subscribe(status => { + if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)) { + this.toastService.showError($localize`Could not add ${status.filename}\: ${status.message}`) + } + }) + + this.newDocumentSubscription = this.consumerStatusService.onDocumentDetected().subscribe(status => { + if (this.showNotification(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT)) { + this.toastService.show({title: $localize`New document detected`, delay: 5000, content: $localize`Document ${status.filename} is being processed by paperless.`}) + } + }) + } + } diff --git a/src-ui/src/app/components/common/toasts/toasts.component.html b/src-ui/src/app/components/common/toasts/toasts.component.html index 04aa15a67..9d4ae6bb6 100644 --- a/src-ui/src/app/components/common/toasts/toasts.component.html +++ b/src-ui/src/app/components/common/toasts/toasts.component.html @@ -3,5 +3,6 @@ [header]="toast.title" [autohide]="true" [delay]="toast.delay" [class]="toast.classname" (hide)="toastService.closeToast(toast)"> - {{toast.content}} - \ No newline at end of file +

{{toast.content}}

+

+ diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html index c10b8ea04..ec2bbd85c 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.html +++ b/src-ui/src/app/components/dashboard/dashboard.component.html @@ -19,6 +19,6 @@ - + diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html index d907c59d6..d05d8a667 100644 --- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html +++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html @@ -18,4 +18,4 @@ - \ No newline at end of file + diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts index 7405c2848..04e7e56bd 100644 --- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts +++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts @@ -1,8 +1,10 @@ -import { Component, Input, OnInit } from '@angular/core'; +import { Component, Input, OnDestroy, OnInit } from '@angular/core'; import { Router } from '@angular/router'; +import { Subscription } from 'rxjs'; import { PaperlessDocument } from 'src/app/data/paperless-document'; import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'; import { DocumentListViewService } from 'src/app/services/document-list-view.service'; +import { ConsumerStatusService } from 'src/app/services/consumer-status.service'; import { DocumentService } from 'src/app/services/rest/document.service'; @Component({ @@ -15,14 +17,28 @@ export class SavedViewWidgetComponent implements OnInit { constructor( private documentService: DocumentService, private router: Router, - private list: DocumentListViewService) { } + private list: DocumentListViewService, + private consumerStatusService: ConsumerStatusService) { } @Input() savedView: PaperlessSavedView documents: PaperlessDocument[] = [] + subscription: Subscription + ngOnInit(): void { + this.reload() + this.subscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(status => { + this.reload() + }) + } + + ngOnDestroy(): void { + this.subscription.unsubscribe() + } + + reload() { this.documentService.listFiltered(1,10,this.savedView.sort_field, this.savedView.sort_reverse, this.savedView.filter_rules).subscribe(result => { this.documents = result.results }) diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html index 0ad0005ea..6ab996675 100644 --- a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html +++ b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html @@ -3,4 +3,4 @@

Documents in inbox: {{statistics.documents_inbox}}

Total documents: {{statistics.documents_total}}

- \ No newline at end of file + diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts index 73eee698c..c3f45d63c 100644 --- a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts +++ b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts @@ -23,7 +23,7 @@ export class StatisticsWidgetComponent implements OnInit { getStatistics(): Observable { return this.http.get(`${environment.apiBaseUrl}statistics/`) } - + ngOnInit(): void { this.getStatistics().subscribe(statistics => { this.statistics = statistics diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html index aa317bd52..3e2908b84 100644 --- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html +++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html @@ -1,18 +1,48 @@ - +
-
-
-

{uploadStatus.length, plural, =1 {Uploading file...} =other {Uploading {{uploadStatus.length}} files...}}

- - +

{{getStatusSummary()}}

+
+ +
+
+

{{getStatusHidden().length}} more hidden

+
+
+ +
+
- \ No newline at end of file + + + + +
{{status.filename}}
+

{{status.message}}

+ +
+ +
+ + diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss index e69de29bb..b37606ff3 100644 --- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss +++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.scss @@ -0,0 +1,35 @@ +@import "/src/theme"; + +form { + position: relative; +} + +.alert-heading { + font-size: 80%; + font-weight: bold; +} + +.alerts-hidden { + .btn { + line-height: 1; + } +} + +.btn-open { + line-height: 1; + + svg { + margin-top: -1px; + } +} + +::ng-deep .progress { + position: absolute; + top: 0; + right: 0; + bottom: 0; + left: 0; + height: auto; + mix-blend-mode: soft-light; + pointer-events: none; +} diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts index 2b14a571a..5ac68d42a 100644 --- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts +++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.ts @@ -1,14 +1,10 @@ import { HttpEventType } from '@angular/common/http'; import { Component, OnInit } from '@angular/core'; import { FileSystemFileEntry, NgxFileDropEntry } from 'ngx-file-drop'; +import { ConsumerStatusService, FileStatus, FileStatusPhase } from 'src/app/services/consumer-status.service'; import { DocumentService } from 'src/app/services/rest/document.service'; -import { ToastService } from 'src/app/services/toast.service'; - -interface UploadStatus { - loaded: number - total: number -} +const MAX_ALERTS = 5 @Component({ selector: 'app-upload-file-widget', @@ -16,8 +12,89 @@ interface UploadStatus { styleUrls: ['./upload-file-widget.component.scss'] }) export class UploadFileWidgetComponent implements OnInit { + alertsExpanded = false - constructor(private documentService: DocumentService, private toastService: ToastService) { } + constructor( + private documentService: DocumentService, + private consumerStatusService: ConsumerStatusService + ) { } + + getStatus() { + return this.consumerStatusService.getConsumerStatus().slice(0, MAX_ALERTS) + } + + getStatusSummary() { + let strings = [] + let countUploadingAndProcessing = this.consumerStatusService.getConsumerStatusNotCompleted().length + let countFailed = this.getStatusFailed().length + let countSuccess = this.getStatusSuccess().length + if (countUploadingAndProcessing > 0) { + strings.push($localize`Processing: ${countUploadingAndProcessing}`) + } + if (countFailed > 0) { + strings.push($localize`Failed: ${countFailed}`) + } + if (countSuccess > 0) { + strings.push($localize`Added: ${countSuccess}`) + } + return strings.join($localize`:this string is used to separate processing, failed and added on the file upload widget:, `) + } + + getStatusHidden() { + if (this.consumerStatusService.getConsumerStatus().length < MAX_ALERTS) return [] + else return this.consumerStatusService.getConsumerStatus().slice(MAX_ALERTS) + } + + getStatusUploading() { + return this.consumerStatusService.getConsumerStatus(FileStatusPhase.UPLOADING) + } + + getStatusFailed() { + return this.consumerStatusService.getConsumerStatus(FileStatusPhase.FAILED) + } + + getStatusSuccess() { + return this.consumerStatusService.getConsumerStatus(FileStatusPhase.SUCCESS) + } + + getStatusCompleted() { + return this.consumerStatusService.getConsumerStatusCompleted() + } + getTotalUploadProgress() { + let current = 0 + let max = 0 + + this.getStatusUploading().forEach(status => { + current += status.currentPhaseProgress + max += status.currentPhaseMaxProgress + }) + + return current / Math.max(max, 1) + } + + isFinished(status: FileStatus) { + return status.phase == FileStatusPhase.FAILED || status.phase == FileStatusPhase.SUCCESS + } + + getStatusColor(status: FileStatus) { + switch (status.phase) { + case FileStatusPhase.PROCESSING: + case FileStatusPhase.UPLOADING: + return "primary" + case FileStatusPhase.FAILED: + return "danger" + case FileStatusPhase.SUCCESS: + return "success" + } + } + + dismiss(status: FileStatus) { + this.consumerStatusService.dismiss(status) + } + + dismissAll() { + this.consumerStatusService.dismissAll() + } ngOnInit(): void { } @@ -28,54 +105,39 @@ export class UploadFileWidgetComponent implements OnInit { public fileLeave(event){ } - uploadStatus: UploadStatus[] = [] - completedFiles = 0 - - uploadVisible = false - - get loadedSum() { - return this.uploadStatus.map(s => s.loaded).reduce((a,b) => a+b, this.completedFiles > 0 ? 1 : 0) - } - - get totalSum() { - return this.uploadStatus.map(s => s.total).reduce((a,b) => a+b, 1) - } - public dropped(files: NgxFileDropEntry[]) { for (const droppedFile of files) { if (droppedFile.fileEntry.isFile) { - let uploadStatusObject: UploadStatus = {loaded: 0, total: 1} - this.uploadStatus.push(uploadStatusObject) - this.uploadVisible = true const fileEntry = droppedFile.fileEntry as FileSystemFileEntry; fileEntry.file((file: File) => { let formData = new FormData() formData.append('document', file, file.name) + let status = this.consumerStatusService.newFileUpload(file.name) + + status.message = $localize`Connecting...` this.documentService.uploadDocument(formData).subscribe(event => { if (event.type == HttpEventType.UploadProgress) { - uploadStatusObject.loaded = event.loaded - uploadStatusObject.total = event.total + status.updateProgress(FileStatusPhase.UPLOADING, event.loaded, event.total) + status.message = $localize`Uploading...` } else if (event.type == HttpEventType.Response) { - this.uploadStatus.splice(this.uploadStatus.indexOf(uploadStatusObject), 1) - this.completedFiles += 1 - this.toastService.showInfo($localize`The document has been uploaded and will be processed by the consumer shortly.`) + status.taskId = event.body["task_id"] + status.message = $localize`Waiting for consumer...` } - + }, error => { - this.uploadStatus.splice(this.uploadStatus.indexOf(uploadStatusObject), 1) - this.completedFiles += 1 switch (error.status) { case 400: { - this.toastService.showInfo($localize`There was an error while uploading the document: ${error.error.document}`) + this.consumerStatusService.fail(status, error.error.document) break; } default: { - this.toastService.showInfo($localize`An error has occurred while uploading the document. Sorry!`) + this.consumerStatusService.fail(status, `${error.status} ${error.statusText}`) break; } } + }) }); } diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index 509c0a735..0d1562c24 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -1,9 +1,11 @@ -import { AfterViewInit, Component, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core'; +import { AfterViewInit, Component, OnDestroy, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core'; import { ActivatedRoute, Router } from '@angular/router'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; +import { Subscription } from 'rxjs'; import { PaperlessDocument } from 'src/app/data/paperless-document'; import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'; import { SortableDirective, SortEvent } from 'src/app/directives/sortable.directive'; +import { ConsumerStatusService } from 'src/app/services/consumer-status.service'; import { DocumentListViewService } from 'src/app/services/document-list-view.service'; import { DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service'; import { SavedViewService } from 'src/app/services/rest/saved-view.service'; @@ -16,7 +18,7 @@ import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-vi templateUrl: './document-list.component.html', styleUrls: ['./document-list.component.scss'] }) -export class DocumentListComponent implements OnInit { +export class DocumentListComponent implements OnInit, OnDestroy { constructor( public list: DocumentListViewService, @@ -24,7 +26,9 @@ export class DocumentListComponent implements OnInit { public route: ActivatedRoute, private router: Router, private toastService: ToastService, - private modalService: NgbModal) { } + private modalService: NgbModal, + private consumerStatusService: ConsumerStatusService + ) { } @ViewChild("filterEditor") private filterEditor: FilterEditorComponent @@ -35,6 +39,8 @@ export class DocumentListComponent implements OnInit { filterRulesModified: boolean = false + private consumptionFinishedSubscription: Subscription + get isFiltered() { return this.list.filterRules?.length > 0 } @@ -63,6 +69,9 @@ export class DocumentListComponent implements OnInit { if (localStorage.getItem('document-list:displayMode') != null) { this.displayMode = localStorage.getItem('document-list:displayMode') } + this.consumptionFinishedSubscription = this.consumerStatusService.onDocumentConsumptionFinished().subscribe(() => { + this.list.reload() + }) this.route.paramMap.subscribe(params => { this.list.clear() if (params.has('id')) { @@ -83,6 +92,12 @@ export class DocumentListComponent implements OnInit { }) } + ngOnDestroy() { + if (this.consumptionFinishedSubscription) { + this.consumptionFinishedSubscription.unsubscribe() + } + } + loadViewConfig(view: PaperlessSavedView) { this.list.load(view) this.list.reload() diff --git a/src-ui/src/app/components/login/login.component.ts b/src-ui/src/app/components/login/login.component.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/manage/settings/settings.component.html b/src-ui/src/app/components/manage/settings/settings.component.html index 4ed5ad1ae..910867ace 100644 --- a/src-ui/src/app/components/manage/settings/settings.component.html +++ b/src-ui/src/app/components/manage/settings/settings.component.html @@ -99,6 +99,20 @@
+

Notifications

+ +
+
+ Consumer status +
+
+ + + + +
+
+

Bulk editing

diff --git a/src-ui/src/app/components/manage/settings/settings.component.ts b/src-ui/src/app/components/manage/settings/settings.component.ts index 47f714c21..47c7b8d7b 100644 --- a/src-ui/src/app/components/manage/settings/settings.component.ts +++ b/src-ui/src/app/components/manage/settings/settings.component.ts @@ -26,6 +26,10 @@ export class SettingsComponent implements OnInit { 'displayLanguage': new FormControl(this.settings.getLanguage()), 'dateLocale': new FormControl(this.settings.get(SETTINGS_KEYS.DATE_LOCALE)), 'dateFormat': new FormControl(this.settings.get(SETTINGS_KEYS.DATE_FORMAT)), + 'notificationsConsumerNewDocument': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT)), + 'notificationsConsumerSuccess': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS)), + 'notificationsConsumerFailed': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED)), + 'notificationsConsumerSuppressOnDashboard': new FormControl(this.settings.get(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD)), }) savedViews: PaperlessSavedView[] @@ -73,6 +77,10 @@ export class SettingsComponent implements OnInit { this.settings.set(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER, this.settingsForm.value.useNativePdfViewer) this.settings.set(SETTINGS_KEYS.DATE_LOCALE, this.settingsForm.value.dateLocale) this.settings.set(SETTINGS_KEYS.DATE_FORMAT, this.settingsForm.value.dateFormat) + this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT, this.settingsForm.value.notificationsConsumerNewDocument) + this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS, this.settingsForm.value.notificationsConsumerSuccess) + this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED, this.settingsForm.value.notificationsConsumerFailed) + this.settings.set(SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD, this.settingsForm.value.notificationsConsumerSuppressOnDashboard) this.settings.setLanguage(this.settingsForm.value.displayLanguage) this.documentListViewService.updatePageSize() this.settings.updateDarkModeSettings() diff --git a/src-ui/src/app/data/websocket-consumer-status-message.ts b/src-ui/src/app/data/websocket-consumer-status-message.ts new file mode 100644 index 000000000..49117b101 --- /dev/null +++ b/src-ui/src/app/data/websocket-consumer-status-message.ts @@ -0,0 +1,11 @@ +export interface WebsocketConsumerStatusMessage { + + filename?: string + task_id?: string + current_progress?: number + max_progress?: number + status?: string + message?: string + document_id: number + +} \ No newline at end of file diff --git a/src-ui/src/app/services/auth.interceptor.ts b/src-ui/src/app/services/auth.interceptor.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/services/consumer-status.service.spec.ts b/src-ui/src/app/services/consumer-status.service.spec.ts new file mode 100644 index 000000000..d19f455e2 --- /dev/null +++ b/src-ui/src/app/services/consumer-status.service.spec.ts @@ -0,0 +1,16 @@ +import { TestBed } from '@angular/core/testing'; + +import { ConsumerStatusService } from './consumer-status.service'; + +describe('ConsumerStatusService', () => { + let service: ConsumerStatusService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(ConsumerStatusService); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/src-ui/src/app/services/consumer-status.service.ts b/src-ui/src/app/services/consumer-status.service.ts new file mode 100644 index 000000000..026c3c64f --- /dev/null +++ b/src-ui/src/app/services/consumer-status.service.ts @@ -0,0 +1,193 @@ +import { Injectable } from '@angular/core'; +import { Subject } from 'rxjs'; +import { WebsocketConsumerStatusMessage } from '../data/websocket-consumer-status-message'; + +export enum FileStatusPhase { + STARTED = 0, + UPLOADING = 1, + PROCESSING = 2, + SUCCESS = 3, + FAILED = 4 +} + +export const FILE_STATUS_MESSAGES = { + "document_already_exists": $localize`Document already exists.`, + "file_not_found": $localize`File not found.`, + "pre_consume_script_not_found": $localize`Pre-consume script does not exist.`, + "pre_consume_script_error": $localize`Error while executing pre-consume script.`, + "post_consume_script_not_found": $localize`Post-consume script does not exist.`, + "post_consume_script_error": $localize`Error while executing post-consume script.`, + "new_file": $localize`Received new file.`, + "unsupported_type": $localize`File type not supported.`, + "parsing_document": $localize`Processing document...`, + "generating_thumbnail": $localize`Generating thumbnail...`, + "parse_date": $localize`Retrieving date from document...`, + "save_document": $localize`Saving document...`, + "finished": $localize`Finished.` +} + +export class FileStatus { + + filename: string + + taskId: string + + phase: FileStatusPhase = FileStatusPhase.STARTED + + currentPhaseProgress: number + + currentPhaseMaxProgress: number + + message: string + + documentId: number + + getProgress(): number { + switch (this.phase) { + case FileStatusPhase.STARTED: + return 0.0 + case FileStatusPhase.UPLOADING: + return this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.2 + case FileStatusPhase.PROCESSING: + return (this.currentPhaseProgress / this.currentPhaseMaxProgress * 0.8) + 0.2 + case FileStatusPhase.SUCCESS: + case FileStatusPhase.FAILED: + return 1.0 + } + } + + updateProgress(status: FileStatusPhase, currentProgress?: number, maxProgress?: number) { + if (status >= this.phase) { + this.phase = status + if (currentProgress != null) { + this.currentPhaseProgress = currentProgress + } + if (maxProgress != null) { + this.currentPhaseMaxProgress = maxProgress + } + } + } + +} + +@Injectable({ + providedIn: 'root' +}) +export class ConsumerStatusService { + + constructor() { } + + private statusWebSocked: WebSocket + + private consumerStatus: FileStatus[] = [] + + private documentDetectedSubject = new Subject() + private documentConsumptionFinishedSubject = new Subject() + private documentConsumptionFailedSubject = new Subject() + + private get(taskId: string, filename?: string) { + let status = this.consumerStatus.find(e => e.taskId == taskId) || this.consumerStatus.find(e => e.filename == filename && e.taskId == null) + let created = false + if (!status) { + status = new FileStatus() + this.consumerStatus.push(status) + created = true + } + status.taskId = taskId + status.filename = filename + return {'status': status, 'created': created} + } + + newFileUpload(filename: string): FileStatus { + let status = new FileStatus() + status.filename = filename + this.consumerStatus.push(status) + return status + } + + getConsumerStatus(phase?: FileStatusPhase) { + if (phase != null) { + return this.consumerStatus.filter(s => s.phase == phase) + } else { + return this.consumerStatus + } + } + + getConsumerStatusNotCompleted() { + return this.consumerStatus.filter(s => s.phase < FileStatusPhase.SUCCESS) + } + + getConsumerStatusCompleted() { + return this.consumerStatus.filter(s => s.phase == FileStatusPhase.FAILED || s.phase == FileStatusPhase.SUCCESS) + } + + connect() { + this.disconnect() + this.statusWebSocked = new WebSocket("ws://localhost:8000/ws/status/"); + this.statusWebSocked.onmessage = (ev) => { + let statusMessage: WebsocketConsumerStatusMessage = JSON.parse(ev['data']) + + let statusMessageGet = this.get(statusMessage.task_id, statusMessage.filename) + let status = statusMessageGet.status + let created = statusMessageGet.created + + status.updateProgress(FileStatusPhase.PROCESSING, statusMessage.current_progress, statusMessage.max_progress) + if (statusMessage.message && statusMessage.message in FILE_STATUS_MESSAGES) { + status.message = FILE_STATUS_MESSAGES[statusMessage.message] + } else if (statusMessage.message) { + status.message = statusMessage.message + } + status.documentId = statusMessage.document_id + + if (created && statusMessage.status == 'STARTING') { + this.documentDetectedSubject.next(status) + } + if (statusMessage.status == "SUCCESS") { + status.phase = FileStatusPhase.SUCCESS + this.documentConsumptionFinishedSubject.next(status) + } + if (statusMessage.status == "FAILED") { + status.phase = FileStatusPhase.FAILED + this.documentConsumptionFailedSubject.next(status) + } + } + } + + fail(status: FileStatus, message: string) { + status.message = message + status.phase = FileStatusPhase.FAILED + this.documentConsumptionFailedSubject.next(status) + } + + disconnect() { + if (this.statusWebSocked) { + this.statusWebSocked.close() + this.statusWebSocked = null + } + } + + dismiss(status: FileStatus) { + let index = this.consumerStatus.findIndex(s => s.filename == status.filename) + + if (index > -1) { + this.consumerStatus.splice(index, 1) + } + } + + dismissAll() { + this.consumerStatus = this.consumerStatus.filter(status => status.phase < FileStatusPhase.SUCCESS) + } + + onDocumentConsumptionFinished() { + return this.documentConsumptionFinishedSubject + } + + onDocumentConsumptionFailed() { + return this.documentConsumptionFailedSubject + } + + onDocumentDetected() { + return this.documentDetectedSubject + } + +} diff --git a/src-ui/src/app/services/settings.service.ts b/src-ui/src/app/services/settings.service.ts index 041fb51ca..f3e437ada 100644 --- a/src-ui/src/app/services/settings.service.ts +++ b/src-ui/src/app/services/settings.service.ts @@ -23,7 +23,11 @@ export const SETTINGS_KEYS = { DARK_MODE_ENABLED: 'general-settings:dark-mode:enabled', USE_NATIVE_PDF_VIEWER: 'general-settings:document-details:native-pdf-viewer', DATE_LOCALE: 'general-settings:date-display:date-locale', - DATE_FORMAT: 'general-settings:date-display:date-format' + DATE_FORMAT: 'general-settings:date-display:date-format', + NOTIFICATIONS_CONSUMER_NEW_DOCUMENT: 'general-settings:notifications:consumer-new-documents', + NOTIFICATIONS_CONSUMER_SUCCESS: 'general-settings:notifications:consumer-success', + NOTIFICATIONS_CONSUMER_FAILED: 'general-settings:notifications:consumer-failed', + NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD: 'general-settings:notifications:consumer-suppress-on-dashboard', } const SETTINGS: PaperlessSettings[] = [ @@ -34,7 +38,11 @@ const SETTINGS: PaperlessSettings[] = [ {key: SETTINGS_KEYS.DARK_MODE_ENABLED, type: "boolean", default: false}, {key: SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER, type: "boolean", default: false}, {key: SETTINGS_KEYS.DATE_LOCALE, type: "string", default: ""}, - {key: SETTINGS_KEYS.DATE_FORMAT, type: "string", default: "mediumDate"} + {key: SETTINGS_KEYS.DATE_FORMAT, type: "string", default: "mediumDate"}, + {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_NEW_DOCUMENT, type: "boolean", default: true}, + {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUCCESS, type: "boolean", default: true}, + {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_FAILED, type: "boolean", default: true}, + {key: SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD, type: "boolean", default: true}, ] @Injectable({ diff --git a/src-ui/src/app/services/toast.service.ts b/src-ui/src/app/services/toast.service.ts index 86d66eee6..fc522e2df 100644 --- a/src-ui/src/app/services/toast.service.ts +++ b/src-ui/src/app/services/toast.service.ts @@ -9,6 +9,10 @@ export interface Toast { delay: number + action?: any + + actionName?: string + } @Injectable({ diff --git a/src-ui/src/environments/environment.ts b/src-ui/src/environments/environment.ts index 29a8f3af6..29dcb7ba1 100644 --- a/src-ui/src/environments/environment.ts +++ b/src-ui/src/environments/environment.ts @@ -6,7 +6,8 @@ export const environment = { production: false, apiBaseUrl: "http://localhost:8000/api/", appTitle: "Paperless-ng", - version: "DEVELOPMENT" + version: "DEVELOPMENT", + wsBaseUrl: "ws://localhost:8000/ws/" }; /* diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 8cf4a93f6..34f575a05 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -111,3 +111,7 @@ body { font-size: 16px; } } + +.ngx-file-drop__drop-zone--over { + background-color: $primaryFaded !important; +} diff --git a/src-ui/src/theme_dark.scss b/src-ui/src/theme_dark.scss index 9a698143d..4e850f017 100644 --- a/src-ui/src/theme_dark.scss +++ b/src-ui/src/theme_dark.scss @@ -352,6 +352,20 @@ $border-color-dark-mode: #47494f; .bg-dark { background-color: $bg-light-dark-mode !important; } + + .ngx-file-drop__drop-zone--over { + background-color: darken($primary-dark-mode, 35%) !important; + } + + .alert-secondary { + background-color: $bg-light-dark-mode; + border-color: darken($bg-light-dark-mode, 10%); + color: $text-color-dark-mode-accent; + } + + .progress-bar.bg-primary { + background-color: darken($primary-dark-mode, 5%) !important; + } } body.color-scheme-dark { diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 5e76ad03a..f8f7576ef 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -1,9 +1,12 @@ import datetime import hashlib import os +import uuid from subprocess import Popen import magic +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer from django.conf import settings from django.db import transaction from django.db.models import Q @@ -27,8 +30,43 @@ class ConsumerError(Exception): pass +MESSAGE_DOCUMENT_ALREADY_EXISTS = "document_already_exists" +MESSAGE_FILE_NOT_FOUND = "file_not_found" +MESSAGE_PRE_CONSUME_SCRIPT_NOT_FOUND = "pre_consume_script_not_found" +MESSAGE_PRE_CONSUME_SCRIPT_ERROR = "pre_consume_script_error" +MESSAGE_POST_CONSUME_SCRIPT_NOT_FOUND = "post_consume_script_not_found" +MESSAGE_POST_CONSUME_SCRIPT_ERROR = "post_consume_script_error" +MESSAGE_NEW_FILE = "new_file" +MESSAGE_UNSUPPORTED_TYPE = "unsupported_type" +MESSAGE_PARSING_DOCUMENT = "parsing_document" +MESSAGE_GENERATING_THUMBNAIL = "generating_thumbnail" +MESSAGE_PARSE_DATE = "parse_date" +MESSAGE_SAVE_DOCUMENT = "save_document" +MESSAGE_FINISHED = "finished" + + class Consumer(LoggingMixin): + def _send_progress(self, current_progress, max_progress, status, + message=None, document_id=None): + payload = { + 'filename': os.path.basename(self.filename) if self.filename else None, # NOQA: E501 + 'task_id': self.task_id, + 'current_progress': current_progress, + 'max_progress': max_progress, + 'status': status, + 'message': message, + 'document_id': document_id + } + async_to_sync(self.channel_layer.group_send)("status_updates", + {'type': 'status_update', + 'data': payload}) + + def _fail(self, message, log_message=None): + self._send_progress(100, 100, 'FAILED', message) + self.log("error", log_message or message) + raise ConsumerError(f"{self.filename}: {log_message or message}") + def __init__(self): super().__init__() self.path = None @@ -37,15 +75,16 @@ class Consumer(LoggingMixin): self.override_correspondent_id = None self.override_tag_ids = None self.override_document_type_id = None + self.task_id = None + + self.channel_layer = get_channel_layer() def pre_check_file_exists(self): if not os.path.isfile(self.path): - self.log( - "error", - "Cannot consume {}: It is not a file.".format(self.path) + self._fail( + MESSAGE_FILE_NOT_FOUND, + f"Cannot consume {self.path}: File not found." ) - raise ConsumerError("Cannot consume {}: It is not a file".format( - self.path)) def pre_check_duplicate(self): with open(self.path, "rb") as f: @@ -53,12 +92,9 @@ class Consumer(LoggingMixin): if Document.objects.filter(Q(checksum=checksum) | Q(archive_checksum=checksum)).exists(): # NOQA: E501 if settings.CONSUMER_DELETE_DUPLICATES: os.unlink(self.path) - self.log( - "error", - "Not consuming {}: It is a duplicate.".format(self.filename) - ) - raise ConsumerError( - "Not consuming {}: It is a duplicate.".format(self.filename) + self._fail( + MESSAGE_DOCUMENT_ALREADY_EXISTS, + f"Not consuming {self.filename}: It is a duplicate." ) def pre_check_directories(self): @@ -72,14 +108,16 @@ class Consumer(LoggingMixin): return if not os.path.isfile(settings.PRE_CONSUME_SCRIPT): - raise ConsumerError( + self._fail( + MESSAGE_PRE_CONSUME_SCRIPT_NOT_FOUND, f"Configured pre-consume script " f"{settings.PRE_CONSUME_SCRIPT} does not exist.") try: Popen((settings.PRE_CONSUME_SCRIPT, self.path)).wait() except Exception as e: - raise ConsumerError( + self._fail( + MESSAGE_PRE_CONSUME_SCRIPT_ERROR, f"Error while executing pre-consume script: {e}" ) @@ -88,9 +126,11 @@ class Consumer(LoggingMixin): return if not os.path.isfile(settings.POST_CONSUME_SCRIPT): - raise ConsumerError( + self._fail( + MESSAGE_POST_CONSUME_SCRIPT_NOT_FOUND, f"Configured post-consume script " - f"{settings.POST_CONSUME_SCRIPT} does not exist.") + f"{settings.POST_CONSUME_SCRIPT} does not exist." + ) try: Popen(( @@ -106,8 +146,9 @@ class Consumer(LoggingMixin): "name", flat=True))) )).wait() except Exception as e: - raise ConsumerError( - f"Error while executing pre-consume script: {e}" + self._fail( + MESSAGE_POST_CONSUME_SCRIPT_ERROR, + f"Error while executing post-consume script: {e}" ) def try_consume_file(self, @@ -116,7 +157,8 @@ class Consumer(LoggingMixin): override_title=None, override_correspondent_id=None, override_document_type_id=None, - override_tag_ids=None): + override_tag_ids=None, + task_id=None): """ Return the document object if it was successfully created. """ @@ -127,6 +169,9 @@ class Consumer(LoggingMixin): self.override_correspondent_id = override_correspondent_id self.override_document_type_id = override_document_type_id self.override_tag_ids = override_tag_ids + self.task_id = task_id or str(uuid.uuid4()) + + self._send_progress(0, 100, 'STARTING', MESSAGE_NEW_FILE) # this is for grouping logging entries for this particular file # together. @@ -149,11 +194,12 @@ class Consumer(LoggingMixin): parser_class = get_parser_class_for_mime_type(mime_type) if not parser_class: - raise ConsumerError( - f"Unsupported mime type {mime_type} of file {self.filename}") + self._fail( + MESSAGE_UNSUPPORTED_TYPE, + f"Unsupported mime type {mime_type}" + ) else: - self.log("debug", - f"Parser: {parser_class.__name__}") + self.log("debug", f"Parser: {parser_class.__name__}") # Notify all listeners that we're going to do some work. @@ -165,35 +211,50 @@ class Consumer(LoggingMixin): self.run_pre_consume_script() + def progress_callback(current_progress, max_progress): + # recalculate progress to be within 20 and 80 + p = int((current_progress / max_progress) * 50 + 20) + self._send_progress(p, 100, "WORKING") + # This doesn't parse the document yet, but gives us a parser. - document_parser = parser_class(self.logging_group) + document_parser = parser_class(self.logging_group, progress_callback) # However, this already created working directories which we have to # clean up. # Parse the document. This may take some time. + text = None + date = None + thumbnail = None + archive_path = None + try: + self._send_progress(20, 100, 'WORKING', MESSAGE_PARSING_DOCUMENT) self.log("debug", "Parsing {}...".format(self.filename)) document_parser.parse(self.path, mime_type, self.filename) self.log("debug", f"Generating thumbnail for {self.filename}...") + self._send_progress(70, 100, 'WORKING', + MESSAGE_GENERATING_THUMBNAIL) thumbnail = document_parser.get_optimised_thumbnail( self.path, mime_type) text = document_parser.get_text() date = document_parser.get_date() if not date: + self._send_progress(90, 100, 'WORKING', + MESSAGE_PARSE_DATE) date = parse_date(self.filename, text) archive_path = document_parser.get_archive_path() except ParseError as e: document_parser.cleanup() - self.log( - "error", - f"Error while consuming document {self.filename}: {e}") - raise ConsumerError(e) + self._fail( + str(e), + f"Error while consuming document {self.filename}: {e}" + ) # Prepare the document classifier. @@ -203,6 +264,7 @@ class Consumer(LoggingMixin): classifier = load_classifier() + self._send_progress(95, 100, 'WORKING', MESSAGE_SAVE_DOCUMENT) # now that everything is done, we can start to store the document # in the system. This will be a transaction and reasonably fast. try: @@ -256,12 +318,11 @@ class Consumer(LoggingMixin): os.unlink(self.path) except Exception as e: - self.log( - "error", + self._fail( + str(e), f"The following error occured while consuming " f"{self.filename}: {e}" ) - raise ConsumerError(e) finally: document_parser.cleanup() @@ -272,6 +333,8 @@ class Consumer(LoggingMixin): "Document {} consumption finished".format(document) ) + self._send_progress(100, 100, 'SUCCESS', MESSAGE_FINISHED, document.id) + return document def _store(self, text, date, mime_type): diff --git a/src/documents/parsers.py b/src/documents/parsers.py index c0039207f..3f0879b3c 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -261,7 +261,7 @@ class DocumentParser(LoggingMixin): `paperless_tesseract.parsers` for inspiration. """ - def __init__(self, logging_group): + def __init__(self, logging_group, progress_callback=None): super().__init__() self.logging_group = logging_group os.makedirs(settings.SCRATCH_DIR, exist_ok=True) @@ -271,6 +271,12 @@ class DocumentParser(LoggingMixin): self.archive_path = None self.text = None self.date = None + self.progress_callback = progress_callback + + def progress(self, current, max): + print(self.progress_callback) + if self.progress_callback: + self.progress_callback(current, max) def extract_metadata(self, document_path, mime_type): return [] diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 70a44d3fd..aa4ef4bf8 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -8,6 +8,8 @@ from .models import Correspondent, Tag, Document, Log, DocumentType, \ SavedView, SavedViewFilterRule from .parsers import is_mime_type_supported +from django.utils.translation import gettext as _ + # https://www.django-rest-framework.org/api-guide/serializers/#example class DynamicFieldsModelSerializer(serializers.ModelSerializer): @@ -378,7 +380,9 @@ class PostDocumentSerializer(serializers.Serializer): if not is_mime_type_supported(mime_type): raise serializers.ValidationError( - "This file type is not supported.") + _("File type %(type)s not supported") % + {'type': mime_type} + ) return document.name, document_data diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 4e74d7350..e0d726d3e 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -66,7 +66,8 @@ def consume_file(path, override_title=None, override_correspondent_id=None, override_document_type_id=None, - override_tag_ids=None): + override_tag_ids=None, + task_id=None): document = Consumer().try_consume_file( path, @@ -74,7 +75,9 @@ def consume_file(path, override_title=override_title, override_correspondent_id=override_correspondent_id, override_document_type_id=override_document_type_id, - override_tag_ids=override_tag_ids) + override_tag_ids=override_tag_ids, + task_id=task_id + ) if document: return "Success. New document id {} created".format( diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index 02d1d0004..a6f0cc55a 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -170,7 +170,7 @@ class DummyParser(DocumentParser): raise NotImplementedError() def __init__(self, logging_group, scratch_dir, archive_path): - super(DummyParser, self).__init__(logging_group) + super(DummyParser, self).__init__(logging_group, None) _, self.fake_thumb = tempfile.mkstemp(suffix=".png", dir=scratch_dir) self.archive_path = archive_path @@ -212,10 +212,24 @@ def fake_magic_from_file(file, mime=False): @mock.patch("documents.consumer.magic.from_file", fake_magic_from_file) class TestConsumer(DirectoriesMixin, TestCase): - def make_dummy_parser(self, logging_group): + def _assert_first_last_send_progress(self, first_status="STARTING", last_status="SUCCESS", first_progress=0, first_progress_max=100, last_progress=100, last_progress_max=100): + + self._send_progress.assert_called() + + args, kwargs = self._send_progress.call_args_list[0] + self.assertEqual(args[0], first_progress) + self.assertEqual(args[1], first_progress_max) + self.assertEqual(args[2], first_status) + + args, kwargs = self._send_progress.call_args_list[len(self._send_progress.call_args_list) - 1] + self.assertEqual(args[0], last_progress) + self.assertEqual(args[1], last_progress_max) + self.assertEqual(args[2], last_status) + + def make_dummy_parser(self, logging_group, progress_callback=None): return DummyParser(logging_group, self.dirs.scratch_dir, self.get_test_archive_file()) - def make_faulty_parser(self, logging_group): + def make_faulty_parser(self, logging_group, progress_callback=None): return FaultyParser(logging_group, self.dirs.scratch_dir) def setUp(self): @@ -228,7 +242,11 @@ class TestConsumer(DirectoriesMixin, TestCase): "mime_types": {"application/pdf": ".pdf"}, "weight": 0 })] + self.addCleanup(patcher.stop) + # this prevents websocket message reports during testing. + patcher = mock.patch("documents.consumer.Consumer._send_progress") + self._send_progress = patcher.start() self.addCleanup(patcher.stop) self.consumer = Consumer() @@ -274,6 +292,8 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertFalse(os.path.isfile(filename)) + self._assert_first_last_send_progress() + def testOverrideFilename(self): filename = self.get_test_file() override_filename = "Statement for November.pdf" @@ -282,21 +302,26 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertEqual(document.title, "Statement for November") + self._assert_first_last_send_progress() + def testOverrideTitle(self): document = self.consumer.try_consume_file(self.get_test_file(), override_title="Override Title") self.assertEqual(document.title, "Override Title") + self._assert_first_last_send_progress() def testOverrideCorrespondent(self): c = Correspondent.objects.create(name="test") document = self.consumer.try_consume_file(self.get_test_file(), override_correspondent_id=c.pk) self.assertEqual(document.correspondent.id, c.id) + self._assert_first_last_send_progress() def testOverrideDocumentType(self): dt = DocumentType.objects.create(name="test") document = self.consumer.try_consume_file(self.get_test_file(), override_document_type_id=dt.pk) self.assertEqual(document.document_type.id, dt.id) + self._assert_first_last_send_progress() def testOverrideTags(self): t1 = Tag.objects.create(name="t1") @@ -307,37 +332,42 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertIn(t1, document.tags.all()) self.assertNotIn(t2, document.tags.all()) self.assertIn(t3, document.tags.all()) + self._assert_first_last_send_progress() def testNotAFile(self): - try: - self.consumer.try_consume_file("non-existing-file") - except ConsumerError as e: - self.assertTrue(str(e).endswith('It is not a file')) - return - self.fail("Should throw exception") + self.assertRaisesMessage( + ConsumerError, + "File not found", + self.consumer.try_consume_file, + "non-existing-file" + ) + + self._assert_first_last_send_progress(last_status="FAILED") def testDuplicates1(self): self.consumer.try_consume_file(self.get_test_file()) - try: - self.consumer.try_consume_file(self.get_test_file()) - except ConsumerError as e: - self.assertTrue(str(e).endswith("It is a duplicate.")) - return + self.assertRaisesMessage( + ConsumerError, + "It is a duplicate", + self.consumer.try_consume_file, + self.get_test_file() + ) - self.fail("Should throw exception") + self._assert_first_last_send_progress(last_status="FAILED") def testDuplicates2(self): self.consumer.try_consume_file(self.get_test_file()) - try: - self.consumer.try_consume_file(self.get_test_archive_file()) - except ConsumerError as e: - self.assertTrue(str(e).endswith("It is a duplicate.")) - return + self.assertRaisesMessage( + ConsumerError, + "It is a duplicate", + self.consumer.try_consume_file, + self.get_test_archive_file() + ) - self.fail("Should throw exception") + self._assert_first_last_send_progress(last_status="FAILED") def testDuplicates3(self): self.consumer.try_consume_file(self.get_test_archive_file()) @@ -347,13 +377,15 @@ class TestConsumer(DirectoriesMixin, TestCase): def testNoParsers(self, m): m.return_value = [] - try: - self.consumer.try_consume_file(self.get_test_file()) - except ConsumerError as e: - self.assertEqual("Unsupported mime type application/pdf of file sample.pdf", str(e)) - return + self.assertRaisesMessage( + ConsumerError, + "sample.pdf: Unsupported mime type application/pdf", + self.consumer.try_consume_file, + self.get_test_file() + ) + + self._assert_first_last_send_progress(last_status="FAILED") - self.fail("Should throw exception") @mock.patch("documents.parsers.document_consumer_declaration.send") def testFaultyParser(self, m): @@ -363,24 +395,28 @@ class TestConsumer(DirectoriesMixin, TestCase): "weight": 0 })] - try: - self.consumer.try_consume_file(self.get_test_file()) - except ConsumerError as e: - self.assertEqual(str(e), "Does not compute.") - return + self.assertRaisesMessage( + ConsumerError, + "sample.pdf: Error while consuming document sample.pdf: Does not compute.", + self.consumer.try_consume_file, + self.get_test_file() + ) - self.fail("Should throw exception.") + self._assert_first_last_send_progress(last_status="FAILED") @mock.patch("documents.consumer.Consumer._write") def testPostSaveError(self, m): filename = self.get_test_file() m.side_effect = OSError("NO.") - try: - self.consumer.try_consume_file(filename) - except ConsumerError as e: - self.assertEqual(str(e), "NO.") - else: - self.fail("Should raise exception") + + self.assertRaisesMessage( + ConsumerError, + "sample.pdf: The following error occured while consuming sample.pdf: NO.", + self.consumer.try_consume_file, + filename + ) + + self._assert_first_last_send_progress(last_status="FAILED") # file not deleted self.assertTrue(os.path.isfile(filename)) @@ -397,6 +433,8 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertEqual(document.title, "new docs") self.assertEqual(document.filename, "none/new docs.pdf") + self._assert_first_last_send_progress() + @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}") @mock.patch("documents.signals.handlers.generate_unique_filename") def testFilenameHandlingUnstableFormat(self, m): @@ -420,6 +458,8 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertIsNotNone(os.path.isfile(document.title)) self.assertTrue(os.path.isfile(document.source_path)) + self._assert_first_last_send_progress() + @mock.patch("documents.consumer.load_classifier") def testClassifyDocument(self, m): correspondent = Correspondent.objects.create(name="test") @@ -439,19 +479,26 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertIn(t1, document.tags.all()) self.assertNotIn(t2, document.tags.all()) + self._assert_first_last_send_progress() + @override_settings(CONSUMER_DELETE_DUPLICATES=True) def test_delete_duplicate(self): dst = self.get_test_file() self.assertTrue(os.path.isfile(dst)) doc = self.consumer.try_consume_file(dst) + self._assert_first_last_send_progress() + self.assertFalse(os.path.isfile(dst)) self.assertIsNotNone(doc) + self._send_progress.reset_mock() + dst = self.get_test_file() self.assertTrue(os.path.isfile(dst)) self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst) self.assertFalse(os.path.isfile(dst)) + self._assert_first_last_send_progress(last_status="FAILED") @override_settings(CONSUMER_DELETE_DUPLICATES=False) def test_no_delete_duplicate(self): @@ -467,6 +514,8 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst) self.assertTrue(os.path.isfile(dst)) + self._assert_first_last_send_progress(last_status="FAILED") + class PreConsumeTestCase(TestCase): @@ -479,9 +528,11 @@ class PreConsumeTestCase(TestCase): m.assert_not_called() @mock.patch("documents.consumer.Popen") + @mock.patch("documents.consumer.Consumer._send_progress") @override_settings(PRE_CONSUME_SCRIPT="does-not-exist") - def test_pre_consume_script_not_found(self, m): + def test_pre_consume_script_not_found(self, m, m2): c = Consumer() + c.filename = "somefile.pdf" c.path = "path-to-file" self.assertRaises(ConsumerError, c.run_pre_consume_script) @@ -503,7 +554,6 @@ class PreConsumeTestCase(TestCase): self.assertEqual(command[1], "path-to-file") - class PostConsumeTestCase(TestCase): @mock.patch("documents.consumer.Popen") @@ -519,12 +569,13 @@ class PostConsumeTestCase(TestCase): m.assert_not_called() - @override_settings(POST_CONSUME_SCRIPT="does-not-exist") - def test_post_consume_script_not_found(self): + @mock.patch("documents.consumer.Consumer._send_progress") + def test_post_consume_script_not_found(self, m): doc = Document.objects.create(title="Test", mime_type="application/pdf") - - self.assertRaises(ConsumerError, Consumer().run_post_consume_script, doc) + c = Consumer() + c.filename = "somefile.pdf" + self.assertRaises(ConsumerError, c.run_post_consume_script, doc) @mock.patch("documents.consumer.Popen") def test_post_consume_script_simple(self, m): diff --git a/src/documents/views.py b/src/documents/views.py index f7c640e77..c2b31c996 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -1,6 +1,7 @@ import logging import os import tempfile +import uuid from datetime import datetime from time import mktime @@ -213,7 +214,7 @@ class DocumentViewSet(RetrieveModelMixin, parser_class = get_parser_class_for_mime_type(mime_type) if parser_class: - parser = parser_class(logging_group=None) + parser = parser_class(progress_callback=None, logging_group=None) try: return parser.extract_metadata(file, mime_type) @@ -403,6 +404,8 @@ class PostDocumentView(APIView): os.utime(f.name, times=(t, t)) temp_filename = f.name + task_id = str(uuid.uuid4()) + async_task("documents.tasks.consume_file", temp_filename, override_filename=doc_name, @@ -410,6 +413,7 @@ class PostDocumentView(APIView): override_correspondent_id=correspondent_id, override_document_type_id=document_type_id, override_tag_ids=tag_ids, + task_id=task_id, task_name=os.path.basename(doc_name)[:100]) return Response("OK") diff --git a/src/locale/en-us/LC_MESSAGES/django.po b/src/locale/en-us/LC_MESSAGES/django.po index 414caba52..fdf3fd809 100644 --- a/src/locale/en-us/LC_MESSAGES/django.po +++ b/src/locale/en-us/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2021-01-10 21:41+0000\n" +"POT-Creation-Date: 2021-01-28 22:02+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -21,323 +21,328 @@ msgstr "" msgid "Documents" msgstr "" -#: documents/models.py:32 +#: documents/models.py:33 msgid "Any word" msgstr "" -#: documents/models.py:33 +#: documents/models.py:34 msgid "All words" msgstr "" -#: documents/models.py:34 +#: documents/models.py:35 msgid "Exact match" msgstr "" -#: documents/models.py:35 +#: documents/models.py:36 msgid "Regular expression" msgstr "" -#: documents/models.py:36 +#: documents/models.py:37 msgid "Fuzzy word" msgstr "" -#: documents/models.py:37 +#: documents/models.py:38 msgid "Automatic" msgstr "" -#: documents/models.py:41 documents/models.py:354 paperless_mail/models.py:25 +#: documents/models.py:42 documents/models.py:352 paperless_mail/models.py:25 #: paperless_mail/models.py:109 msgid "name" msgstr "" -#: documents/models.py:45 +#: documents/models.py:46 msgid "match" msgstr "" -#: documents/models.py:49 +#: documents/models.py:50 msgid "matching algorithm" msgstr "" -#: documents/models.py:55 +#: documents/models.py:56 msgid "is insensitive" msgstr "" -#: documents/models.py:80 documents/models.py:140 +#: documents/models.py:75 documents/models.py:135 msgid "correspondent" msgstr "" -#: documents/models.py:81 +#: documents/models.py:76 msgid "correspondents" msgstr "" -#: documents/models.py:103 +#: documents/models.py:98 msgid "color" msgstr "" -#: documents/models.py:107 +#: documents/models.py:102 msgid "is inbox tag" msgstr "" -#: documents/models.py:109 +#: documents/models.py:104 msgid "" "Marks this tag as an inbox tag: All newly consumed documents will be tagged " "with inbox tags." msgstr "" -#: documents/models.py:114 +#: documents/models.py:109 msgid "tag" msgstr "" -#: documents/models.py:115 documents/models.py:171 +#: documents/models.py:110 documents/models.py:166 msgid "tags" msgstr "" -#: documents/models.py:121 documents/models.py:153 +#: documents/models.py:116 documents/models.py:148 msgid "document type" msgstr "" -#: documents/models.py:122 +#: documents/models.py:117 msgid "document types" msgstr "" -#: documents/models.py:130 +#: documents/models.py:125 msgid "Unencrypted" msgstr "" -#: documents/models.py:131 +#: documents/models.py:126 msgid "Encrypted with GNU Privacy Guard" msgstr "" -#: documents/models.py:144 +#: documents/models.py:139 msgid "title" msgstr "" -#: documents/models.py:157 +#: documents/models.py:152 msgid "content" msgstr "" -#: documents/models.py:159 +#: documents/models.py:154 msgid "" "The raw, text-only data of the document. This field is primarily used for " "searching." msgstr "" -#: documents/models.py:164 +#: documents/models.py:159 msgid "mime type" msgstr "" -#: documents/models.py:175 +#: documents/models.py:170 msgid "checksum" msgstr "" -#: documents/models.py:179 +#: documents/models.py:174 msgid "The checksum of the original document." msgstr "" -#: documents/models.py:183 +#: documents/models.py:178 msgid "archive checksum" msgstr "" -#: documents/models.py:188 +#: documents/models.py:183 msgid "The checksum of the archived document." msgstr "" -#: documents/models.py:192 documents/models.py:332 +#: documents/models.py:187 documents/models.py:330 msgid "created" msgstr "" -#: documents/models.py:196 +#: documents/models.py:191 msgid "modified" msgstr "" -#: documents/models.py:200 +#: documents/models.py:195 msgid "storage type" msgstr "" -#: documents/models.py:208 +#: documents/models.py:203 msgid "added" msgstr "" -#: documents/models.py:212 +#: documents/models.py:207 msgid "filename" msgstr "" -#: documents/models.py:217 +#: documents/models.py:212 msgid "Current filename in storage" msgstr "" -#: documents/models.py:221 +#: documents/models.py:216 msgid "archive serial number" msgstr "" -#: documents/models.py:226 +#: documents/models.py:221 msgid "The position of this document in your physical document archive." msgstr "" -#: documents/models.py:232 +#: documents/models.py:227 msgid "document" msgstr "" -#: documents/models.py:233 +#: documents/models.py:228 msgid "documents" msgstr "" -#: documents/models.py:315 +#: documents/models.py:313 msgid "debug" msgstr "" -#: documents/models.py:316 +#: documents/models.py:314 msgid "information" msgstr "" -#: documents/models.py:317 +#: documents/models.py:315 msgid "warning" msgstr "" -#: documents/models.py:318 +#: documents/models.py:316 msgid "error" msgstr "" -#: documents/models.py:319 +#: documents/models.py:317 msgid "critical" msgstr "" -#: documents/models.py:323 +#: documents/models.py:321 msgid "group" msgstr "" -#: documents/models.py:326 +#: documents/models.py:324 msgid "message" msgstr "" -#: documents/models.py:329 +#: documents/models.py:327 msgid "level" msgstr "" -#: documents/models.py:336 +#: documents/models.py:334 msgid "log" msgstr "" -#: documents/models.py:337 +#: documents/models.py:335 msgid "logs" msgstr "" -#: documents/models.py:348 documents/models.py:398 +#: documents/models.py:346 documents/models.py:396 msgid "saved view" msgstr "" -#: documents/models.py:349 +#: documents/models.py:347 msgid "saved views" msgstr "" -#: documents/models.py:352 +#: documents/models.py:350 msgid "user" msgstr "" -#: documents/models.py:358 +#: documents/models.py:356 msgid "show on dashboard" msgstr "" -#: documents/models.py:361 +#: documents/models.py:359 msgid "show in sidebar" msgstr "" -#: documents/models.py:365 +#: documents/models.py:363 msgid "sort field" msgstr "" -#: documents/models.py:368 +#: documents/models.py:366 msgid "sort reverse" msgstr "" -#: documents/models.py:374 +#: documents/models.py:372 msgid "title contains" msgstr "" -#: documents/models.py:375 +#: documents/models.py:373 msgid "content contains" msgstr "" -#: documents/models.py:376 +#: documents/models.py:374 msgid "ASN is" msgstr "" -#: documents/models.py:377 +#: documents/models.py:375 msgid "correspondent is" msgstr "" -#: documents/models.py:378 +#: documents/models.py:376 msgid "document type is" msgstr "" -#: documents/models.py:379 +#: documents/models.py:377 msgid "is in inbox" msgstr "" -#: documents/models.py:380 +#: documents/models.py:378 msgid "has tag" msgstr "" -#: documents/models.py:381 +#: documents/models.py:379 msgid "has any tag" msgstr "" -#: documents/models.py:382 +#: documents/models.py:380 msgid "created before" msgstr "" -#: documents/models.py:383 +#: documents/models.py:381 msgid "created after" msgstr "" -#: documents/models.py:384 +#: documents/models.py:382 msgid "created year is" msgstr "" -#: documents/models.py:385 +#: documents/models.py:383 msgid "created month is" msgstr "" -#: documents/models.py:386 +#: documents/models.py:384 msgid "created day is" msgstr "" -#: documents/models.py:387 +#: documents/models.py:385 msgid "added before" msgstr "" -#: documents/models.py:388 +#: documents/models.py:386 msgid "added after" msgstr "" -#: documents/models.py:389 +#: documents/models.py:387 msgid "modified before" msgstr "" -#: documents/models.py:390 +#: documents/models.py:388 msgid "modified after" msgstr "" -#: documents/models.py:391 +#: documents/models.py:389 msgid "does not have tag" msgstr "" -#: documents/models.py:402 +#: documents/models.py:400 msgid "rule type" msgstr "" -#: documents/models.py:406 +#: documents/models.py:404 msgid "value" msgstr "" -#: documents/models.py:412 +#: documents/models.py:410 msgid "filter rule" msgstr "" -#: documents/models.py:413 +#: documents/models.py:411 msgid "filter rules" msgstr "" +#: documents/serialisers.py:383 +#, python-format +msgid "File type %(type)s not supported" +msgstr "" + #: documents/templates/index.html:20 msgid "Paperless-ng is loading..." msgstr "" @@ -378,23 +383,23 @@ msgstr "" msgid "Sign in" msgstr "" -#: paperless/settings.py:268 +#: paperless/settings.py:286 msgid "English" msgstr "" -#: paperless/settings.py:269 +#: paperless/settings.py:287 msgid "German" msgstr "" -#: paperless/settings.py:270 +#: paperless/settings.py:288 msgid "Dutch" msgstr "" -#: paperless/settings.py:271 +#: paperless/settings.py:289 msgid "French" msgstr "" -#: paperless/urls.py:108 +#: paperless/urls.py:114 msgid "Paperless-ng administration" msgstr "" diff --git a/src/paperless/asgi.py b/src/paperless/asgi.py new file mode 100644 index 000000000..2f6cc2d5f --- /dev/null +++ b/src/paperless/asgi.py @@ -0,0 +1,23 @@ +import os + +from django.core.asgi import get_asgi_application +# Fetch Django ASGI application early to ensure AppRegistry is populated +# before importing consumers and AuthMiddlewareStack that may import ORM +# models. + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings") +django_asgi_app = get_asgi_application() + +from channels.auth import AuthMiddlewareStack # NOQA: E402 +from channels.routing import ProtocolTypeRouter, URLRouter # NOQA: E402 + +from paperless.urls import websocket_urlpatterns # NOQA: E402 + +application = ProtocolTypeRouter({ + "http": get_asgi_application(), + "websocket": AuthMiddlewareStack( + URLRouter( + websocket_urlpatterns + ) + ), +}) diff --git a/src/paperless/consumers.py b/src/paperless/consumers.py new file mode 100644 index 000000000..21a0e3ede --- /dev/null +++ b/src/paperless/consumers.py @@ -0,0 +1,18 @@ +import json + +from asgiref.sync import async_to_sync +from channels.generic.websocket import WebsocketConsumer + + +class StatusConsumer(WebsocketConsumer): + def connect(self): + self.accept() + async_to_sync(self.channel_layer.group_add)( + 'status_updates', self.channel_name) + + def disconnect(self, close_code): + async_to_sync(self.channel_layer.group_discard)( + 'status_updates', self.channel_name) + + def status_update(self, event): + self.send(json.dumps(event['data'])) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index bc70cb331..b6d01ba53 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -100,6 +100,8 @@ INSTALLED_APPS = [ "django_q", + "channels", + ] + env_apps REST_FRAMEWORK = { @@ -133,6 +135,7 @@ ROOT_URLCONF = 'paperless.urls' FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME") WSGI_APPLICATION = 'paperless.wsgi.application' +ASGI_APPLICATION = "paperless.asgi.application" STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", "/static/") @@ -153,6 +156,15 @@ TEMPLATES = [ }, ] +CHANNEL_LAYERS = { + "default": { + "BACKEND": "channels_redis.core.RedisChannelLayer", + "CONFIG": { + "hosts": [os.getenv("PAPERLESS_REDIS", "redis://localhost:6379")], + }, + }, +} + ############################################################################### # Security # ############################################################################### diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 49e6e8b7a..5b58de454 100755 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -9,6 +9,7 @@ from rest_framework.routers import DefaultRouter from django.utils.translation import gettext_lazy as _ +from paperless.consumers import StatusConsumer from documents.views import ( CorrespondentViewSet, DocumentViewSet, @@ -100,6 +101,11 @@ urlpatterns = [ re_path(r".*", login_required(IndexView.as_view())), ] + +websocket_urlpatterns = [ + re_path(r'ws/status/$', StatusConsumer.as_asgi()), +] + # Text in each page's

(and above login form). admin.site.site_header = 'Paperless-ng' # Text at the end of each page's . diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index 31e956284..52b50e983 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -190,11 +190,11 @@ class RasterisedDocumentParser(DocumentParser): # Also, no archived file. if not self.text: # However, if we don't have anything, fail: - raise ParseError(e) + raise ParseError(e.__class__.__name__ + ": " + str(e)) except Exception as e: # Anything else is probably serious. - raise ParseError(e) + raise ParseError(e.__class__.__name__ + ": " + str(e)) if not self.text: # This may happen for files that don't have any text.