From 572e40ca2713bc5d79fc3ee5fb852b0c7b14d6db Mon Sep 17 00:00:00 2001 From: Jonas Winkler Date: Sat, 7 Nov 2020 11:30:45 +0100 Subject: [PATCH] backend that supports asgi and status update sockets with channels --- Pipfile | 2 + Pipfile.lock | 582 +++++++++++++++++++++++++---- src/documents/consumer.py | 44 ++- src/documents/parsers.py | 3 +- src/paperless/asgi.py | 37 ++ src/paperless/settings.py | 12 + src/paperless_tesseract/parsers.py | 25 +- 7 files changed, 613 insertions(+), 92 deletions(-) create mode 100644 src/paperless/asgi.py diff --git a/Pipfile b/Pipfile index e8f862578..d526ae252 100644 --- a/Pipfile +++ b/Pipfile @@ -26,6 +26,8 @@ fuzzywuzzy = "*" python-Levenshtein = "*" django-extensions = "" watchdog = "*" +channels = "~=3.0" +channels-redis = "*" [dev-packages] coveralls = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 8b3bf705a..642e38214 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2c1558fe7df0aee1ee20b095c2102f802470bf4a4ae09a7749ac487f8bfab8b6" + "sha256": "192d7419b844e6bb81fed793e7766b2ba15f2a016af1a33fc73cf09e12de5fb7" }, "pipfile-spec": 6, "requires": {}, @@ -14,13 +14,151 @@ ] }, "default": { + "aioredis": { + "hashes": [ + "sha256:15f8af30b044c771aee6787e5ec24694c048184c7b9e54c3b60c750a4b93273a", + "sha256:b61808d7e97b7cd5a92ed574937a079c9387fdadd22bfbfa7ad2fd319ecc26e3" + ], + "version": "==1.3.1" + }, "asgiref": { "hashes": [ - "sha256:7e51911ee147dd685c3c8b805c0ad0cb58d360987b56953878f8c06d2d1c6f1a", - "sha256:9fc6fb5d39b8af147ba40765234fa822b39818b12cc80b35ad9b0cef3a476aed" + "sha256:a5098bc870b80e7b872bff60bb363c7f2c2c89078759f6c47b53ff8c525a152e", + "sha256:cd88907ecaec59d78e4ac00ea665b03e571cb37e3a0e37b3702af1a9e86c365a" ], "markers": "python_version >= '3.5'", - "version": "==3.2.10" + "version": "==3.3.0" + }, + "async-timeout": { + "hashes": [ + "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f", + "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3" + ], + "markers": "python_full_version >= '3.5.3'", + "version": "==3.0.1" + }, + "attrs": { + "hashes": [ + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.3.0" + }, + "autobahn": { + "hashes": [ + "sha256:24ce276d313e84d68241c3aef30d484f352b90a40168981b3640312c821df77b", + "sha256:86bbce30cdd407137c57670993a8f9bfdfe3f8e994b889181d85e844d5aa8dfb" + ], + "markers": "python_version >= '3.5'", + "version": "==20.7.1" + }, + "automat": { + "hashes": [ + "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33", + "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111" + ], + "version": "==20.2.0" + }, + "cffi": { + "hashes": [ + "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", + "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", + "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", + "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", + "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", + "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", + "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", + "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", + "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", + "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", + "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", + "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", + "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", + "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", + "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", + "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", + "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", + "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", + "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", + "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", + "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", + "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", + "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", + "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", + "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", + "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", + "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", + "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", + "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", + "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", + "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", + "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", + "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", + "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", + "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", + "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" + ], + "version": "==1.14.3" + }, + "channels": { + "hashes": [ + "sha256:5cdd9c6b9ee663cdf1bbb00de7cdab885a3c418f9d32a29f04b09498828020f6", + "sha256:b02e150b48704ec3607d4168402ac5c26138dd183fcdb7f2aeb965e6e19fd558" + ], + "index": "pypi", + "version": "==3.0.1" + }, + "channels-redis": { + "hashes": [ + "sha256:18d63f6462a58011740dc8eeb57ea4b31ec220eb551cb71b27de9c6779a549de", + "sha256:2fb31a63b05373f6402da2e6a91a22b9e66eb8b56626c6bfc93e156c734c5ae6" + ], + "index": "pypi", + "version": "==3.2.0" + }, + "constantly": { + "hashes": [ + "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35", + "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d" + ], + "version": "==15.1.0" + }, + "cryptography": { + "hashes": [ + "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538", + "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f", + "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77", + "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b", + "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33", + "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e", + "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb", + "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e", + "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7", + "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297", + "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d", + "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7", + "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b", + "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7", + "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4", + "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8", + "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b", + "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851", + "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13", + "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b", + "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3", + "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==3.2.1" + }, + "daphne": { + "hashes": [ + "sha256:60856f7efa0b1e1b969efa074e8698bd09de4713ecc06e6a4d19d04c66c4a3bd", + "sha256:b43e70d74ff832a634ff6c92badd208824e4530e08b340116517e5aad0aca774" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.0" }, "dateparser": { "hashes": [ @@ -32,11 +170,11 @@ }, "django": { "hashes": [ - "sha256:a2127ad0150ec6966655bedf15dbbff9697cc86d61653db2da1afa506c0b04cc", - "sha256:c93c28ccf1d094cbd00d860e83128a39e45d2c571d3b54361713aaaf9a94cac4" + "sha256:14a4b7cd77297fba516fc0d92444cc2e2e388aa9de32d7a68d4a83d58f5a4927", + "sha256:14b87775ffedab2ef6299b73343d1b4b41e5d4e2aa58c6581f114dbec01e3f8f" ], "index": "pypi", - "version": "==3.1.2" + "version": "==3.1.3" }, "django-cors-headers": { "hashes": [ @@ -65,11 +203,10 @@ }, "djangorestframework": { "hashes": [ - "sha256:5c5071fcbad6dce16f566d492015c829ddb0df42965d488b878594aabc3aed21", - "sha256:d54452aedebb4b650254ca092f9f4f5df947cb1de6ab245d817b08b4f4156249" + "sha256:0209bafcb7b5010fdfec784034f059d512256424de2a0f084cb82b096d6dd6a7" ], "index": "pypi", - "version": "==3.12.1" + "version": "==3.12.2" }, "filemagic": { "hashes": [ @@ -94,6 +231,80 @@ "index": "pypi", "version": "==20.0.4" }, + "hiredis": { + "hashes": [ + "sha256:06a039208f83744a702279b894c8cf24c14fd63c59cd917dcde168b79eef0680", + "sha256:0a909bf501459062aa1552be1461456518f367379fdc9fdb1f2ca5e4a1fdd7c0", + "sha256:18402d9e54fb278cb9a8c638df6f1550aca36a009d47ecf5aa263a38600f35b0", + "sha256:1e4cbbc3858ec7e680006e5ca590d89a5e083235988f26a004acf7244389ac01", + "sha256:23344e3c2177baf6975fbfa361ed92eb7d36d08f454636e5054b3faa7c2aff8a", + "sha256:289b31885b4996ce04cadfd5fc03d034dce8e2a8234479f7c9e23b9e245db06b", + "sha256:2c1c570ae7bf1bab304f29427e2475fe1856814312c4a1cf1cd0ee133f07a3c6", + "sha256:2c227c0ed371771ffda256034427320870e8ea2e4fd0c0a618c766e7c49aad73", + "sha256:3bb9b63d319402cead8bbd9dd55dca3b667d2997e9a0d8a1f9b6cc274db4baee", + "sha256:3ef2183de67b59930d2db8b8e8d4d58e00a50fcc5e92f4f678f6eed7a1c72d55", + "sha256:43b8ed3dbfd9171e44c554cb4acf4ee4505caa84c5e341858b50ea27dd2b6e12", + "sha256:47bcf3c5e6c1e87ceb86cdda2ee983fa0fe56a999e6185099b3c93a223f2fa9b", + "sha256:5263db1e2e1e8ae30500cdd75a979ff99dcc184201e6b4b820d0de74834d2323", + "sha256:5b1451727f02e7acbdf6aae4e06d75f66ee82966ff9114550381c3271a90f56c", + "sha256:6996883a8a6ff9117cbb3d6f5b0dcbbae6fb9e31e1a3e4e2f95e0214d9a1c655", + "sha256:6c96f64a54f030366657a54bb90b3093afc9c16c8e0dfa29fc0d6dbe169103a5", + "sha256:7332d5c3e35154cd234fd79573736ddcf7a0ade7a986db35b6196b9171493e75", + "sha256:7885b6f32c4a898e825bb7f56f36a02781ac4a951c63e4169f0afcf9c8c30dfb", + "sha256:7b0f63f10a166583ab744a58baad04e0f52cfea1ac27bfa1b0c21a48d1003c23", + "sha256:819f95d4eba3f9e484dd115ab7ab72845cf766b84286a00d4ecf76d33f1edca1", + "sha256:8968eeaa4d37a38f8ca1f9dbe53526b69628edc9c42229a5b2f56d98bb828c1f", + "sha256:89ebf69cb19a33d625db72d2ac589d26e936b8f7628531269accf4a3196e7872", + "sha256:8daecd778c1da45b8bd54fd41ffcd471a86beed3d8e57a43acf7a8d63bba4058", + "sha256:955ba8ea73cf3ed8bd2f963b4cb9f8f0dcb27becd2f4b3dd536fd24c45533454", + "sha256:964f18a59f5a64c0170f684c417f4fe3e695a536612e13074c4dd5d1c6d7c882", + "sha256:969843fbdfbf56cdb71da6f0bdf50f9985b8b8aeb630102945306cf10a9c6af2", + "sha256:996021ef33e0f50b97ff2d6b5f422a0fe5577de21a8873b58a779a5ddd1c3132", + "sha256:9e9c9078a7ce07e6fce366bd818be89365a35d2e4b163268f0ca9ba7e13bb2f6", + "sha256:a04901757cb0fb0f5602ac11dda48f5510f94372144d06c2563ba56c480b467c", + "sha256:a7bf1492429f18d205f3a818da3ff1f242f60aa59006e53dee00b4ef592a3363", + "sha256:aa0af2deb166a5e26e0d554b824605e660039b161e37ed4f01b8d04beec184f3", + "sha256:abfb15a6a7822f0fae681785cb38860e7a2cb1616a708d53df557b3d76c5bfd4", + "sha256:b253fe4df2afea4dfa6b1fa8c5fef212aff8bcaaeb4207e81eed05cb5e4a7919", + "sha256:b27f082f47d23cffc4cf1388b84fdc45c4ef6015f906cd7e0d988d9e35d36349", + "sha256:b33aea449e7f46738811fbc6f0b3177c6777a572207412bbbf6f525ffed001ae", + "sha256:b44f9421c4505c548435244d74037618f452844c5d3c67719d8a55e2613549da", + "sha256:bcc371151d1512201d0214c36c0c150b1dc64f19c2b1a8c9cb1d7c7c15ebd93f", + "sha256:c2851deeabd96d3f6283e9c6b26e0bfed4de2dc6fb15edf913e78b79fc5909ed", + "sha256:cdfd501c7ac5b198c15df800a3a34c38345f5182e5f80770caf362bccca65628", + "sha256:d2c0caffa47606d6d7c8af94ba42547bd2a441f06c74fd90a1ffe328524a6c64", + "sha256:dcb2db95e629962db5a355047fb8aefb012df6c8ae608930d391619dbd96fd86", + "sha256:e0eeb9c112fec2031927a1745788a181d0eecbacbed941fc5c4f7bc3f7b273bf", + "sha256:e154891263306200260d7f3051982774d7b9ef35af3509d5adbbe539afd2610c", + "sha256:e2e023a42dcbab8ed31f97c2bcdb980b7fbe0ada34037d87ba9d799664b58ded", + "sha256:e64be68255234bb489a574c4f2f8df7029c98c81ec4d160d6cd836e7f0679390", + "sha256:e82d6b930e02e80e5109b678c663a9ed210680ded81c1abaf54635d88d1da298" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.0" + }, + "hyperlink": { + "hashes": [ + "sha256:47fcc7cd339c6cb2444463ec3277bdcfe142c8b1daf2160bdd52248deec815af", + "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63" + ], + "version": "==20.0.1" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "incremental": { + "hashes": [ + "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", + "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3" + ], + "version": "==17.5.0" + }, "joblib": { "hashes": [ "sha256:698c311779f347cf6b7e6b8a39bb682277b8ee4aba8cf9507bc0cf4cd4737b72", @@ -110,45 +321,68 @@ "index": "pypi", "version": "==1.0.8" }, + "msgpack": { + "hashes": [ + "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408", + "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8", + "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84", + "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d", + "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a", + "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322", + "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2", + "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e", + "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97", + "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0", + "sha256:aa5c057eab4f40ec47ea6f5a9825846be2ff6bf34102c560bad5cad5a677c5be", + "sha256:b3758dfd3423e358bbb18a7cccd1c74228dffa7a697e5be6cb9535de625c0dbf", + "sha256:c901e8058dd6653307906c5f157f26ed09eb94a850dddd989621098d347926ab", + "sha256:cec8bf10981ed70998d98431cd814db0ecf3384e6b113366e7f36af71a0fca08", + "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e", + "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272", + "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1", + "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140" + ], + "version": "==1.0.0" + }, "numpy": { "hashes": [ - "sha256:0ee77786eebbfa37f2141fd106b549d37c89207a0d01d8852fde1c82e9bfc0e7", - "sha256:199bebc296bd8a5fc31c16f256ac873dd4d5b4928dfd50e6c4995570fc71a8f3", - "sha256:1a307bdd3dd444b1d0daa356b5f4c7de2e24d63bdc33ea13ff718b8ec4c6a268", - "sha256:1ea7e859f16e72ab81ef20aae69216cfea870676347510da9244805ff9670170", - "sha256:271139653e8b7a046d11a78c0d33bafbddd5c443a5b9119618d0652a4eb3a09f", - "sha256:35bf5316af8dc7c7db1ad45bec603e5fb28671beb98ebd1d65e8059efcfd3b72", - "sha256:463792a249a81b9eb2b63676347f996d3f0082c2666fd0604f4180d2e5445996", - "sha256:50d3513469acf5b2c0406e822d3f314d7ac5788c2b438c24e5dd54d5a81ef522", - "sha256:50f68ebc439821b826823a8da6caa79cd080dee2a6d5ab9f1163465a060495ed", - "sha256:51e8d2ae7c7e985c7bebf218e56f72fa93c900ad0c8a7d9fbbbf362f45710f69", - "sha256:522053b731e11329dd52d258ddf7de5288cae7418b55e4b7d32f0b7e31787e9d", - "sha256:5ea4401ada0d3988c263df85feb33818dc995abc85b8125f6ccb762009e7bc68", - "sha256:604d2e5a31482a3ad2c88206efd43d6fcf666ada1f3188fd779b4917e49b7a98", - "sha256:6ff88bcf1872b79002569c63fe26cd2cda614e573c553c4d5b814fb5eb3d2822", - "sha256:7197ee0a25629ed782c7bd01871ee40702ffeef35bc48004bc2fdcc71e29ba9d", - "sha256:741d95eb2b505bb7a99fbf4be05fa69f466e240c2b4f2d3ddead4f1b5f82a5a5", - "sha256:83af653bb92d1e248ccf5fdb05ccc934c14b936bcfe9b917dc180d3f00250ac6", - "sha256:8802d23e4895e0c65e418abe67cdf518aa5cbb976d97f42fd591f921d6dffad0", - "sha256:8edc4d687a74d0a5f8b9b26532e860f4f85f56c400b3a98899fc44acb5e27add", - "sha256:942d2cdcb362739908c26ce8dd88db6e139d3fa829dd7452dd9ff02cba6b58b2", - "sha256:9a0669787ba8c9d3bb5de5d9429208882fb47764aa79123af25c5edc4f5966b9", - "sha256:9d08d84bb4128abb9fbd9f073e5c69f70e5dab991a9c42e5b4081ea5b01b5db0", - "sha256:9f7f56b5e85b08774939622b7d45a5d00ff511466522c44fc0756ac7692c00f2", - "sha256:a2daea1cba83210c620e359de2861316f49cc7aea8e9a6979d6cb2ddab6dda8c", - "sha256:b9074d062d30c2779d8af587924f178a539edde5285d961d2dfbecbac9c4c931", - "sha256:c4aa79993f5d856765819a3651117520e41ac3f89c3fc1cb6dee11aa562df6da", - "sha256:d78294f1c20f366cde8a75167f822538a7252b6e8b9d6dbfb3bdab34e7c1929e", - "sha256:dfdc8b53aa9838b9d44ed785431ca47aa3efaa51d0d5dd9c412ab5247151a7c4", - "sha256:dffed17848e8b968d8d3692604e61881aa6ef1f8074c99e81647ac84f6038535", - "sha256:e080087148fd70469aade2abfeadee194357defd759f9b59b349c6192aba994c", - "sha256:e983cbabe10a8989333684c98fdc5dd2f28b236216981e0c26ed359aaa676772", - "sha256:ea6171d2d8d648dee717457d0f75db49ad8c2f13100680e284d7becf3dc311a6", - "sha256:eefc13863bf01583a85e8c1121a901cc7cb8f059b960c4eba30901e2e6aba95f", - "sha256:efd656893171bbf1331beca4ec9f2e74358fc732a2084f664fd149cc4b3441d2" + "sha256:08308c38e44cc926bdfce99498b21eec1f848d24c302519e64203a8da99a97db", + "sha256:09c12096d843b90eafd01ea1b3307e78ddd47a55855ad402b157b6c4862197ce", + "sha256:13d166f77d6dc02c0a73c1101dd87fdf01339febec1030bd810dcd53fff3b0f1", + "sha256:141ec3a3300ab89c7f2b0775289954d193cc8edb621ea05f99db9cb181530512", + "sha256:16c1b388cc31a9baa06d91a19366fb99ddbe1c7b205293ed072211ee5bac1ed2", + "sha256:18bed2bcb39e3f758296584337966e68d2d5ba6aab7e038688ad53c8f889f757", + "sha256:1aeef46a13e51931c0b1cf8ae1168b4a55ecd282e6688fdb0a948cc5a1d5afb9", + "sha256:27d3f3b9e3406579a8af3a9f262f5339005dd25e0ecf3cf1559ff8a49ed5cbf2", + "sha256:2a2740aa9733d2e5b2dfb33639d98a64c3b0f24765fed86b0fd2aec07f6a0a08", + "sha256:4377e10b874e653fe96985c05feed2225c912e328c8a26541f7fc600fb9c637b", + "sha256:448ebb1b3bf64c0267d6b09a7cba26b5ae61b6d2dbabff7c91b660c7eccf2bdb", + "sha256:50e86c076611212ca62e5a59f518edafe0c0730f7d9195fec718da1a5c2bb1fc", + "sha256:5734bdc0342aba9dfc6f04920988140fb41234db42381cf7ccba64169f9fe7ac", + "sha256:64324f64f90a9e4ef732be0928be853eee378fd6a01be21a0a8469c4f2682c83", + "sha256:6ae6c680f3ebf1cf7ad1d7748868b39d9f900836df774c453c11c5440bc15b36", + "sha256:6d7593a705d662be5bfe24111af14763016765f43cb6923ed86223f965f52387", + "sha256:8cac8790a6b1ddf88640a9267ee67b1aee7a57dfa2d2dd33999d080bc8ee3a0f", + "sha256:8ece138c3a16db8c1ad38f52eb32be6086cc72f403150a79336eb2045723a1ad", + "sha256:9eeb7d1d04b117ac0d38719915ae169aa6b61fca227b0b7d198d43728f0c879c", + "sha256:a09f98011236a419ee3f49cedc9ef27d7a1651df07810ae430a6b06576e0b414", + "sha256:a5d897c14513590a85774180be713f692df6fa8ecf6483e561a6d47309566f37", + "sha256:ad6f2ff5b1989a4899bf89800a671d71b1612e5ff40866d1f4d8bcf48d4e5764", + "sha256:c42c4b73121caf0ed6cd795512c9c09c52a7287b04d105d112068c1736d7c753", + "sha256:cb1017eec5257e9ac6209ac172058c430e834d5d2bc21961dceeb79d111e5909", + "sha256:d6c7bb82883680e168b55b49c70af29b84b84abb161cbac2800e8fcb6f2109b6", + "sha256:e452dc66e08a4ce642a961f134814258a082832c78c90351b75c41ad16f79f63", + "sha256:e5b6ed0f0b42317050c88022349d994fe72bfe35f5908617512cd8c8ef9da2a9", + "sha256:e9b30d4bd69498fc0c3fe9db5f62fffbb06b8eb9321f92cc970f2969be5e3949", + "sha256:ec149b90019852266fec2341ce1db513b843e496d5a8e8cdb5ced1923a92faab", + "sha256:edb01671b3caae1ca00881686003d16c2209e07b7ef8b7639f1867852b948f7c", + "sha256:f0d3929fe88ee1c155129ecd82f981b8856c5d97bcb0d5f23e9b4242e79d1de3", + "sha256:f29454410db6ef8126c83bd3c968d143304633d45dc57b51252afbd79d700893", + "sha256:fe45becb4c2f72a0907c1d0246ea6449fe7a9e2293bb0e11c4e9a32bb0930a15", + "sha256:fedbd128668ead37f33917820b704784aff695e0019309ad446a6d0b065b57e4" ], "markers": "python_version >= '3.6'", - "version": "==1.19.3" + "version": "==1.19.4" }, "pathtools": { "hashes": [ @@ -202,9 +436,11 @@ "sha256:0deac2af1a587ae12836aa07970f5cb91964f05a7c6cdb69d8425ff4c15d4e2c", "sha256:0e4dc3d5996760104746e6cfcdb519d9d2cd27c738296525d5867ea695774e67", "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0", + "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6", "sha256:1fabed9ea2acc4efe4671b92c669a213db744d2af8a9fc5d69a8e9bc14b7a9db", "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94", "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52", + "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056", "sha256:6a32f3a4cb2f6e1a0b15215f448e8ce2da192fd4ff35084d80d5e39da683e79b", "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd", "sha256:7d92a09b788cbb1aec325af5fcba9fed7203897bbd9269d5691bb1e3bce29550", @@ -236,6 +472,58 @@ "index": "pypi", "version": "==2.8.6" }, + "pyasn1": { + "hashes": [ + "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", + "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", + "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", + "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", + "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", + "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", + "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", + "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12", + "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", + "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", + "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3" + ], + "version": "==0.4.8" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8", + "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199", + "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811", + "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed", + "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4", + "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", + "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74", + "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb", + "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45", + "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd", + "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0", + "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d", + "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405" + ], + "version": "==0.2.8" + }, + "pycparser": { + "hashes": [ + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.20" + }, + "pyhamcrest": { + "hashes": [ + "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316", + "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29" + ], + "markers": "python_version >= '3.5'", + "version": "==2.0.2" + }, "pyocr": { "hashes": [ "sha256:fa15adc7e1cf0d345a2990495fe125a947c6e09a60ddba0256a1c14b2e603179" @@ -243,6 +531,13 @@ "index": "pypi", "version": "==0.7.2" }, + "pyopenssl": { + "hashes": [ + "sha256:621880965a720b8ece2f1b2f54ea2071966ab00e2970ad2ce11d596102063504", + "sha256:9a24494b2602aaf402be5c9e30a0b82d4a5c67528fe8fb475e3f3bc00dd69507" + ], + "version": "==19.1.0" + }, "python-dateutil": { "hashes": [ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", @@ -276,10 +571,10 @@ }, "pytz": { "hashes": [ - "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", - "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268", + "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd" ], - "version": "==2020.1" + "version": "==2020.4" }, "regex": { "hashes": [ @@ -287,26 +582,42 @@ "sha256:06b52815d4ad38d6524666e0d50fe9173533c9cc145a5779b89733284e6f688f", "sha256:11116d424734fe356d8777f89d625f0df783251ada95d6261b4c36ad27a394bb", "sha256:119e0355dbdd4cf593b17f2fc5dbd4aec2b8899d0057e4957ba92f941f704bf5", + "sha256:127a9e0c0d91af572fbb9e56d00a504dbd4c65e574ddda3d45b55722462210de", "sha256:1ec66700a10e3c75f1f92cbde36cca0d3aaee4c73dfa26699495a3a30b09093c", + "sha256:227a8d2e5282c2b8346e7f68aa759e0331a0b4a890b55a5cfbb28bd0261b84c0", + "sha256:2564def9ce0710d510b1fc7e5178ce2d20f75571f788b5197b3c8134c366f50c", + "sha256:297116e79074ec2a2f885d22db00ce6e88b15f75162c5e8b38f66ea734e73c64", "sha256:2dc522e25e57e88b4980d2bdd334825dbf6fa55f28a922fc3bfa60cc09e5ef53", "sha256:3a5f08039eee9ea195a89e180c5762bfb55258bfb9abb61a20d3abee3b37fd12", + "sha256:3dfca201fa6b326239e1bccb00b915e058707028809b8ecc0cf6819ad233a740", "sha256:49461446b783945597c4076aea3f49aee4b4ce922bd241e4fcf62a3e7c61794c", "sha256:4afa350f162551cf402bfa3cd8302165c8e03e689c897d185f16a167328cc6dd", "sha256:4b5a9bcb56cc146c3932c648603b24514447eafa6ce9295234767bf92f69b504", + "sha256:52e83a5f28acd621ba8e71c2b816f6541af7144b69cc5859d17da76c436a5427", "sha256:625116aca6c4b57c56ea3d70369cacc4d62fead4930f8329d242e4fe7a58ce4b", "sha256:654c1635f2313d0843028487db2191530bca45af61ca85d0b16555c399625b0e", "sha256:8092a5a06ad9a7a247f2a76ace121183dc4e1a84c259cf9c2ce3bbb69fac3582", "sha256:832339223b9ce56b7b15168e691ae654d345ac1635eeb367ade9ecfe0e66bee0", "sha256:8ca9dca965bd86ea3631b975d63b0693566d3cc347e55786d5514988b6f5b84c", + "sha256:96f99219dddb33e235a37283306834700b63170d7bb2a1ee17e41c6d589c8eb9", + "sha256:9b6305295b6591e45f069d3553c54d50cc47629eb5c218aac99e0f7fafbf90a1", "sha256:a62162be05edf64f819925ea88d09d18b09bebf20971b363ce0c24e8b4aa14c0", + "sha256:aacc8623ffe7999a97935eeabbd24b1ae701d08ea8f874a6ff050e93c3e658cf", + "sha256:b45bab9f224de276b7bc916f6306b86283f6aa8afe7ed4133423efb42015a898", "sha256:b88fa3b8a3469f22b4f13d045d9bd3eda797aa4e406fde0a2644bc92bbdd4bdd", + "sha256:b8a686a6c98872007aa41fdbb2e86dc03b287d951ff4a7f1da77fb7f14113e4d", + "sha256:bd904c0dec29bbd0769887a816657491721d5f545c29e30fd9d7a1a275dc80ab", + "sha256:bf4f896c42c63d1f22039ad57de2644c72587756c0cfb3cc3b7530cfe228277f", "sha256:c13d311a4c4a8d671f5860317eb5f09591fbe8259676b86a85769423b544451e", "sha256:c2c6c56ee97485a127555c9595c069201b5161de9d05495fbe2132b5ac104786", + "sha256:c32c91a0f1ac779cbd73e62430de3d3502bbc45ffe5bb6c376015acfa848144b", "sha256:c3466a84fce42c2016113101018a9981804097bacbab029c2d5b4fcb224b89de", + "sha256:c454ad88e56e80e44f824ef8366bb7e4c3def12999151fd5c0ea76a18fe9aa3e", "sha256:c8a2b7ccff330ae4c460aff36626f911f918555660cc28163417cb84ffb25789", "sha256:cb905f3d2e290a8b8f1579d3984f2cfa7c3a29cc7cba608540ceeed18513f520", "sha256:cfcf28ed4ce9ced47b9b9670a4f0d3d3c0e4d4779ad4dadb1ad468b097f808aa", "sha256:dd3e6547ecf842a29cf25123fbf8d2461c53c8d37aa20d87ecee130c89b7079b", + "sha256:de7fd57765398d141949946c84f3590a68cf5887dac3fc52388df0639b01eda4", "sha256:ea37320877d56a7f0a1e6a625d892cf963aa7f570013499f5b8d5ab8402b5625", "sha256:f1fce1e4929157b2afeb4bb7069204d4370bab9f4fc03ca1fbec8bd601f8c87d", "sha256:f43109822df2d3faac7aad79613f5f02e4eab0fc8ad7932d2e70e2a83bd49c26" @@ -337,28 +648,41 @@ }, "scipy": { "hashes": [ - "sha256:07b083128beae040f1129bd8a82b01804f5e716a7fd2962c1053fa683433e4ab", - "sha256:0edd67e8a00903aaf7a29c968555a2e27c5a69fea9d1dcfffda80614281a884f", - "sha256:12fdcbfa56cac926a0a9364a30cbf4ad03c2c7b59f75b14234656a5e4fd52bf3", - "sha256:1fee28b6641ecbff6e80fe7788e50f50c5576157d278fa40f36c851940eb0aff", - "sha256:33e6a7439f43f37d4c1135bc95bcd490ffeac6ef4b374892c7005ce2c729cf4a", - "sha256:5163200ab14fd2b83aba8f0c4ddcc1fa982a43192867264ab0f4c8065fd10d17", - "sha256:66ec29348444ed6e8a14c9adc2de65e74a8fc526dc2c770741725464488ede1f", - "sha256:8cc5c39ed287a8b52a5509cd6680af078a40b0e010e2657eca01ffbfec929468", - "sha256:a1a13858b10d41beb0413c4378462b43eafef88a1948d286cb357eadc0aec024", - "sha256:a3db1fe7c6cb29ca02b14c9141151ebafd11e06ffb6da8ecd330eee5c8283a8a", - "sha256:aebb69bcdec209d874fc4b0c7ac36f509d50418a431c1422465fa34c2c0143ea", - "sha256:b9751b39c52a3fa59312bd2e1f40144ee26b51404db5d2f0d5259c511ff6f614", - "sha256:bc0e63daf43bf052aefbbd6c5424bc03f629d115ece828e87303a0bcc04a37e4", - "sha256:d5e3cc60868f396b78fc881d2c76460febccfe90f6d2f082b9952265c79a8788", - "sha256:ddae76784574cc4c172f3d5edd7308be16078dd3b977e8746860c76c195fa707", - "sha256:e2602f79c85924e4486f684aa9bbab74afff90606100db88d0785a0088be7edb", - "sha256:e527c9221b6494bcd06a17f9f16874406b32121385f9ab353b8a9545be458f0b", - "sha256:f574558f1b774864516f3c3fe072ebc90a29186f49b720f60ed339294b7f32ac", - "sha256:ffcbd331f1ffa82e22f1d408e93c37463c9a83088243158635baec61983aaacf" + "sha256:168c45c0c32e23f613db7c9e4e780bc61982d71dcd406ead746c7c7c2f2004ce", + "sha256:213bc59191da2f479984ad4ec39406bf949a99aba70e9237b916ce7547b6ef42", + "sha256:25b241034215247481f53355e05f9e25462682b13bd9191359075682adcd9554", + "sha256:2c872de0c69ed20fb1a9b9cf6f77298b04a26f0b8720a5457be08be254366c6e", + "sha256:3397c129b479846d7eaa18f999369a24322d008fac0782e7828fa567358c36ce", + "sha256:368c0f69f93186309e1b4beb8e26d51dd6f5010b79264c0f1e9ca00cd92ea8c9", + "sha256:3d5db5d815370c28d938cf9b0809dade4acf7aba57eaf7ef733bfedc9b2474c4", + "sha256:4598cf03136067000855d6b44d7a1f4f46994164bcd450fb2c3d481afc25dd06", + "sha256:4a453d5e5689de62e5d38edf40af3f17560bfd63c9c5bd228c18c1f99afa155b", + "sha256:4f12d13ffbc16e988fa40809cbbd7a8b45bc05ff6ea0ba8e3e41f6f4db3a9e47", + "sha256:634568a3018bc16a83cda28d4f7aed0d803dd5618facb36e977e53b2df868443", + "sha256:65923bc3809524e46fb7eb4d6346552cbb6a1ffc41be748535aa502a2e3d3389", + "sha256:6b0ceb23560f46dd236a8ad4378fc40bad1783e997604ba845e131d6c680963e", + "sha256:8c8d6ca19c8497344b810b0b0344f8375af5f6bb9c98bd42e33f747417ab3f57", + "sha256:9ad4fcddcbf5dc67619379782e6aeef41218a79e17979aaed01ed099876c0e62", + "sha256:a254b98dbcc744c723a838c03b74a8a34c0558c9ac5c86d5561703362231107d", + "sha256:b03c4338d6d3d299e8ca494194c0ae4f611548da59e3c038813f1a43976cb437", + "sha256:cc1f78ebc982cd0602c9a7615d878396bec94908db67d4ecddca864d049112f2", + "sha256:d6d25c41a009e3c6b7e757338948d0076ee1dd1770d1c09ec131f11946883c54", + "sha256:d84cadd7d7998433334c99fa55bcba0d8b4aeff0edb123b2a1dfcface538e474", + "sha256:e360cb2299028d0b0d0f65a5c5e51fc16a335f1603aa2357c25766c8dab56938", + "sha256:e98d49a5717369d8241d6cf33ecb0ca72deee392414118198a8e5b4c35c56340", + "sha256:ed572470af2438b526ea574ff8f05e7f39b44ac37f712105e57fc4d53a6fb660", + "sha256:f87b39f4d69cf7d7529d7b1098cb712033b17ea7714aed831b95628f483fd012", + "sha256:fa789583fc94a7689b45834453fec095245c7e69c58561dc159b5d5277057e4c" ], "markers": "python_version >= '3.6'", - "version": "==1.5.3" + "version": "==1.5.4" + }, + "service-identity": { + "hashes": [ + "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36", + "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d" + ], + "version": "==18.1.0" }, "six": { "hashes": [ @@ -384,6 +708,46 @@ "markers": "python_version >= '3.5'", "version": "==2.1.0" }, + "twisted": { + "extras": [ + "tls" + ], + "hashes": [ + "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f", + "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042", + "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c", + "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292", + "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22", + "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec", + "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478", + "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2", + "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29", + "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114", + "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797", + "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa", + "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15", + "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd", + "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274", + "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad", + "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7", + "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a", + "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10", + "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780", + "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504", + "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467", + "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==20.3.0" + }, + "txaio": { + "hashes": [ + "sha256:17938f2bca4a9cabce61346758e482ca4e600160cbc28e861493eac74a19539d", + "sha256:38a469daf93c37e5527cb062653d6393ae11663147c42fab7ddc3f6d00d434ae" + ], + "markers": "python_version >= '3.5'", + "version": "==20.4.1" + }, "tzlocal": { "hashes": [ "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44", @@ -414,6 +778,64 @@ ], "index": "pypi", "version": "==2.7.4" + }, + "zope.interface": { + "hashes": [ + "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1", + "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d", + "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123", + "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232", + "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549", + "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102", + "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5", + "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45", + "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00", + "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc", + "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7", + "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104", + "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034", + "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3", + "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3", + "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4", + "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86", + "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96", + "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546", + "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb", + "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3", + "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b", + "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b", + "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec", + "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae", + "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e", + "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386", + "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2", + "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a", + "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d", + "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a", + "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24", + "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d", + "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b", + "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50", + "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523", + "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a", + "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095", + "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a", + "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520", + "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65", + "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11", + "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c", + "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7", + "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332", + "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e", + "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c", + "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7", + "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20", + "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc", + "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd", + "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==5.2.0" } }, "develop": { @@ -441,11 +863,11 @@ }, "attrs": { "hashes": [ - "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", - "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.2.0" + "version": "==20.3.0" }, "babel": { "hashes": [ @@ -556,11 +978,11 @@ }, "faker": { "hashes": [ - "sha256:30afa8f564350770373f299d2d267bff42aaba699a7ae0a3b6f378b2a8170569", - "sha256:a7a36c3c657f06bd1e3e3821b9480f2a92017d8a26e150e464ab6b97743cbc92" + "sha256:6afc461ab3f779c9c16e299fc731d775e39ea7e8e063b3053ee359ae198a15ca", + "sha256:ce1c38823eb0f927567cde5bf2e7c8ca565c7a70316139342050ce2ca74b4026" ], "markers": "python_version >= '3.5'", - "version": "==4.14.0" + "version": "==4.14.2" }, "filelock": { "hashes": [ @@ -751,10 +1173,10 @@ }, "pytz": { "hashes": [ - "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", - "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268", + "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd" ], - "version": "==2020.1" + "version": "==2020.4" }, "requests": { "hashes": [ @@ -781,11 +1203,11 @@ }, "sphinx": { "hashes": [ - "sha256:321d6d9b16fa381a5306e5a0b76cd48ffbc588e6340059a729c6fdd66087e0e8", - "sha256:ce6fd7ff5b215af39e2fcd44d4a321f6694b4530b6f2b2109b64d120773faea0" + "sha256:1c21e7c5481a31b531e6cbf59c3292852ccde175b504b00ce2ff0b8f4adc3649", + "sha256:3abdb2c57a65afaaa4f8573cbabd5465078eb6fd282c1e4f87f006875a7ec0c7" ], "index": "pypi", - "version": "==3.2.1" + "version": "==3.3.0" }, "sphinxcontrib-applehelp": { "hashes": [ diff --git a/src/documents/consumer.py b/src/documents/consumer.py index f61d11136..3a73fc0cd 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -5,6 +5,8 @@ import os import re import uuid +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer from django.conf import settings from django.db import transaction from django.utils import timezone @@ -33,6 +35,17 @@ class Consumer: 5. Delete the document and image(s) """ + def _send_progress(self, filename, current_progress, max_progress, status, message, document_id=None): + payload = { + 'filename': os.path.basename(filename), + 'current_progress': current_progress, + 'max_progress': max_progress, + 'status': status, + 'message': message, + 'document_id': document_id + } + async_to_sync(self.channel_layer.group_send)("status_updates", {'type': 'status_update', 'data': payload}) + def __init__(self, consume=settings.CONSUMPTION_DIR, scratch=settings.SCRATCH_DIR): @@ -44,6 +57,8 @@ class Consumer: self.classifier = DocumentClassifier() + self.channel_layer = get_channel_layer() + os.makedirs(self.scratch, exist_ok=True) self.storage_type = Document.STORAGE_TYPE_UNENCRYPTED @@ -60,7 +75,6 @@ class Consumer: raise ConsumerError( "Consumption directory {} does not exist".format(self.consume)) - def log(self, level, message): getattr(self.logger, level)(message, extra={ "group": self.logging_group @@ -88,6 +102,7 @@ class Consumer: self.log("info", "Consuming {}".format(doc)) + parser_class = get_parser_class(doc) if not parser_class: self.log( @@ -96,6 +111,7 @@ class Consumer: else: self.log("info", "Parser: {}".format(parser_class.__name__)) + self._send_progress(file, 0, 100, 'WORKING', 'Consumption started') document_consumption_started.send( sender=self.__class__, @@ -103,20 +119,37 @@ class Consumer: logging_group=self.logging_group ) - document_parser = parser_class(doc, self.logging_group) + def progress_callback(current_progress, max_progress, message): + # recalculate progress to be within 20 and 80 + p = int((current_progress / max_progress) * 60 + 20) + self._send_progress(file, p, 100, "WORKING", message) + + document_parser = parser_class(doc, self.logging_group, progress_callback) try: self.log("info", "Generating thumbnail for {}...".format(doc)) + self._send_progress(file, 10, 100, 'WORKING', + 'Generating thumbnail...') thumbnail = document_parser.get_optimised_thumbnail() + self._send_progress(file, 20, 100, 'WORKING', + 'Getting text from document...') + text = document_parser.get_text() + self._send_progress(file, 80, 100, 'WORKING', + 'Getting date from document...') date = document_parser.get_date() + self._send_progress(file, 85, 100, 'WORKING', + 'Storing the document...') document = self._store( - document_parser.get_text(), + text, doc, thumbnail, date ) except ParseError as e: self.log("fatal", "PARSE FAILURE for {}: {}".format(doc, e)) + self._send_progress(file, 100, 100, 'FAILED', + "Failed: {}".format(e)) + document_parser.cleanup() return False else: @@ -136,12 +169,17 @@ class Consumer: except (FileNotFoundError, IncompatibleClassifierVersionError) as e: logging.getLogger(__name__).warning("Cannot classify documents: {}.".format(e)) + self._send_progress(file, 90, 100, 'WORKING', + 'Performing post-consumption tasks...') + document_consumption_finished.send( sender=self.__class__, document=document, logging_group=self.logging_group, classifier=classifier ) + self._send_progress(file, 100, 100, 'SUCCESS', + 'Finished.', document.id) return True def _store(self, text, doc, thumbnail, date): diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 0cbd13987..7976f4739 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -106,11 +106,12 @@ class DocumentParser: `paperless_tesseract.parsers` for inspiration. """ - def __init__(self, path, logging_group): + def __init__(self, path, logging_group, progress_callback): self.document_path = path self.tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR) self.logger = logging.getLogger(__name__) self.logging_group = logging_group + self.progress_callback = progress_callback def get_thumbnail(self): """ diff --git a/src/paperless/asgi.py b/src/paperless/asgi.py new file mode 100644 index 000000000..9c3d17b1b --- /dev/null +++ b/src/paperless/asgi.py @@ -0,0 +1,37 @@ +import json +import os + +from asgiref.sync import async_to_sync +from channels.auth import AuthMiddlewareStack +from channels.generic.websocket import WebsocketConsumer +from channels.routing import ProtocolTypeRouter, URLRouter +from django.core.asgi import get_asgi_application +from django.urls import re_path + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'paperless.settings') + + +class StatusConsumer(WebsocketConsumer): + def connect(self): + self.accept() + async_to_sync(self.channel_layer.group_add)('status_updates', self.channel_name) + + def disconnect(self, close_code): + async_to_sync(self.channel_layer.group_discard)('status_updates', self.channel_name) + + def status_update(self, event): + self.send(json.dumps(event['data'])) + + +websocket_urlpatterns = [ + re_path(r'ws/status/$', StatusConsumer.as_asgi()), +] + +application = ProtocolTypeRouter({ + "http": get_asgi_application(), + "websocket": AuthMiddlewareStack( + URLRouter( + websocket_urlpatterns + ) + ), +}) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index bb71e4764..c9db3e4b1 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -69,6 +69,8 @@ INSTALLED_APPS = [ "rest_framework.authtoken", "django_filters", + "channels", + ] REST_FRAMEWORK = { @@ -98,6 +100,7 @@ LOGIN_URL = "admin:login" FORCE_SCRIPT_NAME = os.getenv("PAPERLESS_FORCE_SCRIPT_NAME") WSGI_APPLICATION = 'paperless.wsgi.application' +ASGI_APPLICATION = "paperless.asgi.application" STATIC_URL = os.getenv("PAPERLESS_STATIC_URL", "/static/") @@ -299,3 +302,12 @@ FILENAME_DATE_ORDER = os.getenv("PAPERLESS_FILENAME_DATE_ORDER") FILENAME_PARSE_TRANSFORMS = [] for t in json.loads(os.getenv("PAPERLESS_FILENAME_PARSE_TRANSFORMS", "[]")): FILENAME_PARSE_TRANSFORMS.append((re.compile(t["pattern"]), t["repl"])) + +CHANNEL_LAYERS = { + "default": { + "BACKEND": "channels_redis.core.RedisChannelLayer", + "CONFIG": { + "hosts": [("127.0.0.1", 6379)], + }, + }, +} diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index befc9bcd7..535ee501c 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -27,8 +27,8 @@ class RasterisedDocumentParser(DocumentParser): image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.) """ - def __init__(self, path, logging_group): - super().__init__(path, logging_group) + def __init__(self, path, logging_group, progress_callback): + super().__init__(path, logging_group, progress_callback) self._text = None def get_thumbnail(self): @@ -91,6 +91,7 @@ class RasterisedDocumentParser(DocumentParser): self._text = get_text_from_pdf(self.document_path) return self._text + self.progress_callback(0,1,"Making greyscale images.") images = self._get_greyscale() if not images: @@ -100,8 +101,10 @@ class RasterisedDocumentParser(DocumentParser): sample_page_index = int(len(images) / 2) self.log("info", "Attempting language detection on page {} of {}...".format(sample_page_index+1, len(images))) + self.progress_callback(0.4, 1, "Language Detection.") sample_page_text = self._ocr([images[sample_page_index]], settings.OCR_LANGUAGE)[0] guessed_language = self._guess_language(sample_page_text) + self.progress_callback(0.6, 1, "OCR all the pages.") if not guessed_language or guessed_language not in ISO639: self.log("warning", "Language detection failed.") @@ -117,7 +120,7 @@ class RasterisedDocumentParser(DocumentParser): else: self.log("info", "Detected language: {}".format(guessed_language)) - ocr_pages = self._ocr(images, ISO639[guessed_language]) + ocr_pages = self._ocr(images, ISO639[guessed_language], report_progress=True) self.log("info", "OCR completed.") self._text = strip_excess_whitespace(" ".join(ocr_pages)) @@ -151,6 +154,8 @@ class RasterisedDocumentParser(DocumentParser): self.log("info", "Running unpaper on {} pages...".format(len(pnms))) + self.progress_callback(0.2,1, "Running unpaper on {} pages...".format(len(pnms))) + # Run unpaper in parallel on converted images with Pool(processes=settings.OCR_THREADS) as pool: pnms = pool.map(run_unpaper, pnms) @@ -165,11 +170,16 @@ class RasterisedDocumentParser(DocumentParser): self.log('debug', "Language detection failed with: {}".format(e)) return None - def _ocr(self, imgs, lang): + def _ocr(self, imgs, lang, report_progress=False): self.log("info", "Performing OCR on {} page(s) with language {}".format(len(imgs), lang)) + r = [] with Pool(processes=settings.OCR_THREADS) as pool: - r = pool.map(image_to_string, itertools.product(imgs, [lang])) - return r + # r = pool.map(image_to_string, itertools.product(imgs, [lang])) + for i, page in enumerate(pool.imap(image_to_string, itertools.product(imgs, [lang]))): + if report_progress: + self.progress_callback(0.6 + (i / len(imgs)) * 0.4, 1, "OCR'ed {} pages".format(i+1)) + r += [page] + return r def _complete_ocr_default_language(self, images, sample_page_index, sample_page): """ @@ -182,14 +192,13 @@ class RasterisedDocumentParser(DocumentParser): del images_copy[sample_page_index] if images_copy: self.log('info', 'Continuing ocr with default language.') - ocr_pages = self._ocr(images_copy, settings.OCR_LANGUAGE) + ocr_pages = self._ocr(images_copy, settings.OCR_LANGUAGE, report_progress=True) ocr_pages.insert(sample_page_index, sample_page) return ocr_pages else: return [sample_page] - def strip_excess_whitespace(text): collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text) no_leading_whitespace = re.sub(