diff --git a/.github/scripts/cleanup-tags.py b/.github/scripts/cleanup-tags.py index ce2a32d27..bf8b119b8 100644 --- a/.github/scripts/cleanup-tags.py +++ b/.github/scripts/cleanup-tags.py @@ -200,7 +200,6 @@ class RegistryTagsCleaner: tag, ) for manifest in image_index.image_pointers: - if manifest.digest in untagged_versions: logger.info( f"Skipping deletion of {manifest.digest}," @@ -287,7 +286,6 @@ class RegistryTagsCleaner: logger.info("Beginning confirmation step") a_tag_failed = False for tag in sorted(self.tags_to_keep): - try: image_index = ImageIndex( f"ghcr.io/{self.repo_owner}/{self.package_name}", @@ -301,7 +299,6 @@ class RegistryTagsCleaner: digest_name = f"ghcr.io/{self.repo_owner}/{self.package_name}@{manifest.digest}" try: - subprocess.run( [ shutil.which("docker"), diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86aed9973..ceb476d6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,16 +37,16 @@ repos: exclude: "(^Pipfile\\.lock$)" # Python hooks - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.259' + rev: 'v0.0.263' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.3.0 hooks: - id: black # Dockerfile hooks - repo: https://github.com/AleksaC/hadolint-py - rev: v2.10.0 + rev: v2.12.0.2 hooks: - id: hadolint # Shell script hooks diff --git a/.ruff.toml b/.ruff.toml index 030c02b0a..0a20a7a2a 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,6 +1,6 @@ # https://beta.ruff.rs/docs/settings/ # https://beta.ruff.rs/docs/rules/ -select = ["F", "E", "W", "UP", "COM", "DJ", "EXE", "ISC", "ICN", "G201", "INP", "PIE", "RSE", "SIM", "TID", "PLC", "PLE", "RUF"] +extend-select = ["I", "W", "UP", "COM", "DJ", "EXE", "ISC", "ICN", "G201", "INP", "PIE", "RSE", "SIM", "TID", "PLC", "PLE", "RUF"] # TODO PTH ignore = ["DJ001", "SIM105"] fix = true diff --git a/Pipfile.lock b/Pipfile.lock index 4e738d14a..3637bbe50 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8395f25f876a71a7307a55dd542e69a4cdcb3be3be38c4e89ed06ce3d52a5345" + "sha256": "77248fee6dad10b9e5189e9ba80f7c506c9f49c875bac8b259e90dadecba03f1" }, "pipfile-spec": 6, "requires": {}, @@ -1508,11 +1508,11 @@ "hiredis" ], "hashes": [ - "sha256:56732e156fe31801c4f43396bd3ca0c2a7f6f83d7936798531b9848d103381aa", - "sha256:7df17a0a2b72a4c8895b462dd07616c51b1dcb48fdd7ecb7b6f4bf39ecb2e94e" + "sha256:2c19e6767c474f2e85167909061d525ed65bea9301c0770bb151e041b7ac89a2", + "sha256:73ec35da4da267d6847e47f68730fdd5f62e2ca69e3ef5885c6a78a9374c3893" ], "index": "pypi", - "version": "==4.5.3" + "version": "==4.5.4" }, "regex": { "hashes": [ @@ -2256,34 +2256,35 @@ }, "black": { "hashes": [ - "sha256:0052dba51dec07ed029ed61b18183942043e00008ec65d5028814afaab9a22fd", - "sha256:0680d4380db3719ebcfb2613f34e86c8e6d15ffeabcf8ec59355c5e7b85bb555", - "sha256:121ca7f10b4a01fd99951234abdbd97728e1240be89fde18480ffac16503d481", - "sha256:162e37d49e93bd6eb6f1afc3e17a3d23a823042530c37c3c42eeeaf026f38468", - "sha256:2a951cc83ab535d248c89f300eccbd625e80ab880fbcfb5ac8afb5f01a258ac9", - "sha256:2bf649fda611c8550ca9d7592b69f0637218c2369b7744694c5e4902873b2f3a", - "sha256:382998821f58e5c8238d3166c492139573325287820963d2f7de4d518bd76958", - "sha256:49f7b39e30f326a34b5c9a4213213a6b221d7ae9d58ec70df1c4a307cf2a1580", - "sha256:57c18c5165c1dbe291d5306e53fb3988122890e57bd9b3dcb75f967f13411a26", - "sha256:7a0f701d314cfa0896b9001df70a530eb2472babb76086344e688829efd97d32", - "sha256:8178318cb74f98bc571eef19068f6ab5613b3e59d4f47771582f04e175570ed8", - "sha256:8b70eb40a78dfac24842458476135f9b99ab952dd3f2dab738c1881a9b38b753", - "sha256:9880d7d419bb7e709b37e28deb5e68a49227713b623c72b2b931028ea65f619b", - "sha256:9afd3f493666a0cd8f8df9a0200c6359ac53940cbde049dcb1a7eb6ee2dd7074", - "sha256:a29650759a6a0944e7cca036674655c2f0f63806ddecc45ed40b7b8aa314b651", - "sha256:a436e7881d33acaf2536c46a454bb964a50eff59b21b51c6ccf5a40601fbef24", - "sha256:a59db0a2094d2259c554676403fa2fac3473ccf1354c1c63eccf7ae65aac8ab6", - "sha256:a8471939da5e824b891b25751955be52ee7f8a30a916d570a5ba8e0f2eb2ecad", - "sha256:b0bd97bea8903f5a2ba7219257a44e3f1f9d00073d6cc1add68f0beec69692ac", - "sha256:b6a92a41ee34b883b359998f0c8e6eb8e99803aa8bf3123bf2b2e6fec505a221", - "sha256:bb460c8561c8c1bec7824ecbc3ce085eb50005883a6203dcfb0122e95797ee06", - "sha256:bfffba28dc52a58f04492181392ee380e95262af14ee01d4bc7bb1b1c6ca8d27", - "sha256:c1c476bc7b7d021321e7d93dc2cbd78ce103b84d5a4cf97ed535fbc0d6660648", - "sha256:c91dfc2c2a4e50df0026f88d2215e166616e0c80e86004d0003ece0488db2739", - "sha256:e6663f91b6feca5d06f2ccd49a10f254f9298cc1f7f49c46e498a0771b507104" + "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5", + "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915", + "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326", + "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940", + "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b", + "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30", + "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c", + "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c", + "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab", + "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27", + "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2", + "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961", + "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9", + "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb", + "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70", + "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331", + "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2", + "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266", + "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d", + "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6", + "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b", + "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925", + "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8", + "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4", + "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3" ], "index": "pypi", - "version": "==23.1.0" + "markers": "python_version >= '3.7'", + "version": "==23.3.0" }, "certifi": { "hashes": [ @@ -2718,11 +2719,11 @@ }, "packaging": { "hashes": [ - "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2", - "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97" + "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", + "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" ], "markers": "python_version >= '3.7'", - "version": "==23.0" + "version": "==23.1" }, "pathspec": { "hashes": [ @@ -2817,11 +2818,11 @@ }, "platformdirs": { "hashes": [ - "sha256:024996549ee88ec1a9aa99ff7f8fc819bb59e2c3477b410d90a16d32d6e707aa", - "sha256:e5986afb596e4bb5bde29a79ac9061aa955b94fca2399b7aaac4090860920dd8" + "sha256:64370d47dc3fca65b4879f89bdead8197e93e05d696d6d1816243ebae8595da5", + "sha256:ea61fd7b85554beecbbd3e9b37fb26689b227ffae38f73353cbcc1cf8bd01878" ], "markers": "python_version >= '3.7'", - "version": "==3.1.1" + "version": "==3.3.0" }, "pluggy": { "hashes": [ @@ -3071,26 +3072,27 @@ }, "ruff": { "hashes": [ - "sha256:22e1e35bf5f12072cd644d22afd9203641ccf258bc14ff91aa1c43dc14f6047d", - "sha256:29e2b77b7d5da6a7dd5cf9b738b511355c5734ece56f78e500d4b5bffd58c1a0", - "sha256:38704f151323aa5858370a2f792e122cc25e5d1aabe7d42ceeab83da18f0b456", - "sha256:40ae87f2638484b7e8a7567b04a7af719f1c484c5bf132038b702bb32e1f6577", - "sha256:428507fb321b386dda70d66cd1a8aa0abf51d7c197983d83bb9e4fa5ee60300b", - "sha256:49e903bcda19f6bb0725a962c058eb5d61f40d84ef52ed53b61939b69402ab4e", - "sha256:5b3c1beacf6037e7f0781d4699d9a2dd4ba2462f475be5b1f45cf84c4ba3c69d", - "sha256:71f0ef1985e9a6696fa97da8459917fa34bdaa2c16bd33bd5edead585b7d44f7", - "sha256:79b02fa17ec1fd8d306ae302cb47fb614b71e1f539997858243769bcbe78c6d9", - "sha256:7cfef26619cba184d59aa7fa17b48af5891d51fc0b755a9bc533478a10d4d066", - "sha256:8b56496063ab3bfdf72339a5fbebb8bd46e5c5fee25ef11a9f03b208fa0562ec", - "sha256:aa9449b898287e621942cc71b9327eceb8f0c357e4065fecefb707ef2d978df8", - "sha256:c5fbaea9167f1852757f02133e5daacdb8c75b3431343205395da5b10499927a", - "sha256:d2fb20e89e85d147c85caa807707a1488bccc1f3854dc3d53533e89b52a0c5ff", - "sha256:daaea322e7e85f4c13d82be9536309e1c4b8b9851bb0cbc7eeb15d490fd46bf9", - "sha256:e4f39e18702de69faaaee3969934b92d7467285627f99a5b6ecd55a7d9f5d086", - "sha256:f3938dc45e2a3f818e9cbd53007265c22246fbfded8837b2c563bf0ebde1a226" + "sha256:04e0b280dd246448564c892bce5607d820ad1f14944f3d535db98692e2a7ac07", + "sha256:1008f211ad8aa1d998517ac5bf3d68fbc68ec516d1da89b6081f25ff2f30b687", + "sha256:15386933dd8e03aafa3186f9e996d6823105492817311338fbcb64d0ecbcd95f", + "sha256:3e9fcee3f81129eabc75da005d839235e32d7d374f2d4c0db0c708dad4703d6e", + "sha256:4010b156f2e9fa6e74b5581098467f6ff68beac48945599b3a9239481e578ab4", + "sha256:4f75fa1632ea065b8f10678e7b6ae9873f84d5046bdf146990112751e98af42a", + "sha256:7890499c2c3dcb1e60de2a8b4c5f5775b2bfcdff7d3e68e38db5cb2d65b12006", + "sha256:82c41f276106017b6f075dd2f2cc68e1a0b434cc75488f816fc98bd41982628d", + "sha256:981e3c4d773f7ff52479c4fd74a65e408f1e13fa5f889b72214d400cd1299ce4", + "sha256:9af932f665e177de62e172901704257fd6e5bfabb95893867ff7382a851459d3", + "sha256:bed1d3fba306e3f7e13ce226927b84200350e25abd1e754e06ee361c6d41de15", + "sha256:c2b79919ebd93674b93dfc2c843e264bf8e52fbe737467e9b58521775c85f4ad", + "sha256:c3b7d4b365207f3e4c40d235127091478e595b31e35b6cd57d940920cdfae68b", + "sha256:ddcee0d91629a4fa4bc9faebf5b94d4615d50d1cd76d1098fa71fbe1c54f4104", + "sha256:ddf4503595b560bfa5fae92fa2e4cb09ec465ee4cf88cc248f10ad2e956deec3", + "sha256:ebc778d95f29c9917e6e7608b2b67815707e6ab8eb5af9341617beda479c3edf", + "sha256:ee6c7a77f142c427fa73e1f5f603fc1a39413a36fe6966ed0fc55e97f6921d9c" ], "index": "pypi", - "version": "==0.0.259" + "markers": "python_version >= '3.7'", + "version": "==0.0.263" }, "scipy": { "hashes": [ @@ -3158,7 +3160,7 @@ "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" ], - "markers": "python_version >= '3.7'", + "markers": "python_version < '3.10'", "version": "==4.5.0" }, "urllib3": { diff --git a/docker/wait-for-redis.py b/docker/wait-for-redis.py index cabfb1dc6..d6ce5c639 100755 --- a/docker/wait-for-redis.py +++ b/docker/wait-for-redis.py @@ -12,7 +12,6 @@ from typing import Final from redis import Redis if __name__ == "__main__": - MAX_RETRY_COUNT: Final[int] = 5 RETRY_SLEEP_SECONDS: Final[int] = 5 diff --git a/docs/usage.md b/docs/usage.md index 14adef26b..04449177f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -204,7 +204,7 @@ for details. ## Permissions -As of version 1.13.0 Paperless-ngx added core support for user / group permissions. Permissions is +As of version 1.14.0 Paperless-ngx added core support for user / group permissions. Permissions is based around an object 'owner' and 'view' and 'edit' permissions can be granted to other users or groups. @@ -212,13 +212,13 @@ Permissions uses the built-in user model of the backend framework, Django. !!! note - After migration to version 1.13.0 all existing documents, tags etc. will have no explicit owner + After migration to version 1.14.0 all existing documents, tags etc. will have no explicit owner set which means they will be visible / editable by all users. Once an object has an owner set, only the owner can explicitly grant / revoke permissions. !!! note - When first migrating to permissions it is recommended to user a 'superuser' account (which + When first migrating to permissions it is recommended to use a 'superuser' account (which would usually have been setup during installation) to ensure you have full permissions. Note that superusers have access to all objects. @@ -230,7 +230,7 @@ do not have an owner set. ### Users and Groups -Paperless-ngx versions after 1.13.0 allow creating and editing users and groups via the 'frontend' UI. +Paperless-ngx versions after 1.14.0 allow creating and editing users and groups via the 'frontend' UI. These can be found under Settings > Users & Groups, assuming the user has access. If a user is designated as a member of a group those permissions will be inherited and this is reflected in the UI. Explicit permissions can be granted to limit access to certain parts of the UI (and corresponding API endpoints). diff --git a/src-ui/cypress/e2e/documents/document-detail.cy.ts b/src-ui/cypress/e2e/documents/document-detail.cy.ts index dd5f8fac8..51d043ce3 100644 --- a/src-ui/cypress/e2e/documents/document-detail.cy.ts +++ b/src-ui/cypress/e2e/documents/document-detail.cy.ts @@ -5,11 +5,15 @@ describe('document-detail', () => { this.modifiedDocuments = [] cy.fixture('documents/documents.json').then((documentsJson) => { - cy.intercept('GET', 'http://localhost:8000/api/documents/1/', (req) => { - let response = { ...documentsJson } - response = response.results.find((d) => d.id == 1) - req.reply(response) - }) + cy.intercept( + 'GET', + 'http://localhost:8000/api/documents/1/?full_perms=true', + (req) => { + let response = { ...documentsJson } + response = response.results.find((d) => d.id == 1) + req.reply(response) + } + ) }) cy.intercept('PUT', 'http://localhost:8000/api/documents/1/', (req) => { diff --git a/src-ui/cypress/fixtures/documents/documents.json b/src-ui/cypress/fixtures/documents/documents.json index e3938dba1..6b284f7b2 100644 --- a/src-ui/cypress/fixtures/documents/documents.json +++ b/src-ui/cypress/fixtures/documents/documents.json @@ -21,6 +21,7 @@ "original_file_name": "2022-03-22 no latin title.pdf", "archived_file_name": "2022-03-22 no latin title.pdf", "owner": null, + "user_can_change": true, "permissions": { "view": { "users": [], @@ -68,6 +69,7 @@ "original_file_name": "2022-03-23 lorem ipsum dolor sit amet.pdf", "archived_file_name": "2022-03-23 llorem ipsum dolor sit amet.pdf", "owner": null, + "user_can_change": true, "permissions": { "view": { "users": [], @@ -98,6 +100,7 @@ "original_file_name": "2022-03-24 dolor.pdf", "archived_file_name": "2022-03-24 dolor.pdf", "owner": null, + "user_can_change": true, "permissions": { "view": { "users": [], @@ -128,6 +131,7 @@ "original_file_name": "2022-06-01 sit amet.pdf", "archived_file_name": "2022-06-01 sit amet.pdf", "owner": null, + "user_can_change": true, "permissions": { "view": { "users": [], diff --git a/src-ui/src/app/components/common/input/select/select.component.ts b/src-ui/src/app/components/common/input/select/select.component.ts index 3d81f6f49..73f2a3a8d 100644 --- a/src-ui/src/app/components/common/input/select/select.component.ts +++ b/src-ui/src/app/components/common/input/select/select.component.ts @@ -43,8 +43,8 @@ export class SelectComponent extends AbstractInputComponent { } checkForPrivateItems(value: any) { - if (Array.isArray(value) && value.length > 0) { - value.forEach((id) => this.checkForPrivateItem(id)) + if (Array.isArray(value)) { + if (value.length > 0) value.forEach((id) => this.checkForPrivateItem(id)) } else { this.checkForPrivateItem(value) } diff --git a/src-ui/src/app/components/manage/management-list/management-list.component.ts b/src-ui/src/app/components/manage/management-list/management-list.component.ts index ec169fdd0..437acef90 100644 --- a/src-ui/src/app/components/manage/management-list/management-list.component.ts +++ b/src-ui/src/app/components/manage/management-list/management-list.component.ts @@ -122,7 +122,8 @@ export abstract class ManagementListComponent null, this.sortField, this.sortReverse, - this._nameFilter + this._nameFilter, + true ) .subscribe((c) => { this.data = c.results diff --git a/src-ui/src/app/data/object-with-permissions.ts b/src-ui/src/app/data/object-with-permissions.ts index 86f5f7394..9346aa85c 100644 --- a/src-ui/src/app/data/object-with-permissions.ts +++ b/src-ui/src/app/data/object-with-permissions.ts @@ -16,4 +16,6 @@ export interface ObjectWithPermissions extends ObjectWithId { owner?: number permissions?: PermissionsObject + + user_can_change?: boolean } diff --git a/src-ui/src/app/services/permissions.service.ts b/src-ui/src/app/services/permissions.service.ts index c19a4ee94..1e7b0d031 100644 --- a/src-ui/src/app/services/permissions.service.ts +++ b/src-ui/src/app/services/permissions.service.ts @@ -58,17 +58,24 @@ export class PermissionsService { action: string, object: ObjectWithPermissions ): boolean { - let actionObject = null - if (action === PermissionAction.View) actionObject = object.permissions.view - else if (action === PermissionAction.Change) - actionObject = object.permissions.change - if (!actionObject) return false - return ( - this.currentUserOwnsObject(object) || - actionObject.users.includes(this.currentUser.id) || - actionObject.groups.filter((g) => this.currentUser.groups.includes(g)) - .length > 0 - ) + if (action === PermissionAction.View) { + return ( + this.currentUserOwnsObject(object) || + object.permissions?.view.users.includes(this.currentUser.id) || + object.permissions?.view.groups.filter((g) => + this.currentUser.groups.includes(g) + ).length > 0 + ) + } else if (action === PermissionAction.Change) { + return ( + this.currentUserOwnsObject(object) || + object.user_can_change || + object.permissions?.change.users.includes(this.currentUser.id) || + object.permissions?.change.groups.filter((g) => + this.currentUser.groups.includes(g) + ).length > 0 + ) + } } public getPermissionCode( diff --git a/src-ui/src/app/services/rest/abstract-name-filter-service.ts b/src-ui/src/app/services/rest/abstract-name-filter-service.ts index 568803fb8..0b5fca835 100644 --- a/src-ui/src/app/services/rest/abstract-name-filter-service.ts +++ b/src-ui/src/app/services/rest/abstract-name-filter-service.ts @@ -9,11 +9,15 @@ export abstract class AbstractNameFilterService< pageSize?: number, sortField?: string, sortReverse?: boolean, - nameFilter?: string + nameFilter?: string, + fullPerms?: boolean ) { let params = {} if (nameFilter) { - params = { name__icontains: nameFilter } + params['name__icontains'] = nameFilter + } + if (fullPerms) { + params['full_perms'] = true } return this.list(page, pageSize, sortField, sortReverse, params) } diff --git a/src-ui/src/app/services/rest/document.service.ts b/src-ui/src/app/services/rest/document.service.ts index 63b447b9a..4ff2ee88f 100644 --- a/src-ui/src/app/services/rest/document.service.ts +++ b/src-ui/src/app/services/rest/document.service.ts @@ -113,6 +113,14 @@ export class DocumentService extends AbstractPaperlessService }).pipe(map((response) => response.results.map((doc) => doc.id))) } + get(id: number): Observable { + return this.http.get(this.getResourceUrl(id), { + params: { + full_perms: true, + }, + }) + } + getPreviewUrl(id: number, original: boolean = false): string { let url = this.getResourceUrl(id, 'preview') if (this._searchQuery) url += `#search="${this._searchQuery}"` diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index 46e74923c..228852877 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -5,7 +5,7 @@ export const environment = { apiBaseUrl: document.baseURI + 'api/', apiVersion: '2', appTitle: 'Paperless-ngx', - version: '1.14.0', + version: '1.14.0-dev', webSocketHost: window.location.host, webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:', webSocketBaseUrl: base_url.pathname + 'ws/', diff --git a/src-ui/src/locale/messages.ca_ES.xlf b/src-ui/src/locale/messages.ca_ES.xlf index c22fb0460..d321c0101 100644 --- a/src-ui/src/locale/messages.ca_ES.xlf +++ b/src-ui/src/locale/messages.ca_ES.xlf @@ -625,7 +625,7 @@ src/app/components/app-frame/app-frame.component.html 134 - Corresponsal és + Corresponsals Tags @@ -4186,7 +4186,7 @@ src/app/components/manage/management-list/management-list.component.html 2 - Creat + Crear Filter by: diff --git a/src-ui/src/locale/messages.de_DE.xlf b/src-ui/src/locale/messages.de_DE.xlf index 1b66fe8ef..ee517488a 100644 --- a/src-ui/src/locale/messages.de_DE.xlf +++ b/src-ui/src/locale/messages.de_DE.xlf @@ -797,7 +797,7 @@ src/app/components/app-frame/app-frame.component.html 235 - Aktualisierung verfügbar + Update verfügbar An error occurred while saving settings. @@ -3076,7 +3076,7 @@ src/app/components/document-detail/document-detail.component.ts 595,597 - Error deleting document: + Fehler beim Löschen des Dokuments Redo OCR confirm @@ -4578,7 +4578,7 @@ src/app/components/manage/settings/settings.component.html 140,142 - Update checking works by pinging the public Github API for the latest release to determine whether a new version is available. Actual updating of the app must still be performed manually. + Die Überprüfung auf Updates erfolgt über Anfragen an die öffentliche Github API, um zu ermitteln, ob eine neue Version verfügbar ist. Das eigentliche Update der Anwendung muss weiterhin manuell durchgeführt werden. No tracking data is collected by the app in any way. @@ -5671,7 +5671,7 @@ src/app/services/settings.service.ts 177 - Catalan + Katalanisch Czech diff --git a/src-ui/src/locale/messages.fi_FI.xlf b/src-ui/src/locale/messages.fi_FI.xlf index 40ddb480c..e5bdf0304 100644 --- a/src-ui/src/locale/messages.fi_FI.xlf +++ b/src-ui/src/locale/messages.fi_FI.xlf @@ -3076,7 +3076,7 @@ src/app/components/document-detail/document-detail.component.ts 595,597 - Error deleting document: + Virhe poistettaessa asiakirjaa: Redo OCR confirm @@ -4578,7 +4578,7 @@ src/app/components/manage/settings/settings.component.html 140,142 - Update checking works by pinging the public Github API for the latest release to determine whether a new version is available. Actual updating of the app must still be performed manually. + Päivityksen tarkistaminen tapahtuu yhteydellä Github API-palveluun viimeisimmän version tarkistamiseksi. Sovelluksen varsinainen päivitys on silti suoritettava manuaalisesti. No tracking data is collected by the app in any way. @@ -5671,7 +5671,7 @@ src/app/services/settings.service.ts 177 - Catalan + Katalaani Czech diff --git a/src-ui/src/locale/messages.fr_FR.xlf b/src-ui/src/locale/messages.fr_FR.xlf index 9350acaca..af41bae11 100644 --- a/src-ui/src/locale/messages.fr_FR.xlf +++ b/src-ui/src/locale/messages.fr_FR.xlf @@ -1327,21 +1327,21 @@ Mot de passe - + Password is token src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.html 18 - Le mot de passe est le jeton d'authentification + Le mot de passe est un jeton d'authentification - + Check if the password above is a token used for authentication src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.html 18 - Vérifier si le mot de passe ci-dessus est un jeton utilisé pour l'authentification + Cocher si le mot de passe ci-dessus est un jeton utilisé pour l'authentification Character Set @@ -1395,13 +1395,13 @@ Chargement ... - + Test src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.html 32 - Test + Tester No encryption @@ -1443,21 +1443,21 @@ Éditer un compte de messagerie - + Successfully connected to the mail server src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.ts 88 - Connexion réussie au serveur de courrier + Connexion réussie au serveur de messagerie électronique - + Unable to connect to the mail server src/app/components/common/edit-dialog/mail-account-edit-dialog/mail-account-edit-dialog.component.ts 89 - Impossible de se connecter au serveur d'impression + Impossible de se connecter au serveur de messagerie électronique Account @@ -2063,21 +2063,21 @@ Tous - + Include src/app/components/common/filterable-dropdown/filterable-dropdown.component.html 24 - Inclure + Inclure - + Exclude src/app/components/common/filterable-dropdown/filterable-dropdown.component.html 26 - Exclure + Exclure Apply @@ -2273,7 +2273,7 @@ Used for both types, correspondents, storage paths Ajouter un élément - + Private src/app/components/common/input/select/select.component.ts @@ -2287,7 +2287,7 @@ src/app/components/common/tag/tag.component.html 8 - Privé + Privé Add tag @@ -2351,13 +2351,13 @@ - + Inherited from group src/app/components/common/permissions-select/permissions-select.component.ts 62 - Hérité du groupe + Hérité du groupe Select @@ -2467,53 +2467,53 @@ Statistiques - + Go to inbox src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html 4 - Accéder à la boîte de réception + Accéder à la boîte de réception - + Documents in inbox src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html 5 - Documents dans la boîte de réception + Documents dans la boîte de réception - + Go to documents src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html 8 - Aller aux documents + Aller aux documents - + Total documents src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html 9 - Nombre total de documents + Nombre total de documents - + Total characters src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html 13 - Nombre total de caractères + Nombre total de caractères - + Other src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.ts 55 - Autres + Autres Upload new documents @@ -2962,13 +2962,13 @@ Saisir le mot de passe - + Notes src/app/components/document-detail/document-detail.component.html 175,176 - Notes + Notes Discard @@ -2986,13 +2986,13 @@ Enregistrer & suivant - + An error occurred loading content: src/app/components/document-detail/document-detail.component.ts 226,228 - Une erreur s'est produite lors du chargement du contenu : + Une erreur s'est produite lors du chargement du contenu : Error retrieving metadata @@ -3010,7 +3010,7 @@ Erreur lors de la récupération des suggestions - + Document saved successfully. src/app/components/document-detail/document-detail.component.ts @@ -3020,7 +3020,7 @@ src/app/components/document-detail/document-detail.component.ts 492 - Document enregistré avec succès. + Document enregistré avec succès. Error saving document @@ -3070,13 +3070,13 @@ Supprimer le document - + Error deleting document: src/app/components/document-detail/document-detail.component.ts 595,597 - Error deleting document: + Erreur lors de la suppression du document : Redo OCR confirm @@ -3532,21 +3532,21 @@ Filtrer par étiquette - + View notes src/app/components/document-list/document-card-large/document-card-large.component.html 70 - Afficher les Notes + Afficher les notes - + Notes src/app/components/document-list/document-card-large/document-card-large.component.html 74 - Notes + Notes Filter by document type @@ -3740,13 +3740,13 @@ Erreur lors du téléchargement du document - + Sort by ASN src/app/components/document-list/document-list.component.html 126 - Trier par ASN + Trier par ASN ASN @@ -3764,31 +3764,31 @@ NSA - + Sort by correspondent src/app/components/document-list/document-list.component.html 133 - Trier par correspondant + Trier par correspondant - + Sort by title src/app/components/document-list/document-list.component.html 140 - Trier par titre + Trier par titre - + Sort by notes src/app/components/document-list/document-list.component.html 147 - Trier par notes + Trier par notes - + Notes src/app/components/document-list/document-list.component.html @@ -3802,39 +3802,39 @@ src/app/services/rest/document.service.ts 25 - Notes + Notes - + Sort by document type src/app/components/document-list/document-list.component.html 154 - Trier par type de documents + Trier par type de documents - + Sort by storage path src/app/components/document-list/document-list.component.html 161 - Trier par chemin de stockage + Trier par chemin de stockage - + Sort by created date src/app/components/document-list/document-list.component.html 168 - Trier par la date de création + Trier par date de création - + Sort by added date src/app/components/document-list/document-list.component.html 175 - Trier par date d'ajout + Trier par date d'ajout Added @@ -4060,31 +4060,31 @@ L'erreur renvoyée était - + Enter note src/app/components/document-notes/document-notes.component.html 4 - Saisir des notes + Saisir une note - + Please enter a note. src/app/components/document-notes/document-notes.component.html 5,7 - Veuillez saisir une note. + Veuillez saisir une note. - + Add note src/app/components/document-notes/document-notes.component.html 11 - Ajouter une note + Ajouter une note - + Delete note src/app/components/document-notes/document-notes.component.html @@ -4094,23 +4094,23 @@ src/app/components/document-notes/document-notes.component.html 25 - Supprimer une note + Supprimer une note - + Error saving note: src/app/components/document-notes/document-notes.component.ts 65 - Erreur lors de l'enregistrement de la note: + Erreur lors de l'enregistrement de la note : - + Error deleting note: src/app/components/document-notes/document-notes.component.ts 81 - Erreur lors de l'enregistrement de la note: + Une erreur s'est produite lors de la suppression de la note : correspondent @@ -4312,7 +4312,7 @@ Aucun - + Error occurred while creating : . src/app/components/manage/management-list/management-list.component.ts @@ -4322,7 +4322,7 @@ src/app/components/manage/management-list/management-list.component.ts 153,155 - Une erreur s'est produite lors de la création de : . + Une erreur s'est produite lors de la création de : . Successfully created . @@ -4332,13 +4332,13 @@ Création de réussie. - + Error occurred while saving . src/app/components/manage/management-list/management-list.component.ts 174,176 - Une erreur s'est produite lors de la sauvegarde de : . + Une erreur s'est produite lors de la sauvegarde de : . Successfully updated . @@ -4348,13 +4348,13 @@ Mise à jour de réussie. - + Error occurred while saving : . src/app/components/manage/management-list/management-list.component.ts 187,189 - Une erreur s'est produite lors de la sauvegarde de : . + Une erreur s'est produite lors de la sauvegarde de : . Do you really want to delete the ? @@ -4572,13 +4572,13 @@ Vérification des mises à jour - + Update checking works by pinging the public Github API for the latest release to determine whether a new version is available. Actual updating of the app must still be performed manually. src/app/components/manage/settings/settings.component.html 140,142 - Update checking works by pinging the public Github API for the latest release to determine whether a new version is available. Actual updating of the app must still be performed manually. + La vérification des mises à jour fonctionne en faisant un ping sur l'API publique Github pour la dernière version afin de déterminer si une nouvelle version est disponible. La mise à jour réelle de l'application doit toujours être effectuée manuellement. No tracking data is collected by the app in any way. @@ -4636,13 +4636,13 @@ Appliquer lors de la fermeture - + Enable notes src/app/components/manage/settings/settings.component.html 163 - Activez les notes + Activer les notes Notifications @@ -4884,13 +4884,13 @@ Une erreur s'est produite lors de l'enregistrement des paramètres sur le serveur : - + Password has been changed, you will be logged out momentarily. src/app/components/manage/settings/settings.component.ts 659 - Le mot de passe a été modifié, vous serez déconnecté momentanément. + Le mot de passe a été modifié, vous serez déconnecté momentanément. Saved user "". @@ -5665,13 +5665,13 @@ Biélorusse - + Catalan src/app/services/settings.service.ts 177 - Catalan + Catalan Czech diff --git a/src-ui/src/locale/messages.sr_CS.xlf b/src-ui/src/locale/messages.sr_CS.xlf index 718d44c42..51a210da0 100644 --- a/src-ui/src/locale/messages.sr_CS.xlf +++ b/src-ui/src/locale/messages.sr_CS.xlf @@ -3076,7 +3076,7 @@ src/app/components/document-detail/document-detail.component.ts 595,597 - Error deleting document: + Greška prilikom brisanja dokumenta: Redo OCR confirm @@ -4578,7 +4578,7 @@ src/app/components/manage/settings/settings.component.html 140,142 - Update checking works by pinging the public Github API for the latest release to determine whether a new version is available. Actual updating of the app must still be performed manually. + Provera ažuriranja funkcioniše pingovanjem javnog Github API za najnovije izdanje da bi se utvrdilo da li je nova verzija dostupna. Stvarno ažuriranje aplikacije i dalje mora da se obavlja ručno. No tracking data is collected by the app in any way. @@ -5671,7 +5671,7 @@ src/app/services/settings.service.ts 177 - Catalan + Katalonski Czech diff --git a/src/documents/admin.py b/src/documents/admin.py index 072e69e6e..cc1b43dc1 100644 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -13,28 +13,24 @@ from .models import Tag class CorrespondentAdmin(GuardedModelAdmin): - list_display = ("name", "match", "matching_algorithm") list_filter = ("matching_algorithm",) list_editable = ("match", "matching_algorithm") class TagAdmin(GuardedModelAdmin): - list_display = ("name", "color", "match", "matching_algorithm") list_filter = ("color", "matching_algorithm") list_editable = ("color", "match", "matching_algorithm") class DocumentTypeAdmin(GuardedModelAdmin): - list_display = ("name", "match", "matching_algorithm") list_filter = ("matching_algorithm",) list_editable = ("match", "matching_algorithm") class DocumentAdmin(GuardedModelAdmin): - search_fields = ("correspondent__name", "title", "content", "tags__name") readonly_fields = ( "added", @@ -99,7 +95,6 @@ class RuleInline(admin.TabularInline): class SavedViewAdmin(GuardedModelAdmin): - list_display = ("name", "owner") inlines = [RuleInline] @@ -116,7 +111,6 @@ class StoragePathAdmin(GuardedModelAdmin): class TaskAdmin(admin.ModelAdmin): - list_display = ("task_id", "task_file_name", "task_name", "date_done", "status") list_filter = ("status", "date_done", "task_file_name", "task_name") search_fields = ("task_name", "task_id", "status") @@ -133,7 +127,6 @@ class TaskAdmin(admin.ModelAdmin): class NotesAdmin(GuardedModelAdmin): - list_display = ("user", "created", "note", "document") list_filter = ("created", "user") list_display_links = ("created",) diff --git a/src/documents/apps.py b/src/documents/apps.py index 9e0dd2554..edec7246c 100644 --- a/src/documents/apps.py +++ b/src/documents/apps.py @@ -3,22 +3,19 @@ from django.utils.translation import gettext_lazy as _ class DocumentsConfig(AppConfig): - name = "documents" verbose_name = _("Documents") def ready(self): from .signals import document_consumption_finished - from .signals.handlers import ( - add_inbox_tags, - set_log_entry, - set_correspondent, - set_document_type, - set_tags, - set_storage_path, - add_to_index, - ) + from .signals.handlers import add_inbox_tags + from .signals.handlers import add_to_index + from .signals.handlers import set_correspondent + from .signals.handlers import set_document_type + from .signals.handlers import set_log_entry + from .signals.handlers import set_storage_path + from .signals.handlers import set_tags document_consumption_finished.connect(add_inbox_tags) document_consumption_finished.connect(set_correspondent) diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 5908b55c8..ab0049eaa 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -1,6 +1,7 @@ import itertools from django.db.models import Q + from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType @@ -54,7 +55,6 @@ def set_document_type(doc_ids, document_type): def add_tag(doc_ids, tag): - qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(tags__id=tag)) affected_docs = [doc.id for doc in qs] @@ -70,7 +70,6 @@ def add_tag(doc_ids, tag): def remove_tag(doc_ids, tag): - qs = Document.objects.filter(Q(id__in=doc_ids) & Q(tags__id=tag)) affected_docs = [doc.id for doc in qs] @@ -122,7 +121,6 @@ def delete(doc_ids): def redo_ocr(doc_ids): - for document_id in doc_ids: update_document_archive_file.delay( document_id=document_id, @@ -132,7 +130,6 @@ def redo_ocr(doc_ids): def set_permissions(doc_ids, set_permissions, owner=None): - qs = Document.objects.filter(id__in=doc_ids) qs.update(owner=owner) diff --git a/src/documents/checks.py b/src/documents/checks.py index a014a0ac2..b2b49193a 100644 --- a/src/documents/checks.py +++ b/src/documents/checks.py @@ -6,6 +6,7 @@ from django.core.checks import register from django.core.exceptions import FieldError from django.db.utils import OperationalError from django.db.utils import ProgrammingError + from documents.signals import document_consumer_declaration @@ -22,7 +23,6 @@ def changed_password_check(app_configs, **kwargs): return [] # No documents table yet if encrypted_doc: - if not settings.PASSPHRASE: return [ Error( @@ -52,7 +52,6 @@ def changed_password_check(app_configs, **kwargs): @register() def parser_check(app_configs, **kwargs): - parsers = [] for response in document_consumer_declaration.send(None): parsers.append(response[1]) diff --git a/src/documents/classifier.py b/src/documents/classifier.py index cbb8b1b90..0848e0105 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -10,6 +10,7 @@ from typing import List from typing import Optional from django.conf import settings + from documents.models import Document from documents.models import MatchingModel @@ -59,7 +60,6 @@ def load_classifier() -> Optional["DocumentClassifier"]: class DocumentClassifier: - # v7 - Updated scikit-learn package version # v8 - Added storage path classifier # v9 - Changed from hashing to time/ids for re-train check @@ -140,7 +140,6 @@ class DocumentClassifier: target_file_temp.rename(target_file) def train(self): - # Get non-inbox documents docs_queryset = Document.objects.exclude( tags__is_inbox_tag=True, @@ -159,7 +158,6 @@ class DocumentClassifier: logger.debug("Gathering data from database...") hasher = sha256() for doc in docs_queryset: - y = -1 dt = doc.document_type if dt and dt.matching_algorithm == MatchingModel.MATCH_AUTO: @@ -334,12 +332,10 @@ class DocumentClassifier: # If the NLTK language is supported, do further processing if settings.NLTK_LANGUAGE is not None and settings.NLTK_ENABLED: - import nltk - - from nltk.tokenize import word_tokenize from nltk.corpus import stopwords from nltk.stem import SnowballStemmer + from nltk.tokenize import word_tokenize # Not really hacky, since it isn't private and is documented, but # set the search path for NLTK data to the single location it should be in diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 12993a750..b5ec58483 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -31,9 +31,9 @@ from .models import DocumentType from .models import FileInfo from .models import Tag from .parsers import DocumentParser +from .parsers import ParseError from .parsers import get_parser_class_for_mime_type from .parsers import parse_date -from .parsers import ParseError from .signals import document_consumption_finished from .signals import document_consumption_started @@ -60,7 +60,6 @@ MESSAGE_FINISHED = "finished" class Consumer(LoggingMixin): - logging_name = "paperless.consumer" def _send_progress( @@ -426,7 +425,6 @@ class Consumer(LoggingMixin): # in the system. This will be a transaction and reasonably fast. try: with transaction.atomic(): - # store the document. document = self._store(text=text, date=date, mime_type=mime_type) @@ -520,7 +518,6 @@ class Consumer(LoggingMixin): date: Optional[datetime.datetime], mime_type: str, ) -> Document: - # If someone gave us the original filename, use it instead of doc. file_info = FileInfo.from_filename(self.filename) diff --git a/src/documents/file_handling.py b/src/documents/file_handling.py index c046ae15a..e382d4965 100644 --- a/src/documents/file_handling.py +++ b/src/documents/file_handling.py @@ -7,6 +7,7 @@ import pathvalidate from django.conf import settings from django.template.defaultfilters import slugify from django.utils import timezone + from documents.models import Document logger = logging.getLogger("paperless.filehandling") diff --git a/src/documents/filters.py b/src/documents/filters.py index 271b91108..56a490bc0 100644 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -11,7 +11,6 @@ from .models import Log from .models import StoragePath from .models import Tag - CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"] ID_KWARGS = ["in", "exact"] INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"] @@ -83,7 +82,6 @@ class TitleContentFilter(Filter): class DocumentFilterSet(FilterSet): - is_tagged = BooleanFilter( label="Is tagged", field_name="tags", diff --git a/src/documents/index.py b/src/documents/index.py index 1bf6a921d..403282403 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -6,8 +6,6 @@ from contextlib import contextmanager from dateutil.parser import isoparse from django.conf import settings from django.utils import timezone -from documents.models import Document -from documents.models import Note from guardian.shortcuts import get_users_with_perms from whoosh import classify from whoosh import highlight @@ -16,8 +14,8 @@ from whoosh.fields import BOOLEAN from whoosh.fields import DATETIME from whoosh.fields import KEYWORD from whoosh.fields import NUMERIC -from whoosh.fields import Schema from whoosh.fields import TEXT +from whoosh.fields import Schema from whoosh.highlight import HtmlFormatter from whoosh.index import create_in from whoosh.index import exists_in @@ -28,6 +26,9 @@ from whoosh.searching import ResultsPage from whoosh.searching import Searcher from whoosh.writing import AsyncWriter +from documents.models import Document +from documents.models import Note + logger = logging.getLogger("paperless.index") @@ -330,7 +331,7 @@ class DelayedMoreLikeThisQuery(DelayedQuery): def autocomplete(ix, term, limit=10): with ix.reader() as reader: terms = [] - for (score, t) in reader.most_distinctive_terms( + for score, t in reader.most_distinctive_terms( "content", number=limit, prefix=term.lower(), diff --git a/src/documents/loggers.py b/src/documents/loggers.py index 0dd109277..0f03135d5 100644 --- a/src/documents/loggers.py +++ b/src/documents/loggers.py @@ -3,7 +3,6 @@ import uuid class LoggingMixin: - logging_group = None logging_name = None diff --git a/src/documents/management/commands/decrypt_documents.py b/src/documents/management/commands/decrypt_documents.py index 2cb98c4e1..8b67ee7d0 100644 --- a/src/documents/management/commands/decrypt_documents.py +++ b/src/documents/management/commands/decrypt_documents.py @@ -3,19 +3,18 @@ import os from django.conf import settings from django.core.management.base import BaseCommand from django.core.management.base import CommandError + from documents.models import Document from paperless.db import GnuPG class Command(BaseCommand): - help = ( "This is how you migrate your stored documents from an encrypted " "state to an unencrypted one (or vice-versa)" ) def add_arguments(self, parser): - parser.add_argument( "--passphrase", help="If PAPERLESS_PASSPHRASE isn't set already, you need to " @@ -23,7 +22,6 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - try: print( "\n\nWARNING: This script is going to work directly on your " @@ -48,13 +46,11 @@ class Command(BaseCommand): @staticmethod def __gpg_to_unencrypted(passphrase): - encrypted_files = Document.objects.filter( storage_type=Document.STORAGE_TYPE_GPG, ) for document in encrypted_files: - print(f"Decrypting {document}".encode()) old_paths = [document.source_path, document.thumbnail_path] diff --git a/src/documents/management/commands/document_archiver.py b/src/documents/management/commands/document_archiver.py index fa78a1963..69d9a8183 100644 --- a/src/documents/management/commands/document_archiver.py +++ b/src/documents/management/commands/document_archiver.py @@ -6,15 +6,14 @@ import tqdm from django import db from django.conf import settings from django.core.management.base import BaseCommand + from documents.models import Document from documents.tasks import update_document_archive_file - logger = logging.getLogger("paperless.management.archiver") class Command(BaseCommand): - help = """ Using the current classification model, assigns correspondents, tags and document types to all documents, effectively allowing you to @@ -51,7 +50,6 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) overwrite = options["overwrite"] @@ -74,7 +72,6 @@ class Command(BaseCommand): db.connections.close_all() try: - logging.getLogger().handlers[0].level = logging.ERROR with multiprocessing.Pool(processes=settings.TASK_WORKERS) as pool: list( diff --git a/src/documents/management/commands/document_consumer.py b/src/documents/management/commands/document_consumer.py index 6cba1ea23..220948a3d 100644 --- a/src/documents/management/commands/document_consumer.py +++ b/src/documents/management/commands/document_consumer.py @@ -13,17 +13,19 @@ from typing import Set from django.conf import settings from django.core.management.base import BaseCommand from django.core.management.base import CommandError +from watchdog.events import FileSystemEventHandler +from watchdog.observers.polling import PollingObserver + from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource from documents.models import Tag from documents.parsers import is_file_ext_supported from documents.tasks import consume_file -from watchdog.events import FileSystemEventHandler -from watchdog.observers.polling import PollingObserver try: - from inotifyrecursive import INotify, flags + from inotifyrecursive import INotify + from inotifyrecursive import flags except ImportError: # pragma: nocover INotify = flags = None diff --git a/src/documents/management/commands/document_create_classifier.py b/src/documents/management/commands/document_create_classifier.py index 04aa9ab2b..88d2092e3 100644 --- a/src/documents/management/commands/document_create_classifier.py +++ b/src/documents/management/commands/document_create_classifier.py @@ -4,7 +4,6 @@ from documents.tasks import train_classifier class Command(BaseCommand): - help = """ Trains the classifier on your data and saves the resulting models to a file. The document consumer will then automatically use this new model. diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index 151868137..b916538e7 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -17,6 +17,10 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandError from django.db import transaction from django.utils import timezone +from filelock import FileLock + +from documents.file_handling import delete_empty_directories +from documents.file_handling import generate_filename from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType @@ -29,18 +33,13 @@ from documents.models import UiSettings from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME -from filelock import FileLock from paperless import version from paperless.db import GnuPG from paperless_mail.models import MailAccount from paperless_mail.models import MailRule -from documents.file_handling import delete_empty_directories -from documents.file_handling import generate_filename - class Command(BaseCommand): - help = """ Decrypt and rename all files in our collection into a given target directory. And include a manifest file containing document data for @@ -144,7 +143,6 @@ class Command(BaseCommand): self.no_thumbnail = False def handle(self, *args, **options): - self.target = Path(options["target"]).resolve() self.split_manifest = options["split_manifest"] self.compare_checksums = options["compare_checksums"] diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index d68f04ac3..b00cb45fa 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -14,16 +14,16 @@ from django.core.management.base import CommandError from django.core.serializers.base import DeserializationError from django.db.models.signals import m2m_changed from django.db.models.signals import post_save +from filelock import FileLock + +from documents.file_handling import create_source_path_directory from documents.models import Document from documents.parsers import run_convert from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME -from filelock import FileLock -from paperless import version - -from documents.file_handling import create_source_path_directory from documents.signals.handlers import update_filename_and_move_files +from paperless import version @contextmanager @@ -36,7 +36,6 @@ def disable_signal(sig, receiver, sender): class Command(BaseCommand): - help = """ Using a manifest.json file, load the data from there, and import the documents it refers to. @@ -61,12 +60,11 @@ class Command(BaseCommand): self.version = None def handle(self, *args, **options): - logging.getLogger().handlers[0].level = logging.ERROR - self.source = options["source"] + self.source = Path(options["source"]).resolve() - if not os.path.exists(self.source): + if not self.source.exists(): raise CommandError("That path doesn't exist") if not os.access(self.source, os.R_OK): @@ -74,39 +72,39 @@ class Command(BaseCommand): manifest_paths = [] - main_manifest_path = os.path.normpath( - os.path.join(self.source, "manifest.json"), - ) + main_manifest_path = self.source / "manifest.json" + self._check_manifest_exists(main_manifest_path) - with open(main_manifest_path) as f: - self.manifest = json.load(f) + with main_manifest_path.open() as infile: + self.manifest = json.load(infile) manifest_paths.append(main_manifest_path) for file in Path(self.source).glob("**/*-manifest.json"): - with open(file) as f: - self.manifest += json.load(f) + with file.open() as infile: + self.manifest += json.load(infile) manifest_paths.append(file) - version_path = os.path.normpath(os.path.join(self.source, "version.json")) - if os.path.exists(version_path): - with open(version_path) as f: - self.version = json.load(f)["version"] - # Provide an initial warning if needed to the user - if self.version != version.__full_version_str__: - self.stdout.write( - self.style.WARNING( - "Version mismatch: " - f"Currently {version.__full_version_str__}," - f" importing {self.version}." - " Continuing, but import may fail.", - ), - ) + version_path = self.source / "version.json" + if version_path.exists(): + with version_path.open() as infile: + self.version = json.load(infile)["version"] + # Provide an initial warning if needed to the user + if self.version != version.__full_version_str__: + self.stdout.write( + self.style.WARNING( + "Version mismatch: " + f"Currently {version.__full_version_str__}," + f" importing {self.version}." + " Continuing, but import may fail.", + ), + ) else: self.stdout.write(self.style.NOTICE("No version.json file located")) - self._check_manifest() + self._check_manifest_valid() + with disable_signal( post_save, receiver=update_filename_and_move_files, @@ -150,16 +148,19 @@ class Command(BaseCommand): ) @staticmethod - def _check_manifest_exists(path): - if not os.path.exists(path): + def _check_manifest_exists(path: Path): + if not path.exists(): raise CommandError( "That directory doesn't appear to contain a manifest.json file.", ) - def _check_manifest(self): - + def _check_manifest_valid(self): + """ + Attempts to verify the manifest is valid. Namely checking the files + referred to exist and the files can be read from + """ + self.stdout.write("Checking the manifest") for record in self.manifest: - if record["model"] != "documents.document": continue @@ -170,22 +171,37 @@ class Command(BaseCommand): ) doc_file = record[EXPORTER_FILE_NAME] - if not os.path.exists(os.path.join(self.source, doc_file)): + doc_path = self.source / doc_file + if not doc_path.exists(): raise CommandError( 'The manifest file refers to "{}" which does not ' "appear to be in the source directory.".format(doc_file), ) + try: + with doc_path.open(mode="rb") as infile: + infile.read(1) + except Exception as e: + raise CommandError( + f"Failed to read from original file {doc_path}", + ) from e if EXPORTER_ARCHIVE_NAME in record: archive_file = record[EXPORTER_ARCHIVE_NAME] - if not os.path.exists(os.path.join(self.source, archive_file)): + doc_archive_path = self.source / archive_file + if not doc_archive_path.exists(): raise CommandError( f"The manifest file refers to {archive_file} which " f"does not appear to be in the source directory.", ) + try: + with doc_archive_path.open(mode="rb") as infile: + infile.read(1) + except Exception as e: + raise CommandError( + f"Failed to read from archive file {doc_archive_path}", + ) from e def _import_files_from_manifest(self, progress_bar_disable): - os.makedirs(settings.ORIGINALS_DIR, exist_ok=True) os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True) os.makedirs(settings.ARCHIVE_DIR, exist_ok=True) @@ -197,7 +213,6 @@ class Command(BaseCommand): ) for record in tqdm.tqdm(manifest_documents, disable=progress_bar_disable): - document = Document.objects.get(pk=record["pk"]) doc_file = record[EXPORTER_FILE_NAME] diff --git a/src/documents/management/commands/document_index.py b/src/documents/management/commands/document_index.py index cf7eb65e5..279408b36 100644 --- a/src/documents/management/commands/document_index.py +++ b/src/documents/management/commands/document_index.py @@ -1,11 +1,11 @@ from django.core.management import BaseCommand from django.db import transaction + from documents.tasks import index_optimize from documents.tasks import index_reindex class Command(BaseCommand): - help = "Manages the document index." def add_arguments(self, parser): diff --git a/src/documents/management/commands/document_renamer.py b/src/documents/management/commands/document_renamer.py index 79c46f905..be1446957 100644 --- a/src/documents/management/commands/document_renamer.py +++ b/src/documents/management/commands/document_renamer.py @@ -3,11 +3,11 @@ import logging import tqdm from django.core.management.base import BaseCommand from django.db.models.signals import post_save + from documents.models import Document class Command(BaseCommand): - help = """ This will rename all documents to match the latest filename format. """.replace( @@ -24,7 +24,6 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - logging.getLogger().handlers[0].level = logging.ERROR for document in tqdm.tqdm( diff --git a/src/documents/management/commands/document_retagger.py b/src/documents/management/commands/document_retagger.py index aa61f0696..e67d6aed0 100644 --- a/src/documents/management/commands/document_retagger.py +++ b/src/documents/management/commands/document_retagger.py @@ -2,20 +2,18 @@ import logging import tqdm from django.core.management.base import BaseCommand + from documents.classifier import load_classifier from documents.models import Document - from documents.signals.handlers import set_correspondent from documents.signals.handlers import set_document_type from documents.signals.handlers import set_storage_path from documents.signals.handlers import set_tags - logger = logging.getLogger("paperless.management.retagger") class Command(BaseCommand): - help = """ Using the current classification model, assigns correspondents, tags and document types to all documents, effectively allowing you to @@ -79,7 +77,6 @@ class Command(BaseCommand): classifier = load_classifier() for document in tqdm.tqdm(documents, disable=options["no_progress_bar"]): - if options["correspondent"]: set_correspondent( sender=None, diff --git a/src/documents/management/commands/document_sanity_checker.py b/src/documents/management/commands/document_sanity_checker.py index 27c119863..4c06d2a84 100644 --- a/src/documents/management/commands/document_sanity_checker.py +++ b/src/documents/management/commands/document_sanity_checker.py @@ -1,9 +1,9 @@ from django.core.management.base import BaseCommand + from documents.sanity_checker import check_sanity class Command(BaseCommand): - help = """ This command checks your document archive for issues. """.replace( @@ -20,7 +20,6 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - messages = check_sanity(progress=not options["no_progress_bar"]) messages.log_messages() diff --git a/src/documents/management/commands/document_thumbnails.py b/src/documents/management/commands/document_thumbnails.py index 462853f84..0f35dba42 100644 --- a/src/documents/management/commands/document_thumbnails.py +++ b/src/documents/management/commands/document_thumbnails.py @@ -5,8 +5,8 @@ import shutil import tqdm from django import db from django.core.management.base import BaseCommand -from documents.models import Document +from documents.models import Document from documents.parsers import get_parser_class_for_mime_type @@ -21,7 +21,6 @@ def _process_document(doc_in): return try: - thumb = parser.get_thumbnail( document.source_path, document.mime_type, @@ -34,7 +33,6 @@ def _process_document(doc_in): class Command(BaseCommand): - help = """ This will regenerate the thumbnails for all documents. """.replace( diff --git a/src/documents/management/commands/manage_superuser.py b/src/documents/management/commands/manage_superuser.py index 2f506b54a..df0502f17 100644 --- a/src/documents/management/commands/manage_superuser.py +++ b/src/documents/management/commands/manage_superuser.py @@ -4,12 +4,10 @@ import os from django.contrib.auth.models import User from django.core.management.base import BaseCommand - logger = logging.getLogger("paperless.management.superuser") class Command(BaseCommand): - help = """ Creates a Django superuser: User named: admin @@ -25,7 +23,6 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - username = os.getenv("PAPERLESS_ADMIN_USER", "admin") mail = os.getenv("PAPERLESS_ADMIN_MAIL", "root@localhost") password = os.getenv("PAPERLESS_ADMIN_PASSWORD") diff --git a/src/documents/matching.py b/src/documents/matching.py index ad80ee0ad..521d49284 100644 --- a/src/documents/matching.py +++ b/src/documents/matching.py @@ -8,7 +8,6 @@ from documents.models import StoragePath from documents.models import Tag from documents.permissions import get_objects_for_user_owner_aware - logger = logging.getLogger("paperless.matching") @@ -23,6 +22,9 @@ def log_reason(matching_model, document, reason): def match_correspondents(document, classifier, user=None): pred_id = classifier.predict_correspondent(document.content) if classifier else None + if user is None and document.owner is not None: + user = document.owner + if user is not None: correspondents = get_objects_for_user_owner_aware( user, @@ -40,6 +42,9 @@ def match_correspondents(document, classifier, user=None): def match_document_types(document, classifier, user=None): pred_id = classifier.predict_document_type(document.content) if classifier else None + if user is None and document.owner is not None: + user = document.owner + if user is not None: document_types = get_objects_for_user_owner_aware( user, @@ -57,6 +62,9 @@ def match_document_types(document, classifier, user=None): def match_tags(document, classifier, user=None): predicted_tag_ids = classifier.predict_tags(document.content) if classifier else [] + if user is None and document.owner is not None: + user = document.owner + if user is not None: tags = get_objects_for_user_owner_aware(user, "documents.view_tag", Tag) else: @@ -70,6 +78,9 @@ def match_tags(document, classifier, user=None): def match_storage_paths(document, classifier, user=None): pred_id = classifier.predict_storage_path(document.content) if classifier else None + if user is None and document.owner is not None: + user = document.owner + if user is not None: storage_paths = get_objects_for_user_owner_aware( user, diff --git a/src/documents/migrations/0001_initial.py b/src/documents/migrations/0001_initial.py index e1b2f2a8b..89c9e29df 100644 --- a/src/documents/migrations/0001_initial.py +++ b/src/documents/migrations/0001_initial.py @@ -1,11 +1,11 @@ # Generated by Django 1.9 on 2015-12-20 19:10 -from django.db import migrations, models from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - initial = True dependencies = [] diff --git a/src/documents/migrations/0002_auto_20151226_1316.py b/src/documents/migrations/0002_auto_20151226_1316.py index b953d8008..ffd240902 100644 --- a/src/documents/migrations/0002_auto_20151226_1316.py +++ b/src/documents/migrations/0002_auto_20151226_1316.py @@ -1,11 +1,11 @@ # Generated by Django 1.9 on 2015-12-26 13:16 -from django.db import migrations, models import django.utils.timezone +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0001_initial"), ] diff --git a/src/documents/migrations/0003_sender.py b/src/documents/migrations/0003_sender.py index c2b274085..fb37746db 100644 --- a/src/documents/migrations/0003_sender.py +++ b/src/documents/migrations/0003_sender.py @@ -1,16 +1,14 @@ # Generated by Django 1.9 on 2016-01-11 12:21 -from django.db import migrations, models -from django.template.defaultfilters import slugify - import django.db.models.deletion - +from django.db import migrations +from django.db import models +from django.template.defaultfilters import slugify DOCUMENT_SENDER_MAP = {} def move_sender_strings_to_sender_model(apps, schema_editor): - sender_model = apps.get_model("documents", "Sender") document_model = apps.get_model("documents", "Document") diff --git a/src/documents/migrations/0004_auto_20160114_1844.py b/src/documents/migrations/0004_auto_20160114_1844.py index b3ccaaa9b..97bda420e 100644 --- a/src/documents/migrations/0004_auto_20160114_1844.py +++ b/src/documents/migrations/0004_auto_20160114_1844.py @@ -1,11 +1,11 @@ # Generated by Django 1.9 on 2016-01-14 18:44 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0003_sender"), ] diff --git a/src/documents/migrations/0005_auto_20160123_0313.py b/src/documents/migrations/0005_auto_20160123_0313.py index 98e2c1b29..b0ccc5825 100644 --- a/src/documents/migrations/0005_auto_20160123_0313.py +++ b/src/documents/migrations/0005_auto_20160123_0313.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("documents", "0004_auto_20160114_1844"), ] diff --git a/src/documents/migrations/0006_auto_20160123_0430.py b/src/documents/migrations/0006_auto_20160123_0430.py index 3f24992e2..315b9646f 100644 --- a/src/documents/migrations/0006_auto_20160123_0430.py +++ b/src/documents/migrations/0006_auto_20160123_0430.py @@ -1,10 +1,10 @@ # Generated by Django 1.9 on 2016-01-23 04:30 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0005_auto_20160123_0313"), ] diff --git a/src/documents/migrations/0007_auto_20160126_2114.py b/src/documents/migrations/0007_auto_20160126_2114.py index f4b3b913d..04ccc0589 100644 --- a/src/documents/migrations/0007_auto_20160126_2114.py +++ b/src/documents/migrations/0007_auto_20160126_2114.py @@ -1,10 +1,10 @@ # Generated by Django 1.9 on 2016-01-26 21:14 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0006_auto_20160123_0430"), ] diff --git a/src/documents/migrations/0008_document_file_type.py b/src/documents/migrations/0008_document_file_type.py index c079f927d..7787f8622 100644 --- a/src/documents/migrations/0008_document_file_type.py +++ b/src/documents/migrations/0008_document_file_type.py @@ -1,10 +1,10 @@ # Generated by Django 1.9 on 2016-01-29 22:58 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0007_auto_20160126_2114"), ] diff --git a/src/documents/migrations/0009_auto_20160214_0040.py b/src/documents/migrations/0009_auto_20160214_0040.py index 82e48ba24..5c95e1035 100644 --- a/src/documents/migrations/0009_auto_20160214_0040.py +++ b/src/documents/migrations/0009_auto_20160214_0040.py @@ -1,10 +1,10 @@ # Generated by Django 1.9 on 2016-02-14 00:40 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0008_document_file_type"), ] diff --git a/src/documents/migrations/0010_log.py b/src/documents/migrations/0010_log.py index 9be7b18ed..8f015cab0 100644 --- a/src/documents/migrations/0010_log.py +++ b/src/documents/migrations/0010_log.py @@ -1,10 +1,10 @@ # Generated by Django 1.9 on 2016-02-27 17:54 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0009_auto_20160214_0040"), ] diff --git a/src/documents/migrations/0012_auto_20160305_0040.py b/src/documents/migrations/0012_auto_20160305_0040.py index 1470ace96..b656ef70e 100644 --- a/src/documents/migrations/0012_auto_20160305_0040.py +++ b/src/documents/migrations/0012_auto_20160305_0040.py @@ -1,12 +1,12 @@ # Generated by Django 1.9.2 on 2016-03-05 00:40 -import gnupg import os import re import shutil import subprocess import tempfile +import gnupg from django.conf import settings from django.db import migrations from django.utils.termcolors import colorize as colourise # Spelling hurts me @@ -34,7 +34,6 @@ class GnuPG: def move_documents_and_create_thumbnails(apps, schema_editor): - os.makedirs( os.path.join(settings.MEDIA_ROOT, "documents", "originals"), exist_ok=True, @@ -67,7 +66,6 @@ def move_documents_and_create_thumbnails(apps, schema_editor): pass for f in sorted(documents): - if not f.endswith("gpg"): continue diff --git a/src/documents/migrations/0013_auto_20160325_2111.py b/src/documents/migrations/0013_auto_20160325_2111.py index 6663edad8..1d3f8b07d 100644 --- a/src/documents/migrations/0013_auto_20160325_2111.py +++ b/src/documents/migrations/0013_auto_20160325_2111.py @@ -1,11 +1,11 @@ # Generated by Django 1.9.4 on 2016-03-25 21:11 -from django.db import migrations, models import django.utils.timezone +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0012_auto_20160305_0040"), ] diff --git a/src/documents/migrations/0014_document_checksum.py b/src/documents/migrations/0014_document_checksum.py index a687e43ef..deef59990 100644 --- a/src/documents/migrations/0014_document_checksum.py +++ b/src/documents/migrations/0014_document_checksum.py @@ -1,12 +1,13 @@ # Generated by Django 1.9.4 on 2016-03-28 19:09 -import gnupg import hashlib import os import django.utils.timezone +import gnupg from django.conf import settings -from django.db import migrations, models +from django.db import migrations +from django.db import models from django.template.defaultfilters import slugify from django.utils.termcolors import colorize as colourise # Spelling hurts me @@ -74,7 +75,6 @@ class Document: def set_checksums(apps, schema_editor): - document_model = apps.get_model("documents", "Document") if not document_model.objects.all().exists(): @@ -94,7 +94,6 @@ def set_checksums(apps, schema_editor): sums = {} for d in document_model.objects.all(): - document = Document(d) print( diff --git a/src/documents/migrations/0015_add_insensitive_to_match.py b/src/documents/migrations/0015_add_insensitive_to_match.py index 796918446..0761cc929 100644 --- a/src/documents/migrations/0015_add_insensitive_to_match.py +++ b/src/documents/migrations/0015_add_insensitive_to_match.py @@ -1,10 +1,10 @@ # Generated by Django 1.10.2 on 2016-10-05 21:38 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0014_document_checksum"), ] diff --git a/src/documents/migrations/0016_auto_20170325_1558.py b/src/documents/migrations/0016_auto_20170325_1558.py index 26ab3a720..743097d0c 100644 --- a/src/documents/migrations/0016_auto_20170325_1558.py +++ b/src/documents/migrations/0016_auto_20170325_1558.py @@ -1,11 +1,11 @@ # Generated by Django 1.10.5 on 2017-03-25 15:58 -from django.db import migrations, models from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0015_add_insensitive_to_match"), ] diff --git a/src/documents/migrations/0017_auto_20170512_0507.py b/src/documents/migrations/0017_auto_20170512_0507.py index f775cdfe0..74cf39854 100644 --- a/src/documents/migrations/0017_auto_20170512_0507.py +++ b/src/documents/migrations/0017_auto_20170512_0507.py @@ -1,10 +1,10 @@ # Generated by Django 1.10.5 on 2017-05-12 05:07 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0016_auto_20170325_1558"), ] diff --git a/src/documents/migrations/0018_auto_20170715_1712.py b/src/documents/migrations/0018_auto_20170715_1712.py index 047531fee..838d79ddc 100644 --- a/src/documents/migrations/0018_auto_20170715_1712.py +++ b/src/documents/migrations/0018_auto_20170715_1712.py @@ -1,11 +1,11 @@ # Generated by Django 1.10.5 on 2017-07-15 17:12 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0017_auto_20170512_0507"), ] diff --git a/src/documents/migrations/0020_document_added.py b/src/documents/migrations/0020_document_added.py index 67c4df4aa..34042eedf 100644 --- a/src/documents/migrations/0020_document_added.py +++ b/src/documents/migrations/0020_document_added.py @@ -1,5 +1,6 @@ -from django.db import migrations, models import django.utils.timezone +from django.db import migrations +from django.db import models def set_added_time_to_created_time(apps, schema_editor): diff --git a/src/documents/migrations/0021_document_storage_type.py b/src/documents/migrations/0021_document_storage_type.py index bde86ceea..b35fe75ed 100644 --- a/src/documents/migrations/0021_document_storage_type.py +++ b/src/documents/migrations/0021_document_storage_type.py @@ -1,10 +1,10 @@ # Generated by Django 1.11.10 on 2018-02-04 13:07 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "0020_document_added"), ] diff --git a/src/documents/migrations/0022_auto_20181007_1420.py b/src/documents/migrations/0022_auto_20181007_1420.py index ecebf80d8..02dfa6d2b 100644 --- a/src/documents/migrations/0022_auto_20181007_1420.py +++ b/src/documents/migrations/0022_auto_20181007_1420.py @@ -1,6 +1,7 @@ # Generated by Django 2.0.8 on 2018-10-07 14:20 -from django.db import migrations, models +from django.db import migrations +from django.db import models from django.utils.text import slugify @@ -19,7 +20,6 @@ def re_slug_all_the_things(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "0021_document_storage_type"), ] diff --git a/src/documents/migrations/0023_document_current_filename.py b/src/documents/migrations/0023_document_current_filename.py index 65300c11e..5f52e1c89 100644 --- a/src/documents/migrations/0023_document_current_filename.py +++ b/src/documents/migrations/0023_document_current_filename.py @@ -1,6 +1,7 @@ # Generated by Django 2.0.10 on 2019-04-26 18:57 -from django.db import migrations, models +from django.db import migrations +from django.db import models def set_filename(apps, schema_editor): @@ -18,7 +19,6 @@ def set_filename(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "0022_auto_20181007_1420"), ] diff --git a/src/documents/migrations/1000_update_paperless_all.py b/src/documents/migrations/1000_update_paperless_all.py index 4301a5b21..6026ce3d2 100644 --- a/src/documents/migrations/1000_update_paperless_all.py +++ b/src/documents/migrations/1000_update_paperless_all.py @@ -1,8 +1,9 @@ # Generated by Django 3.1.3 on 2020-11-07 12:35 import uuid -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models def logs_set_default_group(apps, schema_editor): @@ -14,7 +15,6 @@ def logs_set_default_group(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "0023_document_current_filename"), ] diff --git a/src/documents/migrations/1001_auto_20201109_1636.py b/src/documents/migrations/1001_auto_20201109_1636.py index 9f6e152b6..7477a118c 100644 --- a/src/documents/migrations/1001_auto_20201109_1636.py +++ b/src/documents/migrations/1001_auto_20201109_1636.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("documents", "1000_update_paperless_all"), ] diff --git a/src/documents/migrations/1002_auto_20201111_1105.py b/src/documents/migrations/1002_auto_20201111_1105.py index f502fba3f..1835c4ca9 100644 --- a/src/documents/migrations/1002_auto_20201111_1105.py +++ b/src/documents/migrations/1002_auto_20201111_1105.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.3 on 2020-11-11 11:05 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1001_auto_20201109_1636"), ] diff --git a/src/documents/migrations/1003_mime_types.py b/src/documents/migrations/1003_mime_types.py index f5ac94756..446657495 100644 --- a/src/documents/migrations/1003_mime_types.py +++ b/src/documents/migrations/1003_mime_types.py @@ -3,7 +3,8 @@ import os import magic from django.conf import settings -from django.db import migrations, models +from django.db import migrations +from django.db import models from paperless.db import GnuPG @@ -29,7 +30,6 @@ def add_mime_types(apps, schema_editor): for d in documents: f = open(source_path(d), "rb") if d.storage_type == STORAGE_TYPE_GPG: - data = GnuPG.decrypted(f) else: data = f.read(1024) @@ -50,7 +50,6 @@ def add_file_extensions(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "1002_auto_20201111_1105"), ] diff --git a/src/documents/migrations/1004_sanity_check_schedule.py b/src/documents/migrations/1004_sanity_check_schedule.py index 0437fbd57..018cf2492 100644 --- a/src/documents/migrations/1004_sanity_check_schedule.py +++ b/src/documents/migrations/1004_sanity_check_schedule.py @@ -5,7 +5,6 @@ from django.db.migrations import RunPython class Migration(migrations.Migration): - dependencies = [ ("documents", "1003_mime_types"), ] diff --git a/src/documents/migrations/1005_checksums.py b/src/documents/migrations/1005_checksums.py index b1bfc6eac..4637e06ce 100644 --- a/src/documents/migrations/1005_checksums.py +++ b/src/documents/migrations/1005_checksums.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.3 on 2020-11-29 00:48 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1004_sanity_check_schedule"), ] diff --git a/src/documents/migrations/1006_auto_20201208_2209.py b/src/documents/migrations/1006_auto_20201208_2209.py index 8a7e7a99d..425f0a768 100644 --- a/src/documents/migrations/1006_auto_20201208_2209.py +++ b/src/documents/migrations/1006_auto_20201208_2209.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("documents", "1005_checksums"), ] diff --git a/src/documents/migrations/1007_savedview_savedviewfilterrule.py b/src/documents/migrations/1007_savedview_savedviewfilterrule.py index 357d65c2d..64564c6af 100644 --- a/src/documents/migrations/1007_savedview_savedviewfilterrule.py +++ b/src/documents/migrations/1007_savedview_savedviewfilterrule.py @@ -1,12 +1,12 @@ # Generated by Django 3.1.4 on 2020-12-12 14:41 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("documents", "1006_auto_20201208_2209"), diff --git a/src/documents/migrations/1008_auto_20201216_1736.py b/src/documents/migrations/1008_auto_20201216_1736.py index 2f6c5c59d..76f0343b1 100644 --- a/src/documents/migrations/1008_auto_20201216_1736.py +++ b/src/documents/migrations/1008_auto_20201216_1736.py @@ -1,11 +1,10 @@ # Generated by Django 3.1.4 on 2020-12-16 17:36 -from django.db import migrations import django.db.models.functions.text +from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("documents", "1007_savedview_savedviewfilterrule"), ] diff --git a/src/documents/migrations/1009_auto_20201216_2005.py b/src/documents/migrations/1009_auto_20201216_2005.py index 9584789bc..37bae8881 100644 --- a/src/documents/migrations/1009_auto_20201216_2005.py +++ b/src/documents/migrations/1009_auto_20201216_2005.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("documents", "1008_auto_20201216_1736"), ] diff --git a/src/documents/migrations/1010_auto_20210101_2159.py b/src/documents/migrations/1010_auto_20210101_2159.py index 1d05d8f47..0c6a42d30 100644 --- a/src/documents/migrations/1010_auto_20210101_2159.py +++ b/src/documents/migrations/1010_auto_20210101_2159.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.4 on 2021-01-01 21:59 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1009_auto_20201216_2005"), ] diff --git a/src/documents/migrations/1011_auto_20210101_2340.py b/src/documents/migrations/1011_auto_20210101_2340.py index c7c387226..dea107715 100644 --- a/src/documents/migrations/1011_auto_20210101_2340.py +++ b/src/documents/migrations/1011_auto_20210101_2340.py @@ -1,13 +1,13 @@ # Generated by Django 3.1.4 on 2021-01-01 23:40 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("documents", "1010_auto_20210101_2159"), diff --git a/src/documents/migrations/1012_fix_archive_files.py b/src/documents/migrations/1012_fix_archive_files.py index 51eb8ec2e..ee38e16db 100644 --- a/src/documents/migrations/1012_fix_archive_files.py +++ b/src/documents/migrations/1012_fix_archive_files.py @@ -8,11 +8,12 @@ from time import sleep import pathvalidate from django.conf import settings -from django.db import migrations, models +from django.db import migrations +from django.db import models from django.template.defaultfilters import slugify -from documents.file_handling import defaultdictNoStr, many_to_dictionary - +from documents.file_handling import defaultdictNoStr +from documents.file_handling import many_to_dictionary logger = logging.getLogger("paperless.migrations") @@ -160,11 +161,9 @@ def parse_wrapper(parser, path, mime_type, file_name): def create_archive_version(doc, retry_count=3): - from documents.parsers import ( - get_parser_class_for_mime_type, - DocumentParser, - ParseError, - ) + from documents.parsers import DocumentParser + from documents.parsers import ParseError + from documents.parsers import get_parser_class_for_mime_type logger.info(f"Regenerating archive document for document ID:{doc.id}") parser_class = get_parser_class_for_mime_type(doc.mime_type) @@ -255,7 +254,6 @@ def move_old_to_new_locations(apps, schema_editor): ) for doc in Document.objects.filter(archive_checksum__isnull=False): - if doc.id in affected_document_ids: old_path = archive_path_old(doc) # remove affected archive versions @@ -305,7 +303,6 @@ def move_new_to_old_locations(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "1011_auto_20210101_2340"), ] diff --git a/src/documents/migrations/1013_migrate_tag_colour.py b/src/documents/migrations/1013_migrate_tag_colour.py index 8346ff184..6cae10898 100644 --- a/src/documents/migrations/1013_migrate_tag_colour.py +++ b/src/documents/migrations/1013_migrate_tag_colour.py @@ -1,6 +1,7 @@ # Generated by Django 3.1.4 on 2020-12-02 21:43 -from django.db import migrations, models +from django.db import migrations +from django.db import models COLOURS_OLD = { 1: "#a6cee3", @@ -46,7 +47,6 @@ def reverse(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "1012_fix_archive_files"), ] diff --git a/src/documents/migrations/1014_auto_20210228_1614.py b/src/documents/migrations/1014_auto_20210228_1614.py index c3f16b841..5785bcb53 100644 --- a/src/documents/migrations/1014_auto_20210228_1614.py +++ b/src/documents/migrations/1014_auto_20210228_1614.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.7 on 2021-02-28 15:14 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1013_migrate_tag_colour"), ] diff --git a/src/documents/migrations/1015_remove_null_characters.py b/src/documents/migrations/1015_remove_null_characters.py index cea9d5a64..9872b3a75 100644 --- a/src/documents/migrations/1015_remove_null_characters.py +++ b/src/documents/migrations/1015_remove_null_characters.py @@ -3,7 +3,6 @@ import logging from django.db import migrations - logger = logging.getLogger("paperless.migrations") @@ -19,7 +18,6 @@ def remove_null_characters(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "1014_auto_20210228_1614"), ] diff --git a/src/documents/migrations/1016_auto_20210317_1351.py b/src/documents/migrations/1016_auto_20210317_1351.py index d41fae849..67147fd4c 100644 --- a/src/documents/migrations/1016_auto_20210317_1351.py +++ b/src/documents/migrations/1016_auto_20210317_1351.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.7 on 2021-03-17 12:51 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1015_remove_null_characters"), ] diff --git a/src/documents/migrations/1017_alter_savedviewfilterrule_rule_type.py b/src/documents/migrations/1017_alter_savedviewfilterrule_rule_type.py index 52db95138..ab18f1bc1 100644 --- a/src/documents/migrations/1017_alter_savedviewfilterrule_rule_type.py +++ b/src/documents/migrations/1017_alter_savedviewfilterrule_rule_type.py @@ -1,10 +1,10 @@ # Generated by Django 3.2.12 on 2022-03-17 11:59 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1016_auto_20210317_1351"), ] diff --git a/src/documents/migrations/1018_alter_savedviewfilterrule_value.py b/src/documents/migrations/1018_alter_savedviewfilterrule_value.py index 8453a86d8..95ef4861d 100644 --- a/src/documents/migrations/1018_alter_savedviewfilterrule_value.py +++ b/src/documents/migrations/1018_alter_savedviewfilterrule_value.py @@ -1,10 +1,10 @@ # Generated by Django 4.0.3 on 2022-04-01 22:50 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1017_alter_savedviewfilterrule_rule_type"), ] diff --git a/src/documents/migrations/1019_storagepath_document_storage_path.py b/src/documents/migrations/1019_storagepath_document_storage_path.py index ff7f38692..b09941bf5 100644 --- a/src/documents/migrations/1019_storagepath_document_storage_path.py +++ b/src/documents/migrations/1019_storagepath_document_storage_path.py @@ -1,11 +1,11 @@ # Generated by Django 4.0.4 on 2022-05-02 15:56 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1018_alter_savedviewfilterrule_value"), ] diff --git a/src/documents/migrations/1019_uisettings.py b/src/documents/migrations/1019_uisettings.py index edc944a37..e84138077 100644 --- a/src/documents/migrations/1019_uisettings.py +++ b/src/documents/migrations/1019_uisettings.py @@ -1,12 +1,12 @@ # Generated by Django 4.0.4 on 2022-05-07 05:10 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("documents", "1018_alter_savedviewfilterrule_value"), diff --git a/src/documents/migrations/1020_merge_20220518_1839.py b/src/documents/migrations/1020_merge_20220518_1839.py index f455fc5a3..a766aaa20 100644 --- a/src/documents/migrations/1020_merge_20220518_1839.py +++ b/src/documents/migrations/1020_merge_20220518_1839.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("documents", "1019_storagepath_document_storage_path"), ("documents", "1019_uisettings"), diff --git a/src/documents/migrations/1021_webp_thumbnail_conversion.py b/src/documents/migrations/1021_webp_thumbnail_conversion.py index c7ae1eaae..3b2ac9b16 100644 --- a/src/documents/migrations/1021_webp_thumbnail_conversion.py +++ b/src/documents/migrations/1021_webp_thumbnail_conversion.py @@ -8,6 +8,7 @@ from pathlib import Path from django.conf import settings from django.db import migrations + from documents.parsers import run_convert logger = logging.getLogger("paperless.migrations") @@ -16,7 +17,6 @@ logger = logging.getLogger("paperless.migrations") def _do_convert(work_package): existing_thumbnail, converted_thumbnail = work_package try: - logger.info(f"Converting thumbnail: {existing_thumbnail}") # Run actual conversion @@ -50,7 +50,6 @@ def _convert_thumbnails_to_webp(apps, schema_editor): start = time.time() with tempfile.TemporaryDirectory() as tempdir: - work_packages = [] for file in Path(settings.THUMBNAIL_DIR).glob("*.png"): @@ -72,7 +71,6 @@ def _convert_thumbnails_to_webp(apps, schema_editor): ) if len(work_packages): - logger.info( "\n\n" " This is a one-time only migration to convert thumbnails for all of your\n" @@ -94,7 +92,6 @@ def _convert_thumbnails_to_webp(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("documents", "1020_merge_20220518_1839"), ] diff --git a/src/documents/migrations/1022_paperlesstask.py b/src/documents/migrations/1022_paperlesstask.py index e398402f3..c7b3f7744 100644 --- a/src/documents/migrations/1022_paperlesstask.py +++ b/src/documents/migrations/1022_paperlesstask.py @@ -1,11 +1,11 @@ # Generated by Django 4.0.4 on 2022-05-23 07:14 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1021_webp_thumbnail_conversion"), ] diff --git a/src/documents/migrations/1023_add_comments.py b/src/documents/migrations/1023_add_comments.py index 2c89947e9..0b26739bc 100644 --- a/src/documents/migrations/1023_add_comments.py +++ b/src/documents/migrations/1023_add_comments.py @@ -1,6 +1,7 @@ -from django.db import migrations, models import django.utils.timezone from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): diff --git a/src/documents/migrations/1024_document_original_filename.py b/src/documents/migrations/1024_document_original_filename.py index b0f03cd50..05be7269e 100644 --- a/src/documents/migrations/1024_document_original_filename.py +++ b/src/documents/migrations/1024_document_original_filename.py @@ -1,10 +1,10 @@ # Generated by Django 4.0.6 on 2022-07-25 06:34 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1023_add_comments"), ] diff --git a/src/documents/migrations/1025_alter_savedviewfilterrule_rule_type.py b/src/documents/migrations/1025_alter_savedviewfilterrule_rule_type.py index 1e5b005b8..a2deb9579 100644 --- a/src/documents/migrations/1025_alter_savedviewfilterrule_rule_type.py +++ b/src/documents/migrations/1025_alter_savedviewfilterrule_rule_type.py @@ -1,10 +1,10 @@ # Generated by Django 4.0.5 on 2022-08-26 16:49 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1024_document_original_filename"), ] diff --git a/src/documents/migrations/1026_transition_to_celery.py b/src/documents/migrations/1026_transition_to_celery.py index 786ca36c4..227188d22 100644 --- a/src/documents/migrations/1026_transition_to_celery.py +++ b/src/documents/migrations/1026_transition_to_celery.py @@ -1,11 +1,11 @@ # Generated by Django 4.1.1 on 2022-09-27 19:31 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("django_celery_results", "0011_taskresult_periodic_task_name"), ("documents", "1025_alter_savedviewfilterrule_rule_type"), diff --git a/src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py b/src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py index fc8ff8ec2..c169c3096 100644 --- a/src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py +++ b/src/documents/migrations/1027_remove_paperlesstask_attempted_task_and_more.py @@ -1,11 +1,11 @@ # Generated by Django 4.1.2 on 2022-10-17 16:31 -from django.db import migrations, models import django.utils.timezone +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1026_transition_to_celery"), ] diff --git a/src/documents/migrations/1028_remove_paperlesstask_task_args_and_more.py b/src/documents/migrations/1028_remove_paperlesstask_task_args_and_more.py index 83f3eadfa..6e03c124b 100644 --- a/src/documents/migrations/1028_remove_paperlesstask_task_args_and_more.py +++ b/src/documents/migrations/1028_remove_paperlesstask_task_args_and_more.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("documents", "1027_remove_paperlesstask_attempted_task_and_more"), ] diff --git a/src/documents/migrations/1029_alter_document_archive_serial_number.py b/src/documents/migrations/1029_alter_document_archive_serial_number.py index d7e4b662d..57848b2dc 100644 --- a/src/documents/migrations/1029_alter_document_archive_serial_number.py +++ b/src/documents/migrations/1029_alter_document_archive_serial_number.py @@ -1,11 +1,11 @@ # Generated by Django 4.1.4 on 2023-01-24 17:56 import django.core.validators -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1028_remove_paperlesstask_task_args_and_more"), ] diff --git a/src/documents/migrations/1030_alter_paperlesstask_task_file_name.py b/src/documents/migrations/1030_alter_paperlesstask_task_file_name.py index 130358bec..37e918bee 100644 --- a/src/documents/migrations/1030_alter_paperlesstask_task_file_name.py +++ b/src/documents/migrations/1030_alter_paperlesstask_task_file_name.py @@ -1,10 +1,10 @@ # Generated by Django 4.1.5 on 2023-02-03 21:53 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1029_alter_document_archive_serial_number"), ] diff --git a/src/documents/migrations/1031_remove_savedview_user_correspondent_owner_and_more.py b/src/documents/migrations/1031_remove_savedview_user_correspondent_owner_and_more.py index b187871e7..56e4355ef 100644 --- a/src/documents/migrations/1031_remove_savedview_user_correspondent_owner_and_more.py +++ b/src/documents/migrations/1031_remove_savedview_user_correspondent_owner_and_more.py @@ -1,12 +1,12 @@ # Generated by Django 4.1.4 on 2022-02-03 04:24 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("documents", "1030_alter_paperlesstask_task_file_name"), diff --git a/src/documents/migrations/1032_alter_correspondent_matching_algorithm_and_more.py b/src/documents/migrations/1032_alter_correspondent_matching_algorithm_and_more.py index 88aa7f2bc..3d1c5658a 100644 --- a/src/documents/migrations/1032_alter_correspondent_matching_algorithm_and_more.py +++ b/src/documents/migrations/1032_alter_correspondent_matching_algorithm_and_more.py @@ -1,10 +1,10 @@ # Generated by Django 4.1.7 on 2023-02-22 00:45 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1031_remove_savedview_user_correspondent_owner_and_more"), ] diff --git a/src/documents/migrations/1033_alter_documenttype_options_alter_tag_options_and_more.py b/src/documents/migrations/1033_alter_documenttype_options_alter_tag_options_and_more.py index 433363e9f..1368ac641 100644 --- a/src/documents/migrations/1033_alter_documenttype_options_alter_tag_options_and_more.py +++ b/src/documents/migrations/1033_alter_documenttype_options_alter_tag_options_and_more.py @@ -1,10 +1,10 @@ # Generated by Django 4.1.5 on 2023-03-04 22:33 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1032_alter_correspondent_matching_algorithm_and_more"), ] diff --git a/src/documents/migrations/1034_alter_savedviewfilterrule_rule_type.py b/src/documents/migrations/1034_alter_savedviewfilterrule_rule_type.py index aa56d9629..b648ac839 100644 --- a/src/documents/migrations/1034_alter_savedviewfilterrule_rule_type.py +++ b/src/documents/migrations/1034_alter_savedviewfilterrule_rule_type.py @@ -1,10 +1,10 @@ # Generated by Django 4.1.5 on 2023-03-15 07:10 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1033_alter_documenttype_options_alter_tag_options_and_more"), ] diff --git a/src/documents/migrations/1035_rename_comment_note.py b/src/documents/migrations/1035_rename_comment_note.py index 11c3da380..9f9aaca94 100644 --- a/src/documents/migrations/1035_rename_comment_note.py +++ b/src/documents/migrations/1035_rename_comment_note.py @@ -1,13 +1,12 @@ # Generated by Django 4.1.5 on 2023-03-17 22:15 +import django.db.models.deletion from django.conf import settings from django.db import migrations from django.db import models -import django.db.models.deletion class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("documents", "1034_alter_savedviewfilterrule_rule_type"), diff --git a/src/documents/models.py b/src/documents/models.py index 079459d00..d28664a68 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -17,6 +17,7 @@ from django.core.validators import MinValueValidator from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ + from documents.parsers import get_default_file_extension ALL_STATES = sorted(states.ALL_STATES) @@ -37,7 +38,6 @@ class ModelWithOwner(models.Model): class MatchingModel(ModelWithOwner): - MATCH_NONE = 0 MATCH_ANY = 1 MATCH_ALL = 2 @@ -94,7 +94,6 @@ class Correspondent(MatchingModel): class Tag(MatchingModel): - color = models.CharField(_("color"), max_length=7, default="#a6cee3") is_inbox_tag = models.BooleanField( @@ -129,7 +128,6 @@ class StoragePath(MatchingModel): class Document(ModelWithOwner): - STORAGE_TYPE_UNENCRYPTED = "unencrypted" STORAGE_TYPE_GPG = "gpg" STORAGE_TYPES = ( @@ -279,7 +277,6 @@ class Document(ModelWithOwner): verbose_name_plural = _("documents") def __str__(self) -> str: - # Convert UTC database time to local time created = datetime.date.isoformat(timezone.localdate(self.created)) @@ -364,7 +361,6 @@ class Document(ModelWithOwner): class Log(models.Model): - LEVELS = ( (logging.DEBUG, _("debug")), (logging.INFO, _("information")), @@ -396,7 +392,6 @@ class Log(models.Model): class SavedView(ModelWithOwner): class Meta: - ordering = ("name",) verbose_name = _("saved view") verbose_name_plural = _("saved views") @@ -480,7 +475,6 @@ class SavedViewFilterRule(models.Model): # the filename, if possible, as a higher priority than either document filename or # content parsing class FileInfo: - REGEXES = OrderedDict( [ ( @@ -502,7 +496,6 @@ class FileInfo: tags=(), extension=None, ): - self.created = created self.title = title self.extension = extension @@ -529,7 +522,7 @@ class FileInfo: def from_filename(cls, filename) -> "FileInfo": # Mutate filename in-place before parsing its components # by applying at most one of the configured transformations. - for (pattern, repl) in settings.FILENAME_PARSE_TRANSFORMS: + for pattern, repl in settings.FILENAME_PARSE_TRANSFORMS: (filename, count) = pattern.subn(repl, filename) if count: break @@ -563,7 +556,6 @@ class FileInfo: # Extending User Model Using a One-To-One Link class UiSettings(models.Model): - user = models.OneToOneField( User, on_delete=models.CASCADE, diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 4bec79c61..80968912c 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -14,6 +14,7 @@ from typing import Set from django.conf import settings from django.utils import timezone + from documents.loggers import LoggingMixin from documents.signals import document_consumer_declaration @@ -139,7 +140,6 @@ def run_convert( extra=None, logging_group=None, ) -> None: - environment = os.environ.copy() if settings.CONVERT_MEMORY_LIMIT: environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT diff --git a/src/documents/permissions.py b/src/documents/permissions.py index d4114e488..d5712f670 100644 --- a/src/documents/permissions.py +++ b/src/documents/permissions.py @@ -2,6 +2,7 @@ from django.contrib.auth.models import Group from django.contrib.auth.models import Permission from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType +from guardian.core import ObjectPermissionChecker from guardian.models import GroupObjectPermission from guardian.shortcuts import assign_perm from guardian.shortcuts import get_objects_for_user @@ -114,3 +115,8 @@ def get_objects_for_user_owner_aware(user, perms, Model): accept_global_perms=False, ) return objects_owned | objects_unowned | objects_with_perms + + +def has_perms_owner_aware(user, perms, obj): + checker = ObjectPermissionChecker(user) + return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj) diff --git a/src/documents/sanity_checker.py b/src/documents/sanity_checker.py index b74d07bd5..497d586f1 100644 --- a/src/documents/sanity_checker.py +++ b/src/documents/sanity_checker.py @@ -5,9 +5,10 @@ from pathlib import Path from typing import Final from django.conf import settings -from documents.models import Document from tqdm import tqdm +from documents.models import Document + class SanityCheckMessages: def __init__(self): @@ -32,7 +33,6 @@ class SanityCheckMessages: if len(self._messages) == 0: logger.info("Sanity checker detected no issues.") else: - # Query once all_docs = Document.objects.all() diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 2ad392f4c..ad2b5d0f6 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -10,32 +10,31 @@ except ImportError: from backports import zoneinfo import magic from django.conf import settings +from django.contrib.auth.models import Group +from django.contrib.auth.models import User from django.utils.text import slugify from django.utils.translation import gettext as _ +from guardian.core import ObjectPermissionChecker +from guardian.shortcuts import get_users_with_perms from rest_framework import serializers from rest_framework.fields import SerializerMethodField +from documents.permissions import get_groups_with_only_permission +from documents.permissions import set_permissions_for_object + from . import bulk_edit from .models import Correspondent from .models import Document from .models import DocumentType from .models import MatchingModel +from .models import PaperlessTask from .models import SavedView from .models import SavedViewFilterRule from .models import StoragePath from .models import Tag from .models import UiSettings -from .models import PaperlessTask from .parsers import is_mime_type_supported -from guardian.shortcuts import get_users_with_perms - -from django.contrib.auth.models import User -from django.contrib.auth.models import Group - -from documents.permissions import get_groups_with_only_permission -from documents.permissions import set_permissions_for_object - # https://www.django-rest-framework.org/api-guide/serializers/#example class DynamicFieldsModelSerializer(serializers.ModelSerializer): @@ -60,7 +59,6 @@ class DynamicFieldsModelSerializer(serializers.ModelSerializer): class MatchingModelSerializer(serializers.ModelSerializer): - document_count = serializers.IntegerField(read_only=True) def get_slug(self, obj): @@ -152,11 +150,21 @@ class SetPermissionsMixin: class OwnedObjectSerializer(serializers.ModelSerializer, SetPermissionsMixin): def __init__(self, *args, **kwargs): self.user = kwargs.pop("user", None) + full_perms = kwargs.pop("full_perms", False) super().__init__(*args, **kwargs) + try: + if full_perms: + self.fields.pop("user_can_change") + else: + self.fields.pop("permissions") + except KeyError: + pass + def get_permissions(self, obj): view_codename = f"view_{obj.__class__.__name__.lower()}" change_codename = f"change_{obj.__class__.__name__.lower()}" + return { "view": { "users": get_users_with_perms( @@ -182,7 +190,19 @@ class OwnedObjectSerializer(serializers.ModelSerializer, SetPermissionsMixin): }, } + def get_user_can_change(self, obj): + checker = ObjectPermissionChecker(self.user) if self.user is not None else None + return ( + obj.owner is None + or obj.owner == self.user + or ( + self.user is not None + and checker.has_perm(f"change_{obj.__class__.__name__.lower()}", obj) + ) + ) + permissions = SerializerMethodField(read_only=True) + user_can_change = SerializerMethodField(read_only=True) set_permissions = serializers.DictField( label="Set permissions", @@ -223,7 +243,6 @@ class OwnedObjectSerializer(serializers.ModelSerializer, SetPermissionsMixin): class CorrespondentSerializer(MatchingModelSerializer, OwnedObjectSerializer): - last_correspondence = serializers.DateTimeField(read_only=True) class Meta: @@ -239,6 +258,7 @@ class CorrespondentSerializer(MatchingModelSerializer, OwnedObjectSerializer): "last_correspondence", "owner", "permissions", + "user_can_change", "set_permissions", ) @@ -256,12 +276,12 @@ class DocumentTypeSerializer(MatchingModelSerializer, OwnedObjectSerializer): "document_count", "owner", "permissions", + "user_can_change", "set_permissions", ) class ColorField(serializers.Field): - COLOURS = ( (1, "#a6cee3"), (2, "#1f78b4"), @@ -292,7 +312,6 @@ class ColorField(serializers.Field): class TagSerializerVersion1(MatchingModelSerializer, OwnedObjectSerializer): - colour = ColorField(source="color", default="#a6cee3") class Meta: @@ -309,6 +328,7 @@ class TagSerializerVersion1(MatchingModelSerializer, OwnedObjectSerializer): "document_count", "owner", "permissions", + "user_can_change", "set_permissions", ) @@ -344,6 +364,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer): "document_count", "owner", "permissions", + "user_can_change", "set_permissions", ) @@ -375,7 +396,6 @@ class StoragePathField(serializers.PrimaryKeyRelatedField): class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer): - correspondent = CorrespondentField(allow_null=True) tags = TagsField(many=True) document_type = DocumentTypeField(allow_null=True) @@ -444,6 +464,7 @@ class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer): "archived_file_name", "owner", "permissions", + "user_can_change", "set_permissions", "notes", ) @@ -456,7 +477,6 @@ class SavedViewFilterRuleSerializer(serializers.ModelSerializer): class SavedViewSerializer(OwnedObjectSerializer): - filter_rules = SavedViewFilterRuleSerializer(many=True) class Meta: @@ -472,6 +492,7 @@ class SavedViewSerializer(OwnedObjectSerializer): "filter_rules", "owner", "permissions", + "user_can_change", "set_permissions", ] @@ -502,7 +523,6 @@ class SavedViewSerializer(OwnedObjectSerializer): class DocumentListSerializer(serializers.Serializer): - documents = serializers.ListField( required=True, label="Documents", @@ -527,7 +547,6 @@ class DocumentListSerializer(serializers.Serializer): class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin): - method = serializers.ChoiceField( choices=[ "set_correspondent", @@ -653,7 +672,6 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin): self._validate_owner(parameters["owner"]) def validate(self, attrs): - method = attrs["method"] parameters = attrs["parameters"] @@ -674,7 +692,6 @@ class BulkEditSerializer(DocumentListSerializer, SetPermissionsMixin): class PostDocumentSerializer(serializers.Serializer): - created = serializers.DateTimeField( label="Created", allow_null=True, @@ -756,7 +773,6 @@ class PostDocumentSerializer(serializers.Serializer): class BulkDownloadSerializer(DocumentListSerializer): - content = serializers.ChoiceField( choices=["archive", "originals", "both"], default="archive", @@ -796,6 +812,7 @@ class StoragePathSerializer(MatchingModelSerializer, OwnedObjectSerializer): "document_count", "owner", "permissions", + "user_can_change", "set_permissions", ) @@ -907,7 +924,6 @@ class TasksViewSerializer(serializers.ModelSerializer): class AcknowledgeTasksViewSerializer(serializers.Serializer): - tasks = serializers.ListField( required=True, label="Tasks", diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index a7f75c489..4abb772ce 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -28,12 +28,21 @@ from documents.models import Document from documents.models import MatchingModel from documents.models import PaperlessTask from documents.models import Tag +from documents.permissions import get_objects_for_user_owner_aware logger = logging.getLogger("paperless.handlers") def add_inbox_tags(sender, document=None, logging_group=None, **kwargs): - inbox_tags = Tag.objects.filter(is_inbox_tag=True) + if document.owner is not None: + tags = get_objects_for_user_owner_aware( + document.owner, + "documents.view_documenttype", + Tag, + ) + else: + tags = Tag.objects.all() + inbox_tags = tags.filter(is_inbox_tag=True) document.tags.add(*inbox_tags) @@ -175,7 +184,6 @@ def set_tags( color=False, **kwargs, ): - if replace: Document.tags.through.objects.filter(document=document).exclude( Q(tag__is_inbox_tag=True), @@ -376,7 +384,6 @@ def validate_move(instance, old_path, new_path): @receiver(models.signals.m2m_changed, sender=Document.tags.through) @receiver(models.signals.post_save, sender=Document) def update_filename_and_move_files(sender, instance: Document, **kwargs): - if not instance.filename: # Can't update the filename if there is no filename to begin with # This happens when the consumer creates a new document. @@ -390,7 +397,6 @@ def update_filename_and_move_files(sender, instance: Document, **kwargs): with FileLock(settings.MEDIA_LOCK): try: - # If this was waiting for the lock, the filename or archive_filename # of this document may have been updated. This happens if multiple updates # get queued from the UI for the same document @@ -407,7 +413,6 @@ def update_filename_and_move_files(sender, instance: Document, **kwargs): old_archive_path = instance.archive_path if instance.has_archive_version: - instance.archive_filename = generate_unique_filename( instance, archive_filename=True, @@ -487,7 +492,6 @@ def update_filename_and_move_files(sender, instance: Document, **kwargs): def set_log_entry(sender, document=None, logging_group=None, **kwargs): - ct = ContentType.objects.get(model="document") user = User.objects.get(username="consumer") diff --git a/src/documents/tasks.py b/src/documents/tasks.py index c2d726405..f51fa9828 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -12,6 +12,10 @@ from channels.layers import get_channel_layer from django.conf import settings from django.db import transaction from django.db.models.signals import post_save +from filelock import FileLock +from redis.exceptions import ConnectionError +from whoosh.writing import AsyncWriter + from documents import barcodes from documents import index from documents import sanity_checker @@ -32,10 +36,6 @@ from documents.models import Tag from documents.parsers import DocumentParser from documents.parsers import get_parser_class_for_mime_type from documents.sanity_checker import SanityCheckFailedException -from filelock import FileLock -from redis.exceptions import ConnectionError -from whoosh.writing import AsyncWriter - logger = logging.getLogger("paperless.tasks") @@ -65,7 +65,6 @@ def train_classifier(): and not Correspondent.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists() and not StoragePath.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists() ): - return classifier = load_classifier() @@ -91,7 +90,6 @@ def consume_file( input_doc: ConsumableDocument, overrides: Optional[DocumentMetadataOverrides] = None, ): - # Default no overrides if overrides is None: overrides = DocumentMetadataOverrides() @@ -117,7 +115,6 @@ def consume_file( ) if document_list: - # If the file is an upload, it's in the scratch directory # Move it to consume directory to be picked up # Otherwise, use the current parent to keep possible tags diff --git a/src/documents/tests/test_admin.py b/src/documents/tests/test_admin.py index 7a39c95be..a32a31adf 100644 --- a/src/documents/tests/test_admin.py +++ b/src/documents/tests/test_admin.py @@ -1,6 +1,7 @@ from django.contrib.admin.sites import AdminSite from django.test import TestCase from django.utils import timezone + from documents import index from documents.admin import DocumentAdmin from documents.models import Document diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 6cd6b610a..a6307e2d5 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -20,30 +20,31 @@ except ImportError: from backports import zoneinfo import pytest +from dateutil.relativedelta import relativedelta from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth.models import Permission from django.contrib.auth.models import User from django.test import override_settings from django.utils import timezone -from dateutil.relativedelta import relativedelta from rest_framework import status +from rest_framework.test import APITestCase +from whoosh.writing import AsyncWriter + from documents import bulk_edit from documents import index from documents.models import Correspondent from documents.models import Document -from documents.tests.utils import DocumentConsumeDelayMixin from documents.models import DocumentType from documents.models import MatchingModel +from documents.models import Note from documents.models import PaperlessTask from documents.models import SavedView from documents.models import StoragePath from documents.models import Tag -from documents.models import Note from documents.tests.utils import DirectoriesMixin +from documents.tests.utils import DocumentConsumeDelayMixin from paperless import version -from rest_framework.test import APITestCase -from whoosh.writing import AsyncWriter class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): @@ -54,7 +55,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.client.force_authenticate(user=self.user) def testDocuments(self): - response = self.client.get("/api/documents/").data self.assertEqual(response["count"], 0) @@ -170,7 +170,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(len(results[0]), 0) def test_document_actions(self): - _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir) content = b"This is a test" @@ -206,9 +205,69 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.content, content_thumbnail) + def test_document_actions_with_perms(self): + """ + GIVEN: + - Document with owner and without granted permissions + - User is then granted permissions + WHEN: + - User tries to load preview, thumbnail + THEN: + - Initially, HTTP 403 Forbidden + - With permissions, HTTP 200 OK + """ + _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir) + + content = b"This is a test" + content_thumbnail = b"thumbnail content" + + with open(filename, "wb") as f: + f.write(content) + + user1 = User.objects.create_user(username="test1") + user2 = User.objects.create_user(username="test2") + user1.user_permissions.add(*Permission.objects.filter(codename="view_document")) + user2.user_permissions.add(*Permission.objects.filter(codename="view_document")) + + self.client.force_authenticate(user2) + + doc = Document.objects.create( + title="none", + filename=os.path.basename(filename), + mime_type="application/pdf", + owner=user1, + ) + + with open( + os.path.join(self.dirs.thumbnail_dir, f"{doc.pk:07d}.webp"), + "wb", + ) as f: + f.write(content_thumbnail) + + response = self.client.get(f"/api/documents/{doc.pk}/download/") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + response = self.client.get(f"/api/documents/{doc.pk}/preview/") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + response = self.client.get(f"/api/documents/{doc.pk}/thumb/") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + from guardian.shortcuts import assign_perm + + assign_perm("view_document", user2, doc) + + response = self.client.get(f"/api/documents/{doc.pk}/download/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + + response = self.client.get(f"/api/documents/{doc.pk}/preview/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + + response = self.client.get(f"/api/documents/{doc.pk}/thumb/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + @override_settings(FILENAME_FORMAT="") def test_download_with_archive(self): - content = b"This is a test" content_archive = b"This is the same test but archived" @@ -250,7 +309,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(response.content, content) def test_document_actions_not_existing_file(self): - doc = Document.objects.create( title="none", filename=os.path.basename("asd"), @@ -267,7 +325,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) def test_document_filters(self): - doc1 = Document.objects.create( title="none1", checksum="A", @@ -365,7 +422,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(len(results), 0) def test_documents_title_content_filter(self): - doc1 = Document.objects.create( title="title A", content="content A", @@ -1039,7 +1095,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): ) def test_statistics(self): - doc1 = Document.objects.create( title="none1", checksum="A", @@ -1087,7 +1142,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(response.data["inbox_tag"], None) def test_upload(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1115,7 +1169,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertIsNone(overrides.tag_ids) def test_upload_empty_metadata(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1143,7 +1196,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertIsNone(overrides.tag_ids) def test_upload_invalid_form(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1160,7 +1212,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.consume_file_mock.assert_not_called() def test_upload_invalid_file(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1177,7 +1228,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.consume_file_mock.assert_not_called() def test_upload_with_title(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1202,7 +1252,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertIsNone(overrides.tag_ids) def test_upload_with_correspondent(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1228,7 +1277,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertIsNone(overrides.tag_ids) def test_upload_with_invalid_correspondent(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1246,7 +1294,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.consume_file_mock.assert_not_called() def test_upload_with_document_type(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1272,7 +1319,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertIsNone(overrides.tag_ids) def test_upload_with_invalid_document_type(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1290,7 +1336,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.consume_file_mock.assert_not_called() def test_upload_with_tags(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1317,7 +1362,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertIsNone(overrides.title) def test_upload_with_invalid_tags(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1337,7 +1381,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.consume_file_mock.assert_not_called() def test_upload_with_created(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1369,7 +1412,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertEqual(overrides.created, created) def test_upload_with_asn(self): - self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), ) @@ -1593,7 +1635,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): ) def test_create_update_patch(self): - User.objects.create_user("user1") view = { @@ -2072,7 +2113,6 @@ class TestDocumentApiV2(DirectoriesMixin, APITestCase): class TestApiUiSettings(DirectoriesMixin, APITestCase): - ENDPOINT = "/api/ui_settings/" def setUp(self): @@ -2868,7 +2908,6 @@ class TestBulkEdit(DirectoriesMixin, APITestCase): class TestBulkDownload(DirectoriesMixin, APITestCase): - ENDPOINT = "/api/documents/bulk_download/" def setUp(self): @@ -3190,7 +3229,6 @@ class TestBulkDownload(DirectoriesMixin, APITestCase): class TestApiAuth(DirectoriesMixin, APITestCase): def test_auth_required(self): - d = Document.objects.create(title="Test") self.assertEqual( @@ -3255,7 +3293,6 @@ class TestApiAuth(DirectoriesMixin, APITestCase): ) def test_api_version_no_auth(self): - response = self.client.get("/api/") self.assertNotIn("X-Api-Version", response) self.assertNotIn("X-Version", response) @@ -3359,6 +3396,36 @@ class TestApiAuth(DirectoriesMixin, APITestCase): status.HTTP_404_NOT_FOUND, ) + def test_dynamic_permissions_fields(self): + Document.objects.create(title="Test", content="content 1", checksum="1") + + user1 = User.objects.create_superuser(username="test1") + self.client.force_authenticate(user1) + + response = self.client.get( + "/api/documents/", + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + resp_data = response.json() + + self.assertNotIn("permissions", resp_data["results"][0]) + self.assertIn("user_can_change", resp_data["results"][0]) + + response = self.client.get( + "/api/documents/?full_perms=true", + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + resp_data = response.json() + + self.assertIn("permissions", resp_data["results"][0]) + self.assertNotIn("user_can_change", resp_data["results"][0]) + class TestApiRemoteVersion(DirectoriesMixin, APITestCase): ENDPOINT = "/api/remote_version/" @@ -3368,7 +3435,6 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase): @mock.patch("urllib.request.urlopen") def test_remote_version_enabled_no_update_prefix(self, urlopen_mock): - cm = MagicMock() cm.getcode.return_value = status.HTTP_200_OK cm.read.return_value = json.dumps({"tag_name": "ngx-1.6.0"}).encode() @@ -3388,7 +3454,6 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase): @mock.patch("urllib.request.urlopen") def test_remote_version_enabled_no_update_no_prefix(self, urlopen_mock): - cm = MagicMock() cm.getcode.return_value = status.HTTP_200_OK cm.read.return_value = json.dumps( @@ -3410,7 +3475,6 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase): @mock.patch("urllib.request.urlopen") def test_remote_version_enabled_update(self, urlopen_mock): - new_version = ( version.__version__[0], version.__version__[1], @@ -3439,7 +3503,6 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase): @mock.patch("urllib.request.urlopen") def test_remote_version_bad_json(self, urlopen_mock): - cm = MagicMock() cm.getcode.return_value = status.HTTP_200_OK cm.read.return_value = b'{ "blah":' @@ -3459,7 +3522,6 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase): @mock.patch("urllib.request.urlopen") def test_remote_version_exception(self, urlopen_mock): - cm = MagicMock() cm.getcode.return_value = status.HTTP_200_OK cm.read.side_effect = urllib.error.URLError("an error") diff --git a/src/documents/tests/test_barcodes.py b/src/documents/tests/test_barcodes.py index 2520c7df6..5a6c3edf9 100644 --- a/src/documents/tests/test_barcodes.py +++ b/src/documents/tests/test_barcodes.py @@ -4,8 +4,10 @@ from unittest import mock import pytest from django.conf import settings -from django.test import override_settings from django.test import TestCase +from django.test import override_settings +from PIL import Image + from documents import barcodes from documents import tasks from documents.consumer import ConsumerError @@ -13,7 +15,6 @@ from documents.data_models import ConsumableDocument from documents.data_models import DocumentSource from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin -from PIL import Image try: import zxingcpp # noqa: F401 diff --git a/src/documents/tests/test_checks.py b/src/documents/tests/test_checks.py index a331572da..07af596bb 100644 --- a/src/documents/tests/test_checks.py +++ b/src/documents/tests/test_checks.py @@ -2,8 +2,9 @@ import textwrap from unittest import mock from django.core.checks import Error -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from documents.checks import changed_password_check from documents.checks import parser_check from documents.models import Document @@ -35,7 +36,6 @@ class TestDocumentChecks(TestCase): @mock.patch("paperless.db.GnuPG.decrypted") @mock.patch("documents.models.Document.source_file") def test_encrypted_decrypt_fails(self, mock_decrypted, mock_source_file): - mock_decrypted.return_value = None mock_source_file.return_value = b"" @@ -60,7 +60,6 @@ class TestDocumentChecks(TestCase): ) def test_parser_check(self): - self.assertEqual(parser_check(None), []) with mock.patch("documents.checks.document_consumer_declaration.send") as m: diff --git a/src/documents/tests/test_classifier.py b/src/documents/tests/test_classifier.py index f0aa5894e..4e361aa8e 100644 --- a/src/documents/tests/test_classifier.py +++ b/src/documents/tests/test_classifier.py @@ -5,8 +5,9 @@ from unittest import mock import pytest from django.conf import settings -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from documents.classifier import ClassifierModelCorruptError from documents.classifier import DocumentClassifier from documents.classifier import IncompatibleClassifierVersionError @@ -325,7 +326,6 @@ class TestClassifier(DirectoriesMixin, TestCase): classifier2.load() def testSaveClassifier(self): - self.generate_train_and_save() new_classifier = DocumentClassifier() @@ -335,7 +335,6 @@ class TestClassifier(DirectoriesMixin, TestCase): self.assertFalse(new_classifier.train()) def test_load_and_classify(self): - self.generate_train_and_save() new_classifier = DocumentClassifier() diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index cd06e9782..8ab40f5ac 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -15,9 +15,9 @@ except ImportError: from backports import zoneinfo from django.conf import settings -from django.utils import timezone -from django.test import override_settings from django.test import TestCase +from django.test import override_settings +from django.utils import timezone from documents.consumer import Consumer from documents.consumer import ConsumerError @@ -29,12 +29,12 @@ from documents.models import Tag from documents.parsers import DocumentParser from documents.parsers import ParseError from documents.tasks import sanity_check -from .utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin +from .utils import DirectoriesMixin + class TestAttributes(TestCase): - TAGS = ("tag1", "tag2", "tag3") def _test_guess_attributes_from_name(self, filename, sender, title, tags): @@ -67,7 +67,6 @@ class TestAttributes(TestCase): class TestFieldPermutations(TestCase): - valid_dates = ( "20150102030405Z", "20150102Z", @@ -84,7 +83,6 @@ class TestFieldPermutations(TestCase): title=None, tags=None, ): - info = FileInfo.from_filename(filename) # Created @@ -131,7 +129,6 @@ class TestFieldPermutations(TestCase): self.assertIsNone(info.created) def test_filename_parse_transforms(self): - filename = "tag1,tag2_20190908_180610_0001.pdf" all_patt = re.compile("^.*$") none_patt = re.compile("$a") @@ -214,7 +211,6 @@ class FaultyParser(DocumentParser): def fake_magic_from_file(file, mime=False): - if mime: if os.path.splitext(file)[1] == ".pdf": return "application/pdf" @@ -239,7 +235,6 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase): last_progress=100, last_progress_max=100, ): - self._send_progress.assert_called() args, kwargs = self._send_progress.call_args_list[0] @@ -314,7 +309,6 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase): @override_settings(FILENAME_FORMAT=None, TIME_ZONE="America/Chicago") def testNormalOperation(self): - filename = self.get_test_file() # Get the local time, as an aware datetime @@ -436,7 +430,6 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self._assert_first_last_send_progress() def testNotAFile(self): - self.assertRaisesMessage( ConsumerError, "File not found", @@ -544,7 +537,6 @@ class TestConsumer(DirectoriesMixin, FileSystemAssertsMixin, TestCase): @override_settings(FILENAME_FORMAT="{correspondent}/{title}") @mock.patch("documents.signals.handlers.generate_unique_filename") def testFilenameHandlingUnstableFormat(self, m): - filenames = ["this", "that", "now this", "i cant decide"] def get_filename(): @@ -791,7 +783,6 @@ class TestConsumerCreatedDate(DirectoriesMixin, TestCase): class PreConsumeTestCase(TestCase): def setUp(self) -> None: - # this prevents websocket message reports during testing. patcher = mock.patch("documents.consumer.Consumer._send_progress") self._send_progress = patcher.start() @@ -899,7 +890,6 @@ class PreConsumeTestCase(TestCase): class PostConsumeTestCase(TestCase): def setUp(self) -> None: - # this prevents websocket message reports during testing. patcher = mock.patch("documents.consumer.Consumer._send_progress") self._send_progress = patcher.start() diff --git a/src/documents/tests/test_date_parsing.py b/src/documents/tests/test_date_parsing.py index e1b179976..a0e59d3e6 100644 --- a/src/documents/tests/test_date_parsing.py +++ b/src/documents/tests/test_date_parsing.py @@ -5,14 +5,14 @@ from uuid import uuid4 from dateutil import tz from django.conf import settings -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from documents.parsers import parse_date from documents.parsers import parse_date_generator class TestDate(TestCase): - SAMPLE_FILES = os.path.join( os.path.dirname(__file__), "../../paperless_tesseract/tests/samples", diff --git a/src/documents/tests/test_document_model.py b/src/documents/tests/test_document_model.py index 763f5049c..ad357e30a 100644 --- a/src/documents/tests/test_document_model.py +++ b/src/documents/tests/test_document_model.py @@ -8,8 +8,8 @@ try: except ImportError: from backports import zoneinfo -from django.test import override_settings from django.test import TestCase +from django.test import override_settings from django.utils import timezone from documents.models import Correspondent @@ -52,7 +52,6 @@ class TestDocument(TestCase): self.assertEqual(mock_unlink.call_count, 2) def test_file_name(self): - doc = Document( mime_type="application/pdf", title="test", @@ -64,7 +63,6 @@ class TestDocument(TestCase): TIME_ZONE="Europe/Berlin", ) def test_file_name_with_timezone(self): - # See https://docs.djangoproject.com/en/4.0/ref/utils/#django.utils.timezone.now # The default for created is an aware datetime in UTC # This does that, just manually, with a fixed date @@ -107,7 +105,6 @@ class TestDocument(TestCase): self.assertEqual(doc.get_public_filename(), "2020-01-01 test.pdf") def test_file_name_jpg(self): - doc = Document( mime_type="image/jpeg", title="test", @@ -116,7 +113,6 @@ class TestDocument(TestCase): self.assertEqual(doc.get_public_filename(), "2020-12-25 test.jpg") def test_file_name_unknown(self): - doc = Document( mime_type="application/zip", title="test", @@ -125,7 +121,6 @@ class TestDocument(TestCase): self.assertEqual(doc.get_public_filename(), "2020-12-25 test.zip") def test_file_name_invalid_type(self): - doc = Document( mime_type="image/jpegasd", title="test", diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index d2f61eb1c..e4d4d0673 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -7,8 +7,8 @@ from unittest import mock from django.conf import settings from django.contrib.auth.models import User from django.db import DatabaseError -from django.test import override_settings from django.test import TestCase +from django.test import override_settings from django.utils import timezone from documents.file_handling import create_source_path_directory @@ -119,7 +119,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}") def test_file_renaming_database_error(self): - Document.objects.create( mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_UNENCRYPTED, @@ -842,7 +841,6 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test @override_settings(FILENAME_FORMAT="{correspondent}/{title}") def test_database_error(self): - original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf") archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf") Path(original).touch() @@ -868,7 +866,6 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test class TestFilenameGeneration(DirectoriesMixin, TestCase): @override_settings(FILENAME_FORMAT="{title}") def test_invalid_characters(self): - doc = Document.objects.create( title="This. is the title.", mime_type="application/pdf", diff --git a/src/documents/tests/test_importer.py b/src/documents/tests/test_importer.py index 8a659dbb6..d86101fd8 100644 --- a/src/documents/tests/test_importer.py +++ b/src/documents/tests/test_importer.py @@ -1,8 +1,13 @@ +import tempfile +from pathlib import Path + +from django.core.management import call_command from django.core.management.base import CommandError from django.test import TestCase -from documents.settings import EXPORTER_FILE_NAME from documents.management.commands.document_importer import Command +from documents.settings import EXPORTER_ARCHIVE_NAME +from documents.settings import EXPORTER_FILE_NAME class TestImporter(TestCase): @@ -14,17 +19,16 @@ class TestImporter(TestCase): self.assertRaises( CommandError, cmd._check_manifest_exists, - "/tmp/manifest.json", + Path("/tmp/manifest.json"), ) def test_check_manifest(self): - cmd = Command() - cmd.source = "/tmp" + cmd.source = Path("/tmp") cmd.manifest = [{"model": "documents.document"}] with self.assertRaises(CommandError) as cm: - cmd._check_manifest() + cmd._check_manifest_valid() self.assertIn("The manifest file contains a record", str(cm.exception)) cmd.manifest = [ @@ -32,8 +36,80 @@ class TestImporter(TestCase): ] # self.assertRaises(CommandError, cmd._check_manifest) with self.assertRaises(CommandError) as cm: - cmd._check_manifest() + cmd._check_manifest_valid() self.assertIn( 'The manifest file refers to "noexist.pdf"', str(cm.exception), ) + + def test_import_permission_error(self): + """ + GIVEN: + - Original file which cannot be read from + - Archive file which cannot be read from + WHEN: + - Import is attempted + THEN: + - CommandError is raised indicating the issue + """ + with tempfile.TemporaryDirectory() as temp_dir: + # Create empty files + original_path = Path(temp_dir) / "original.pdf" + archive_path = Path(temp_dir) / "archive.pdf" + original_path.touch() + archive_path.touch() + + # No read permissions + original_path.chmod(0o222) + + cmd = Command() + cmd.source = Path(temp_dir) + cmd.manifest = [ + { + "model": "documents.document", + EXPORTER_FILE_NAME: "original.pdf", + EXPORTER_ARCHIVE_NAME: "archive.pdf", + }, + ] + with self.assertRaises(CommandError) as cm: + cmd._check_manifest_valid() + self.assertInt("Failed to read from original file", str(cm.exception)) + + original_path.chmod(0o444) + archive_path.chmod(0o222) + + with self.assertRaises(CommandError) as cm: + cmd._check_manifest_valid() + self.assertInt("Failed to read from archive file", str(cm.exception)) + + def test_import_source_not_existing(self): + """ + GIVEN: + - Source given doesn't exist + WHEN: + - Import is attempted + THEN: + - CommandError is raised indicating the issue + """ + with self.assertRaises(CommandError) as cm: + call_command("document_importer", Path("/tmp/notapath")) + self.assertInt("That path doesn't exist", str(cm.exception)) + + def test_import_source_not_readable(self): + """ + GIVEN: + - Source given isn't readable + WHEN: + - Import is attempted + THEN: + - CommandError is raised indicating the issue + """ + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) + path.chmod(0o222) + with self.assertRaises(CommandError) as cm: + call_command("document_importer", path) + self.assertInt( + "That path doesn't appear to be readable", + str(cm.exception), + ) diff --git a/src/documents/tests/test_index.py b/src/documents/tests/test_index.py index bf1865a43..11cbee443 100644 --- a/src/documents/tests/test_index.py +++ b/src/documents/tests/test_index.py @@ -1,6 +1,7 @@ from unittest import mock from django.test import TestCase + from documents import index from documents.models import Document from documents.tests.utils import DirectoriesMixin @@ -8,7 +9,6 @@ from documents.tests.utils import DirectoriesMixin class TestAutoComplete(DirectoriesMixin, TestCase): def test_auto_complete(self): - doc1 = Document.objects.create( title="doc1", checksum="A", diff --git a/src/documents/tests/test_management.py b/src/documents/tests/test_management.py index d5b81a5e1..1115325ca 100644 --- a/src/documents/tests/test_management.py +++ b/src/documents/tests/test_management.py @@ -7,15 +7,15 @@ from pathlib import Path from unittest import mock from django.core.management import call_command -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from documents.file_handling import generate_filename from documents.models import Document from documents.tasks import update_document_archive_file from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin - sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf") @@ -30,7 +30,6 @@ class TestArchiver(DirectoriesMixin, FileSystemAssertsMixin, TestCase): ) def test_archiver(self): - doc = self.make_models() shutil.copy( sample_file, @@ -40,7 +39,6 @@ class TestArchiver(DirectoriesMixin, FileSystemAssertsMixin, TestCase): call_command("document_archiver") def test_handle_document(self): - doc = self.make_models() shutil.copy( sample_file, @@ -114,7 +112,6 @@ class TestDecryptDocuments(FileSystemAssertsMixin, TestCase): ) @mock.patch("documents.management.commands.decrypt_documents.input") def test_decrypt(self, m): - media_dir = tempfile.mkdtemp() originals_dir = os.path.join(media_dir, "documents", "originals") thumb_dir = os.path.join(media_dir, "documents", "thumbnails") diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index 150880116..99d5d410e 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -7,10 +7,11 @@ from time import sleep from unittest import mock from django.conf import settings -from django.core.management import call_command from django.core.management import CommandError -from django.test import override_settings +from django.core.management import call_command from django.test import TransactionTestCase +from django.test import override_settings + from documents.consumer import ConsumerError from documents.data_models import ConsumableDocument from documents.management.commands import document_consumer @@ -149,7 +150,6 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): @mock.patch("documents.management.commands.document_consumer.logger.error") def test_slow_write_pdf(self, error_logger): - self.consume_file_mock.side_effect = self.bogus_task self.t_start() @@ -170,7 +170,6 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): @mock.patch("documents.management.commands.document_consumer.logger.error") def test_slow_write_and_move(self, error_logger): - self.consume_file_mock.side_effect = self.bogus_task self.t_start() @@ -193,7 +192,6 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): @mock.patch("documents.management.commands.document_consumer.logger.error") def test_slow_write_incomplete(self, error_logger): - self.consume_file_mock.side_effect = self.bogus_task self.t_start() @@ -214,12 +212,10 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): @override_settings(CONSUMPTION_DIR="does_not_exist") def test_consumption_directory_invalid(self): - self.assertRaises(CommandError, call_command, "document_consumer", "--oneshot") @override_settings(CONSUMPTION_DIR="") def test_consumption_directory_unset(self): - self.assertRaises(CommandError, call_command, "document_consumer", "--oneshot") def test_mac_write(self): @@ -271,25 +267,11 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): "ignore": False, }, { - "path": os.path.join(self.dirs.consumption_dir, ".DS_STORE", "foo.pdf"), + "path": os.path.join(self.dirs.consumption_dir, ".DS_STORE"), "ignore": True, }, { - "path": os.path.join( - self.dirs.consumption_dir, - "foo", - ".DS_STORE", - "bar.pdf", - ), - "ignore": True, - }, - { - "path": os.path.join( - self.dirs.consumption_dir, - ".DS_STORE", - "foo", - "bar.pdf", - ), + "path": os.path.join(self.dirs.consumption_dir, ".DS_Store"), "ignore": True, }, { @@ -345,7 +327,6 @@ class TestConsumer(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): @mock.patch("documents.management.commands.document_consumer.open") def test_consume_file_busy(self, open_mock): - # Calling this mock always raises this open_mock.side_effect = OSError @@ -391,7 +372,6 @@ class TestConsumerRecursivePolling(TestConsumer): class TestConsumerTags(DirectoriesMixin, ConsumerThreadMixin, TransactionTestCase): @override_settings(CONSUMER_RECURSIVE=True, CONSUMER_SUBDIRS_AS_TAGS=True) def test_consume_file_with_path_tags(self): - tag_names = ("existingTag", "Space Tag") # Create a Tag prior to consuming a file using it in path tag_ids = [ diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 05b6db5b3..e0b0e3543 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -9,9 +9,10 @@ from zipfile import ZipFile from django.core.management import call_command from django.core.management.base import CommandError -from django.test import override_settings from django.test import TestCase +from django.test import override_settings from django.utils import timezone + from documents.management.commands import document_exporter from documents.models import Correspondent from documents.models import Document @@ -212,7 +213,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): Tag.objects.all().delete() self.assertEqual(Document.objects.count(), 0) - call_command("document_importer", self.target) + call_command("document_importer", "--no-progress-bar", self.target) self.assertEqual(Document.objects.count(), 4) self.assertEqual(Tag.objects.count(), 1) self.assertEqual(Correspondent.objects.count(), 1) @@ -363,7 +364,6 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): ) def test_export_missing_files(self): - target = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, target) Document.objects.create( @@ -457,7 +457,6 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): args = ["document_exporter", "/tmp/foo/bar"] with self.assertRaises(CommandError) as e: - call_command(*args) self.assertEqual("That path isn't a directory", str(e)) @@ -473,11 +472,9 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): """ with tempfile.NamedTemporaryFile() as tmp_file: - args = ["document_exporter", tmp_file.name] with self.assertRaises(CommandError) as e: - call_command(*args) self.assertEqual("That path isn't a directory", str(e)) @@ -492,13 +489,11 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): - Error is raised """ with tempfile.TemporaryDirectory() as tmp_dir: - os.chmod(tmp_dir, 0o000) args = ["document_exporter", tmp_dir] with self.assertRaises(CommandError) as e: - call_command(*args) self.assertEqual("That path doesn't appear to be writable", str(e)) @@ -541,7 +536,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(Document.objects.count(), 4) Document.objects.all().delete() self.assertEqual(Document.objects.count(), 0) - call_command("document_importer", self.target) + call_command("document_importer", "--no-progress-bar", self.target) self.assertEqual(Document.objects.count(), 4) def test_no_thumbnail(self): @@ -584,7 +579,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(Document.objects.count(), 4) Document.objects.all().delete() self.assertEqual(Document.objects.count(), 0) - call_command("document_importer", self.target) + call_command("document_importer", "--no-progress-bar", self.target) self.assertEqual(Document.objects.count(), 4) def test_split_manifest(self): @@ -613,7 +608,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(Document.objects.count(), 4) Document.objects.all().delete() self.assertEqual(Document.objects.count(), 0) - call_command("document_importer", self.target) + call_command("document_importer", "--no-progress-bar", self.target) self.assertEqual(Document.objects.count(), 4) def test_folder_prefix(self): @@ -637,5 +632,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(Document.objects.count(), 4) Document.objects.all().delete() self.assertEqual(Document.objects.count(), 0) - call_command("document_importer", self.target) + call_command("document_importer", "--no-progress-bar", self.target) self.assertEqual(Document.objects.count(), 4) diff --git a/src/documents/tests/test_management_retagger.py b/src/documents/tests/test_management_retagger.py index 2b7aae649..75e5f7dee 100644 --- a/src/documents/tests/test_management_retagger.py +++ b/src/documents/tests/test_management_retagger.py @@ -1,5 +1,6 @@ from django.core.management import call_command from django.test import TestCase + from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType @@ -10,7 +11,6 @@ from documents.tests.utils import DirectoriesMixin class TestRetagger(DirectoriesMixin, TestCase): def make_models(self): - self.sp1 = StoragePath.objects.create( name="dummy a", path="{created_data}/{title}", diff --git a/src/documents/tests/test_management_superuser.py b/src/documents/tests/test_management_superuser.py index b79a9f9a6..01f03c8e1 100644 --- a/src/documents/tests/test_management_superuser.py +++ b/src/documents/tests/test_management_superuser.py @@ -5,6 +5,7 @@ from unittest import mock from django.contrib.auth.models import User from django.core.management import call_command from django.test import TestCase + from documents.tests.utils import DirectoriesMixin diff --git a/src/documents/tests/test_management_thumbnails.py b/src/documents/tests/test_management_thumbnails.py index 0767e4e37..6e4e18ea3 100644 --- a/src/documents/tests/test_management_thumbnails.py +++ b/src/documents/tests/test_management_thumbnails.py @@ -4,6 +4,7 @@ from unittest import mock from django.core.management import call_command from django.test import TestCase + from documents.management.commands.document_thumbnails import _process_document from documents.models import Document from documents.tests.utils import DirectoriesMixin diff --git a/src/documents/tests/test_matchables.py b/src/documents/tests/test_matchables.py index 56d47ee46..4dac4f58f 100644 --- a/src/documents/tests/test_matchables.py +++ b/src/documents/tests/test_matchables.py @@ -5,8 +5,8 @@ from typing import Iterable from django.contrib.admin.models import LogEntry from django.contrib.auth.models import User -from django.test import override_settings from django.test import TestCase +from django.test import override_settings from documents import matching from documents.models import Correspondent @@ -48,7 +48,6 @@ class _TestMatchingBase(TestCase): class TestMatching(_TestMatchingBase): def test_match_none(self): - self._test_matching( "", "MATCH_NONE", @@ -60,7 +59,6 @@ class TestMatching(_TestMatchingBase): ) def test_match_all(self): - self._test_matching( "alpha charlie gamma", "MATCH_ALL", @@ -107,7 +105,6 @@ class TestMatching(_TestMatchingBase): ) def test_match_any(self): - self._test_matching( "alpha charlie gamma", "MATCH_ANY", @@ -152,7 +149,6 @@ class TestMatching(_TestMatchingBase): ) def test_match_literal(self): - self._test_matching( "alpha charlie gamma", "MATCH_LITERAL", @@ -187,7 +183,6 @@ class TestMatching(_TestMatchingBase): ) def test_match_regex(self): - self._test_matching( r"alpha\w+gamma", "MATCH_REGEX", @@ -211,7 +206,6 @@ class TestMatching(_TestMatchingBase): self._test_matching("[", "MATCH_REGEX", [], ["Don't match this"]) def test_match_fuzzy(self): - self._test_matching( "Springfield, Miss.", "MATCH_FUZZY", @@ -331,7 +325,6 @@ class TestCaseSensitiveMatching(_TestMatchingBase): ) def test_match_literal(self): - self._test_matching( "alpha charlie gamma", "MATCH_LITERAL", diff --git a/src/documents/tests/test_migration_archive_files.py b/src/documents/tests/test_migration_archive_files.py index 32929d92c..ca20b558f 100644 --- a/src/documents/tests/test_migration_archive_files.py +++ b/src/documents/tests/test_migration_archive_files.py @@ -6,12 +6,12 @@ from unittest import mock from django.conf import settings from django.test import override_settings + from documents.parsers import ParseError from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import TestMigrations - STORAGE_TYPE_GPG = "gpg" @@ -114,7 +114,6 @@ simple_png2 = os.path.join(os.path.dirname(__file__), "examples", "no-text.png") @override_settings(FILENAME_FORMAT="") class TestMigrateArchiveFiles(DirectoriesMixin, FileSystemAssertsMixin, TestMigrations): - migrate_from = "1011_auto_20210101_2340" migrate_to = "1012_fix_archive_files" @@ -282,13 +281,11 @@ def fake_parse_wrapper(parser, path, mime_type, file_name): @override_settings(FILENAME_FORMAT="") class TestMigrateArchiveFilesErrors(DirectoriesMixin, TestMigrations): - migrate_from = "1011_auto_20210101_2340" migrate_to = "1012_fix_archive_files" auto_migrate = False def test_archive_missing(self): - Document = self.apps.get_model("documents", "Document") doc = make_test_document( @@ -454,12 +451,10 @@ class TestMigrateArchiveFilesBackwards( FileSystemAssertsMixin, TestMigrations, ): - migrate_from = "1012_fix_archive_files" migrate_to = "1011_auto_20210101_2340" def setUpBeforeMigration(self, apps): - Document = apps.get_model("documents", "Document") make_test_document( @@ -519,13 +514,11 @@ class TestMigrateArchiveFilesBackwardsWithFilenameFormat( @override_settings(FILENAME_FORMAT="") class TestMigrateArchiveFilesBackwardsErrors(DirectoriesMixin, TestMigrations): - migrate_from = "1012_fix_archive_files" migrate_to = "1011_auto_20210101_2340" auto_migrate = False def test_filename_clash(self): - Document = self.apps.get_model("documents", "Document") self.clashA = make_test_document( @@ -554,7 +547,6 @@ class TestMigrateArchiveFilesBackwardsErrors(DirectoriesMixin, TestMigrations): ) def test_filename_exists(self): - Document = self.apps.get_model("documents", "Document") self.clashA = make_test_document( diff --git a/src/documents/tests/test_migration_mime_type.py b/src/documents/tests/test_migration_mime_type.py index 7b607a0cf..63892eb79 100644 --- a/src/documents/tests/test_migration_mime_type.py +++ b/src/documents/tests/test_migration_mime_type.py @@ -3,6 +3,7 @@ import shutil from django.conf import settings from django.test import override_settings + from documents.parsers import get_default_file_extension from documents.tests.utils import DirectoriesMixin from documents.tests.utils import TestMigrations @@ -39,7 +40,6 @@ def source_path_after(doc): @override_settings(PASSPHRASE="test") class TestMigrateMimeType(DirectoriesMixin, TestMigrations): - migrate_from = "1002_auto_20201111_1105" migrate_to = "1003_mime_types" @@ -85,7 +85,6 @@ class TestMigrateMimeType(DirectoriesMixin, TestMigrations): @override_settings(PASSPHRASE="test") class TestMigrateMimeTypeBackwards(DirectoriesMixin, TestMigrations): - migrate_from = "1003_mime_types" migrate_to = "1002_auto_20201111_1105" diff --git a/src/documents/tests/test_migration_remove_null_characters.py b/src/documents/tests/test_migration_remove_null_characters.py index 09fd80883..c47bc80ca 100644 --- a/src/documents/tests/test_migration_remove_null_characters.py +++ b/src/documents/tests/test_migration_remove_null_characters.py @@ -3,7 +3,6 @@ from documents.tests.utils import TestMigrations class TestMigrateNullCharacters(DirectoriesMixin, TestMigrations): - migrate_from = "1014_auto_20210228_1614" migrate_to = "1015_remove_null_characters" diff --git a/src/documents/tests/test_migration_tag_colors.py b/src/documents/tests/test_migration_tag_colors.py index 6cc2fa3f7..0643fe883 100644 --- a/src/documents/tests/test_migration_tag_colors.py +++ b/src/documents/tests/test_migration_tag_colors.py @@ -3,7 +3,6 @@ from documents.tests.utils import TestMigrations class TestMigrateTagColor(DirectoriesMixin, TestMigrations): - migrate_from = "1012_fix_archive_files" migrate_to = "1013_migrate_tag_colour" @@ -21,7 +20,6 @@ class TestMigrateTagColor(DirectoriesMixin, TestMigrations): class TestMigrateTagColorBackwards(DirectoriesMixin, TestMigrations): - migrate_from = "1013_migrate_tag_colour" migrate_to = "1012_fix_archive_files" diff --git a/src/documents/tests/test_migration_webp_conversion.py b/src/documents/tests/test_migration_webp_conversion.py index 9631c5704..db5fd83af 100644 --- a/src/documents/tests/test_migration_webp_conversion.py +++ b/src/documents/tests/test_migration_webp_conversion.py @@ -7,6 +7,7 @@ from typing import Union from unittest import mock from django.test import override_settings + from documents.tests.utils import TestMigrations @@ -15,7 +16,6 @@ from documents.tests.utils import TestMigrations ) @mock.patch("documents.migrations.1021_webp_thumbnail_conversion.run_convert") class TestMigrateWebPThumbnails(TestMigrations): - migrate_from = "1020_merge_20220518_1839" migrate_to = "1021_webp_thumbnail_conversion" auto_migrate = False @@ -104,13 +104,11 @@ class TestMigrateWebPThumbnails(TestMigrations): self.assert_file_count_by_extension("webp", dir, expected_count) def setUp(self): - self.thumbnail_dir = Path(tempfile.mkdtemp()).resolve() return super().setUp() def tearDown(self) -> None: - shutil.rmtree(self.thumbnail_dir) return super().tearDown() @@ -133,7 +131,6 @@ class TestMigrateWebPThumbnails(TestMigrations): with override_settings( THUMBNAIL_DIR=self.thumbnail_dir, ): - self.create_webp_thumbnail_files(self.thumbnail_dir, 3) self.performMigration() @@ -188,7 +185,6 @@ class TestMigrateWebPThumbnails(TestMigrations): with override_settings( THUMBNAIL_DIR=self.thumbnail_dir, ): - self.create_png_thumbnail_file(self.thumbnail_dir, 3) self.performMigration() @@ -217,7 +213,6 @@ class TestMigrateWebPThumbnails(TestMigrations): with override_settings( THUMBNAIL_DIR=self.thumbnail_dir, ): - self.create_png_thumbnail_file(self.thumbnail_dir, 3) self.create_webp_thumbnail_files(self.thumbnail_dir, 2, start_count=3) diff --git a/src/documents/tests/test_models.py b/src/documents/tests/test_models.py index ee882dd84..f41214a65 100644 --- a/src/documents/tests/test_models.py +++ b/src/documents/tests/test_models.py @@ -2,6 +2,7 @@ from django.test import TestCase from documents.models import Correspondent from documents.models import Document + from .factories import CorrespondentFactory from .factories import DocumentFactory @@ -15,7 +16,6 @@ class CorrespondentTestCase(TestCase): class DocumentTestCase(TestCase): def test_correspondent_deletion_does_not_cascade(self): - self.assertEqual(Correspondent.objects.all().count(), 0) correspondent = CorrespondentFactory.create() self.assertEqual(Correspondent.objects.all().count(), 1) diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index 7ec06d1a0..fee7234e8 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -2,8 +2,9 @@ from tempfile import TemporaryDirectory from unittest import mock from django.apps import apps -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from documents.parsers import get_default_file_extension from documents.parsers import get_parser_class_for_mime_type from documents.parsers import get_supported_file_extensions diff --git a/src/documents/tests/test_sanity_check.py b/src/documents/tests/test_sanity_check.py index 5fb4adfcc..2f4024762 100644 --- a/src/documents/tests/test_sanity_check.py +++ b/src/documents/tests/test_sanity_check.py @@ -6,6 +6,7 @@ from pathlib import Path import filelock from django.conf import settings from django.test import TestCase + from documents.models import Document from documents.sanity_checker import check_sanity from documents.tests.utils import DirectoriesMixin @@ -13,7 +14,6 @@ from documents.tests.utils import DirectoriesMixin class TestSanityCheck(DirectoriesMixin, TestCase): def make_test_data(self): - with filelock.FileLock(settings.MEDIA_LOCK): # just make sure that the lockfile is present. shutil.copy( diff --git a/src/documents/tests/test_task_signals.py b/src/documents/tests/test_task_signals.py index d63df0b3c..4a54220e0 100644 --- a/src/documents/tests/test_task_signals.py +++ b/src/documents/tests/test_task_signals.py @@ -3,13 +3,14 @@ from unittest import mock import celery from django.test import TestCase + from documents.data_models import ConsumableDocument from documents.data_models import DocumentSource from documents.models import PaperlessTask from documents.signals.handlers import before_task_publish_handler +from documents.signals.handlers import task_failure_handler from documents.signals.handlers import task_postrun_handler from documents.signals.handlers import task_prerun_handler -from documents.signals.handlers import task_failure_handler from documents.tests.test_consumer import fake_magic_from_file from documents.tests.utils import DirectoriesMixin diff --git a/src/documents/tests/test_tasks.py b/src/documents/tests/test_tasks.py index 2d27f93ec..334345b6a 100644 --- a/src/documents/tests/test_tasks.py +++ b/src/documents/tests/test_tasks.py @@ -4,6 +4,7 @@ from unittest import mock from django.conf import settings from django.test import TestCase from django.utils import timezone + from documents import tasks from documents.models import Correspondent from documents.models import Document diff --git a/src/documents/tests/test_views.py b/src/documents/tests/test_views.py index a8c6a67da..d84d7759b 100644 --- a/src/documents/tests/test_views.py +++ b/src/documents/tests/test_views.py @@ -3,8 +3,8 @@ import tempfile from django.conf import settings from django.contrib.auth.models import User -from django.test import override_settings from django.test import TestCase +from django.test import override_settings from rest_framework import status @@ -34,7 +34,7 @@ class TestViews(TestCase): def test_index(self): self.client.force_login(self.user) - for (language_given, language_actual) in [ + for language_given, language_actual in [ ("", "en-US"), ("en-US", "en-US"), ("de", "de-DE"), diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index 26760f780..fbde3345c 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -12,14 +12,14 @@ from unittest import mock from django.apps import apps from django.db import connection from django.db.migrations.executor import MigrationExecutor -from django.test import override_settings from django.test import TransactionTestCase +from django.test import override_settings + from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides def setup_directories(): - dirs = namedtuple("Dirs", ()) dirs.data_dir = Path(tempfile.mkdtemp()) diff --git a/src/documents/views.py b/src/documents/views.py index 1edbdccc3..234c4dda1 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -26,21 +26,15 @@ from django.db.models.functions import Lower from django.http import Http404 from django.http import HttpResponse from django.http import HttpResponseBadRequest +from django.http import HttpResponseForbidden from django.shortcuts import get_object_or_404 from django.utils.decorators import method_decorator from django.utils.translation import get_language from django.views.decorators.cache import cache_control from django.views.generic import TemplateView from django_filters.rest_framework import DjangoFilterBackend -from documents.filters import ObjectOwnedOrGrantedPermissionsFilter -from documents.permissions import PaperlessAdminPermissions -from documents.permissions import PaperlessObjectPermissions -from documents.tasks import consume_file from langdetect import detect from packaging import version as packaging_version -from paperless import version -from paperless.db import GnuPG -from paperless.views import StandardPagination from rest_framework import parsers from rest_framework.decorators import action from rest_framework.exceptions import NotFound @@ -60,6 +54,16 @@ from rest_framework.viewsets import ModelViewSet from rest_framework.viewsets import ReadOnlyModelViewSet from rest_framework.viewsets import ViewSet +from documents.filters import ObjectOwnedOrGrantedPermissionsFilter +from documents.permissions import PaperlessAdminPermissions +from documents.permissions import PaperlessObjectPermissions +from documents.permissions import get_objects_for_user_owner_aware +from documents.permissions import has_perms_owner_aware +from documents.tasks import consume_file +from paperless import version +from paperless.db import GnuPG +from paperless.views import StandardPagination + from .bulk_download import ArchiveOnlyStrategy from .bulk_download import OriginalAndArchiveStrategy from .bulk_download import OriginalsOnlyStrategy @@ -153,6 +157,10 @@ class PassUserMixin(CreateModelMixin): def get_serializer(self, *args, **kwargs): kwargs.setdefault("user", self.request.user) + kwargs.setdefault( + "full_perms", + self.request.query_params.get("full_perms", False), + ) return super().get_serializer(*args, **kwargs) @@ -190,7 +198,6 @@ class TagViewSet(ModelViewSet, PassUserMixin): ) def get_serializer_class(self, *args, **kwargs): - print(self.request.version) if int(self.request.version) == 1: return TagSerializerVersion1 else: @@ -271,6 +278,10 @@ class DocumentViewSet( kwargs.setdefault("context", self.get_serializer_context()) kwargs.setdefault("fields", fields) kwargs.setdefault("truncate_content", truncate_content.lower() in ["true", "1"]) + kwargs.setdefault( + "full_perms", + self.request.query_params.get("full_perms", False), + ) return serializer_class(*args, **kwargs) def update(self, request, *args, **kwargs): @@ -295,6 +306,12 @@ class DocumentViewSet( def file_response(self, pk, request, disposition): doc = Document.objects.get(id=pk) + if request.user is not None and not has_perms_owner_aware( + request.user, + "view_document", + doc, + ): + return HttpResponseForbidden("Insufficient permissions") if not self.original_requested(request) and doc.has_archive_version: file_handle = doc.archive_file filename = doc.get_public_filename(archive=True) @@ -354,6 +371,12 @@ class DocumentViewSet( def metadata(self, request, pk=None): try: doc = Document.objects.get(pk=pk) + if request.user is not None and not has_perms_owner_aware( + request.user, + "view_document", + doc, + ): + return HttpResponseForbidden("Insufficient permissions") except Document.DoesNotExist: raise Http404 @@ -391,6 +414,12 @@ class DocumentViewSet( @action(methods=["get"], detail=True) def suggestions(self, request, pk=None): doc = get_object_or_404(Document, pk=pk) + if request.user is not None and not has_perms_owner_aware( + request.user, + "view_document", + doc, + ): + return HttpResponseForbidden("Insufficient permissions") classifier = load_classifier() @@ -430,6 +459,12 @@ class DocumentViewSet( def thumb(self, request, pk=None): try: doc = Document.objects.get(id=pk) + if request.user is not None and not has_perms_owner_aware( + request.user, + "view_document", + doc, + ): + return HttpResponseForbidden("Insufficient permissions") if doc.storage_type == Document.STORAGE_TYPE_GPG: handle = GnuPG.decrypted(doc.thumbnail_file) else: @@ -468,6 +503,12 @@ class DocumentViewSet( def notes(self, request, pk=None): try: doc = Document.objects.get(pk=pk) + if request.user is not None and not has_perms_owner_aware( + request.user, + "view_document", + doc, + ): + return HttpResponseForbidden("Insufficient permissions") except Document.DoesNotExist: raise Http404 @@ -597,7 +638,6 @@ class UnifiedSearchViewSet(DocumentViewSet): class LogViewSet(ViewSet): - permission_classes = (IsAuthenticated, PaperlessAdminPermissions) log_files = ["paperless", "mail"] @@ -643,7 +683,6 @@ class SavedViewViewSet(ModelViewSet, PassUserMixin): class BulkEditView(GenericAPIView): - permission_classes = (IsAuthenticated,) serializer_class = BulkEditSerializer parser_classes = (parsers.JSONParser,) @@ -665,13 +704,11 @@ class BulkEditView(GenericAPIView): class PostDocumentView(GenericAPIView): - permission_classes = (IsAuthenticated,) serializer_class = PostDocumentSerializer parser_classes = (parsers.MultiPartParser,) def post(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) @@ -719,7 +756,6 @@ class PostDocumentView(GenericAPIView): class SelectionDataView(GenericAPIView): - permission_classes = (IsAuthenticated,) serializer_class = DocumentListSerializer parser_classes = (parsers.MultiPartParser, parsers.JSONParser) @@ -777,7 +813,6 @@ class SelectionDataView(GenericAPIView): class SearchAutoCompleteView(APIView): - permission_classes = (IsAuthenticated,) def get(self, request, format=None): @@ -801,22 +836,38 @@ class SearchAutoCompleteView(APIView): class StatisticsView(APIView): - permission_classes = (IsAuthenticated,) def get(self, request, format=None): - documents_total = Document.objects.all().count() + user = request.user if request.user is not None else None - inbox_tag = Tag.objects.filter(is_inbox_tag=True) + documents = ( + Document.objects.all() + if user is None + else get_objects_for_user_owner_aware( + user, + "documents.view_document", + Document, + ) + ) + tags = ( + Tag.objects.all() + if user is None + else get_objects_for_user_owner_aware(user, "documents.view_tag", Tag) + ) + + documents_total = documents.count() + + inbox_tag = tags.filter(is_inbox_tag=True) documents_inbox = ( - Document.objects.filter(tags__is_inbox_tag=True).distinct().count() + documents.filter(tags__is_inbox_tag=True).distinct().count() if inbox_tag.exists() else None ) document_file_type_counts = ( - Document.objects.values("mime_type") + documents.values("mime_type") .annotate(mime_type_count=Count("mime_type")) .order_by("-mime_type_count") if documents_total > 0 @@ -824,7 +875,7 @@ class StatisticsView(APIView): ) character_count = ( - Document.objects.annotate( + documents.annotate( characters=Length("content"), ) .aggregate(Sum("characters")) @@ -843,7 +894,6 @@ class StatisticsView(APIView): class BulkDownloadView(GenericAPIView): - permission_classes = (IsAuthenticated,) serializer_class = BulkDownloadSerializer parser_classes = (parsers.JSONParser,) @@ -938,13 +988,16 @@ class StoragePathViewSet(ModelViewSet, PassUserMixin): serializer_class = StoragePathSerializer pagination_class = StandardPagination permission_classes = (IsAuthenticated, PaperlessObjectPermissions) - filter_backends = (DjangoFilterBackend, OrderingFilter) + filter_backends = ( + DjangoFilterBackend, + OrderingFilter, + ObjectOwnedOrGrantedPermissionsFilter, + ) filterset_class = StoragePathFilterSet ordering_fields = ("name", "path", "matching_algorithm", "match", "document_count") class UiSettingsView(GenericAPIView): - permission_classes = (IsAuthenticated,) serializer_class = UiSettingsViewSerializer @@ -993,7 +1046,6 @@ class UiSettingsView(GenericAPIView): class TasksViewSet(ReadOnlyModelViewSet): - permission_classes = (IsAuthenticated,) serializer_class = TasksViewSerializer @@ -1012,7 +1064,6 @@ class TasksViewSet(ReadOnlyModelViewSet): class AcknowledgeTasksView(GenericAPIView): - permission_classes = (IsAuthenticated,) serializer_class = AcknowledgeTasksViewSerializer diff --git a/src/manage.py b/src/manage.py index e708eaba6..9e5a74758 100755 --- a/src/manage.py +++ b/src/manage.py @@ -3,7 +3,6 @@ import os import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings") from django.core.management import execute_from_command_line diff --git a/src/paperless/apps.py b/src/paperless/apps.py index 323099745..b4147a2e3 100644 --- a/src/paperless/apps.py +++ b/src/paperless/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig from django.utils.translation import gettext_lazy as _ + from paperless.signals import handle_failed_login diff --git a/src/paperless/asgi.py b/src/paperless/asgi.py index 633c75ce0..8d63c347a 100644 --- a/src/paperless/asgi.py +++ b/src/paperless/asgi.py @@ -10,7 +10,8 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings") django_asgi_app = get_asgi_application() from channels.auth import AuthMiddlewareStack # noqa: E402 -from channels.routing import ProtocolTypeRouter, URLRouter # noqa: E402 +from channels.routing import ProtocolTypeRouter # noqa: E402 +from channels.routing import URLRouter # noqa: E402 from paperless.urls import websocket_urlpatterns # noqa: E402 diff --git a/src/paperless/auth.py b/src/paperless/auth.py index 368d3e47b..ec853743e 100644 --- a/src/paperless/auth.py +++ b/src/paperless/auth.py @@ -10,7 +10,11 @@ class AutoLoginMiddleware(MiddlewareMixin): def process_request(self, request): try: request.user = User.objects.get(username=settings.AUTO_LOGIN_USERNAME) - auth.login(request, request.user) + auth.login( + request=request, + user=request.user, + backend="django.contrib.auth.backends.ModelBackend", + ) except User.DoesNotExist: pass diff --git a/src/paperless/checks.py b/src/paperless/checks.py index b6a931ecc..cda14baad 100644 --- a/src/paperless/checks.py +++ b/src/paperless/checks.py @@ -6,8 +6,8 @@ import stat from django.conf import settings from django.core.checks import Error -from django.core.checks import register from django.core.checks import Warning +from django.core.checks import register exists_message = "{} is set but doesn't exist." exists_hint = "Create a directory at {}" diff --git a/src/paperless/db.py b/src/paperless/db.py index 51184750d..286ccb094 100644 --- a/src/paperless/db.py +++ b/src/paperless/db.py @@ -11,7 +11,6 @@ class GnuPG: @classmethod def decrypted(cls, file_handle, passphrase=None): - if not passphrase: passphrase = settings.PASSPHRASE diff --git a/src/paperless/filters.py b/src/paperless/filters.py index 5be7cc2f7..a3c09d50f 100644 --- a/src/paperless/filters.py +++ b/src/paperless/filters.py @@ -1,6 +1,7 @@ from django.contrib.auth.models import Group from django.contrib.auth.models import User from django_filters.rest_framework import FilterSet + from documents.filters import CHAR_KWARGS diff --git a/src/paperless/middleware.py b/src/paperless/middleware.py index ddf12812b..cc54e892d 100644 --- a/src/paperless/middleware.py +++ b/src/paperless/middleware.py @@ -1,4 +1,5 @@ from django.conf import settings + from paperless import version diff --git a/src/paperless/serialisers.py b/src/paperless/serialisers.py index 54bc2b353..4094a6538 100644 --- a/src/paperless/serialisers.py +++ b/src/paperless/serialisers.py @@ -17,7 +17,6 @@ class ObfuscatedUserPasswordField(serializers.Field): class UserSerializer(serializers.ModelSerializer): - password = ObfuscatedUserPasswordField(required=False) user_permissions = serializers.SlugRelatedField( many=True, @@ -85,7 +84,6 @@ class UserSerializer(serializers.ModelSerializer): class GroupSerializer(serializers.ModelSerializer): - permissions = serializers.SlugRelatedField( many=True, queryset=Permission.objects.all(), diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 4930a168c..e556e6293 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -298,7 +298,7 @@ REST_FRAMEWORK = { "DEFAULT_VERSION": "1", # Make sure these are ordered and that the most recent version appears # last - "ALLOWED_VERSIONS": ["1", "2"], + "ALLOWED_VERSIONS": ["1", "2", "3"], } if DEBUG: @@ -719,7 +719,7 @@ CONSUMER_IGNORE_PATTERNS = list( json.loads( os.getenv( "PAPERLESS_CONSUMER_IGNORE_PATTERNS", - '[".DS_STORE/*", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]', # noqa: E501 + '[".DS_Store", ".DS_STORE", "._*", ".stfolder/*", ".stversions/*", ".localized/*", "desktop.ini", "@eaDir/*"]', # noqa: E501 ), ), ) diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py index 7c233de23..cd706c532 100644 --- a/src/paperless/tests/test_checks.py +++ b/src/paperless/tests/test_checks.py @@ -1,7 +1,8 @@ import os -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from documents.tests.utils import DirectoriesMixin from paperless.checks import binaries_check from paperless.checks import debug_mode_check diff --git a/src/paperless/tests/test_settings.py b/src/paperless/tests/test_settings.py index 44ca46ec4..6b69765dd 100644 --- a/src/paperless/tests/test_settings.py +++ b/src/paperless/tests/test_settings.py @@ -1,9 +1,10 @@ import datetime import os -from unittest import mock from unittest import TestCase +from unittest import mock from celery.schedules import crontab + from paperless.settings import _parse_beat_schedule from paperless.settings import _parse_ignore_dates from paperless.settings import _parse_redis_url @@ -23,7 +24,6 @@ class TestIgnoreDateParsing(TestCase): test_cases (_type_): _description_ """ for env_str, date_format, expected_date_set in test_cases: - self.assertSetEqual( _parse_ignore_dates(env_str, date_format), expected_date_set, diff --git a/src/paperless/tests/test_signals.py b/src/paperless/tests/test_signals.py index 1a4d7892f..0b1ca1b22 100644 --- a/src/paperless/tests/test_signals.py +++ b/src/paperless/tests/test_signals.py @@ -1,5 +1,6 @@ from django.http import HttpRequest from django.test import TestCase + from paperless.signals import handle_failed_login diff --git a/src/paperless/tests/test_websockets.py b/src/paperless/tests/test_websockets.py index cebbddf39..bf838821a 100644 --- a/src/paperless/tests/test_websockets.py +++ b/src/paperless/tests/test_websockets.py @@ -2,10 +2,10 @@ from unittest import mock from channels.layers import get_channel_layer from channels.testing import WebsocketCommunicator -from django.test import override_settings from django.test import TestCase -from paperless.asgi import application +from django.test import override_settings +from paperless.asgi import application TEST_CHANNEL_LAYERS = { "default": { diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 217268508..c2b72d7b9 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -7,6 +7,9 @@ from django.urls import re_path from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt from django.views.generic import RedirectView +from rest_framework.authtoken import views +from rest_framework.routers import DefaultRouter + from documents.views import AcknowledgeTasksView from documents.views import BulkDownloadView from documents.views import BulkEditView @@ -32,8 +35,6 @@ from paperless.views import UserViewSet from paperless_mail.views import MailAccountTestView from paperless_mail.views import MailAccountViewSet from paperless_mail.views import MailRuleViewSet -from rest_framework.authtoken import views -from rest_framework.routers import DefaultRouter api_router = DefaultRouter() api_router.register(r"correspondents", CorrespondentViewSet) diff --git a/src/paperless/views.py b/src/paperless/views.py index 00945e5fb..588b534e3 100644 --- a/src/paperless/views.py +++ b/src/paperless/views.py @@ -6,15 +6,16 @@ from django.db.models.functions import Lower from django.http import HttpResponse from django.views.generic import View from django_filters.rest_framework import DjangoFilterBackend +from rest_framework.filters import OrderingFilter +from rest_framework.pagination import PageNumberPagination +from rest_framework.permissions import IsAuthenticated +from rest_framework.viewsets import ModelViewSet + from documents.permissions import PaperlessObjectPermissions from paperless.filters import GroupFilterSet from paperless.filters import UserFilterSet from paperless.serialisers import GroupSerializer from paperless.serialisers import UserSerializer -from rest_framework.filters import OrderingFilter -from rest_framework.pagination import PageNumberPagination -from rest_framework.permissions import IsAuthenticated -from rest_framework.viewsets import ModelViewSet class StandardPagination(PageNumberPagination): diff --git a/src/paperless_mail/admin.py b/src/paperless_mail/admin.py index 07582cc0f..b035d14e4 100644 --- a/src/paperless_mail/admin.py +++ b/src/paperless_mail/admin.py @@ -1,6 +1,7 @@ from django import forms from django.contrib import admin from django.utils.translation import gettext_lazy as _ + from paperless_mail.models import MailAccount from paperless_mail.models import MailRule from paperless_mail.models import ProcessedMail @@ -31,7 +32,6 @@ class MailAccountAdminForm(forms.ModelForm): class MailAccountAdmin(admin.ModelAdmin): - list_display = ("name", "imap_server", "username") fieldsets = [ @@ -46,7 +46,6 @@ class MailAccountAdmin(admin.ModelAdmin): class MailRuleAdmin(admin.ModelAdmin): - radio_fields = { "attachment_type": admin.VERTICAL, "action": admin.VERTICAL, @@ -121,7 +120,6 @@ class MailRuleAdmin(admin.ModelAdmin): class ProcessedMailAdmin(admin.ModelAdmin): class Meta: - model = ProcessedMail fields = "__all__" diff --git a/src/paperless_mail/apps.py b/src/paperless_mail/apps.py index 719400e76..a07a059b4 100644 --- a/src/paperless_mail/apps.py +++ b/src/paperless_mail/apps.py @@ -1,6 +1,7 @@ from django.apps import AppConfig from django.conf import settings from django.utils.translation import gettext_lazy as _ + from paperless_mail.signals import mail_consumer_declaration diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index d792b5a97..a40f7e0d1 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -21,6 +21,16 @@ from django.conf import settings from django.db import DatabaseError from django.utils.timezone import is_naive from django.utils.timezone import make_aware +from imap_tools import AND +from imap_tools import NOT +from imap_tools import MailBox +from imap_tools import MailboxFolderSelectError +from imap_tools import MailBoxUnencrypted +from imap_tools import MailMessage +from imap_tools import MailMessageFlags +from imap_tools.mailbox import MailBoxTls +from imap_tools.query import LogicOperator + from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource @@ -28,15 +38,6 @@ from documents.loggers import LoggingMixin from documents.models import Correspondent from documents.parsers import is_mime_type_supported from documents.tasks import consume_file -from imap_tools import AND -from imap_tools import MailBox -from imap_tools import MailboxFolderSelectError -from imap_tools import MailBoxUnencrypted -from imap_tools import MailMessage -from imap_tools import MailMessageFlags -from imap_tools import NOT -from imap_tools.mailbox import MailBoxTls -from imap_tools.query import LogicOperator from paperless_mail.models import MailAccount from paperless_mail.models import MailRule from paperless_mail.models import ProcessedMail @@ -145,10 +146,8 @@ class TagMailAction(BaseMailAction): """ def __init__(self, parameter): - # The custom tag should look like "apple:" if "apple:" in parameter.lower(): - _, self.color = parameter.split(":") self.color = self.color.strip() @@ -162,7 +161,6 @@ class TagMailAction(BaseMailAction): self.color = None def get_criteria(self): - # AppleMail: We only need to check if mails are \Flagged if self.color: return {"flagged": False} @@ -177,7 +175,6 @@ class TagMailAction(BaseMailAction): # AppleMail elif self.color: - # Remove all existing $MailFlagBits M.flag( message_uid, @@ -204,7 +201,6 @@ def mailbox_login(mailbox: MailBox, account: MailAccount): logger = logging.getLogger("paperless_mail") try: - if account.is_token: mailbox.xoauth2(account.username, account.password) else: @@ -252,7 +248,6 @@ def apply_mail_action( message_date = make_aware(message_date) try: - action = get_rule_action(rule) with get_mailbox( @@ -381,7 +376,8 @@ def make_criterias(rule): rule_query = get_rule_action(rule).get_criteria() if isinstance(rule_query, dict): - return AND(**rule_query, **criterias) + if len(rule_query) or len(criterias): + return AND(**rule_query, **criterias) else: return AND(rule_query, **criterias) @@ -475,7 +471,6 @@ class MailAccountHandler(LoggingMixin): account.imap_port, account.imap_security, ) as M: - supports_gmail_labels = "X-GM-EXT-1" in M.client.capabilities supports_auth_plain = "AUTH=PLAIN" in M.client.capabilities @@ -518,13 +513,11 @@ class MailAccountHandler(LoggingMixin): M: MailBox, rule: MailRule, ): - self.log("debug", f"Rule {rule}: Selecting folder {rule.folder}") try: M.folder.set(rule.folder) except MailboxFolderSelectError as err: - self.log( "error", f"Unable to access folder {rule.folder}, attempting folder listing", @@ -651,7 +644,6 @@ class MailAccountHandler(LoggingMixin): consume_tasks = list() for att in message.attachments: - if ( att.content_disposition != "attachment" and rule.attachment_type @@ -680,7 +672,6 @@ class MailAccountHandler(LoggingMixin): mime_type = magic.from_buffer(att.payload, mime=True) if is_mime_type_supported(mime_type): - os.makedirs(settings.SCRATCH_DIR, exist_ok=True) _, temp_filename = tempfile.mkstemp( prefix="paperless-mail-", diff --git a/src/paperless_mail/management/commands/mail_fetcher.py b/src/paperless_mail/management/commands/mail_fetcher.py index e2bc8262c..ec87723eb 100644 --- a/src/paperless_mail/management/commands/mail_fetcher.py +++ b/src/paperless_mail/management/commands/mail_fetcher.py @@ -1,9 +1,9 @@ from django.core.management.base import BaseCommand + from paperless_mail import tasks class Command(BaseCommand): - help = """ """.replace( " ", @@ -11,5 +11,4 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - tasks.process_mail_accounts() diff --git a/src/paperless_mail/migrations/0001_initial.py b/src/paperless_mail/migrations/0001_initial.py index dbc6e467f..f7e717f0e 100644 --- a/src/paperless_mail/migrations/0001_initial.py +++ b/src/paperless_mail/migrations/0001_initial.py @@ -1,11 +1,11 @@ # Generated by Django 3.1.3 on 2020-11-15 22:54 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - initial = True dependencies = [ diff --git a/src/paperless_mail/migrations/0002_auto_20201117_1334.py b/src/paperless_mail/migrations/0002_auto_20201117_1334.py index 72e37e342..1f4df3f6d 100644 --- a/src/paperless_mail/migrations/0002_auto_20201117_1334.py +++ b/src/paperless_mail/migrations/0002_auto_20201117_1334.py @@ -5,7 +5,6 @@ from django.db.migrations import RunPython class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0001_initial"), ] diff --git a/src/paperless_mail/migrations/0003_auto_20201118_1940.py b/src/paperless_mail/migrations/0003_auto_20201118_1940.py index 30a882b03..a1263db05 100644 --- a/src/paperless_mail/migrations/0003_auto_20201118_1940.py +++ b/src/paperless_mail/migrations/0003_auto_20201118_1940.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.3 on 2020-11-18 19:40 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0002_auto_20201117_1334"), ] diff --git a/src/paperless_mail/migrations/0004_mailrule_order.py b/src/paperless_mail/migrations/0004_mailrule_order.py index 71b404185..4ffc0a6e5 100644 --- a/src/paperless_mail/migrations/0004_mailrule_order.py +++ b/src/paperless_mail/migrations/0004_mailrule_order.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.3 on 2020-11-21 21:51 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0003_auto_20201118_1940"), ] diff --git a/src/paperless_mail/migrations/0005_help_texts.py b/src/paperless_mail/migrations/0005_help_texts.py index 2d7a543b3..8e49238e9 100644 --- a/src/paperless_mail/migrations/0005_help_texts.py +++ b/src/paperless_mail/migrations/0005_help_texts.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.3 on 2020-11-22 10:36 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0004_mailrule_order"), ] diff --git a/src/paperless_mail/migrations/0006_auto_20210101_2340.py b/src/paperless_mail/migrations/0006_auto_20210101_2340.py index 90d768842..2c2ff9fa8 100644 --- a/src/paperless_mail/migrations/0006_auto_20210101_2340.py +++ b/src/paperless_mail/migrations/0006_auto_20210101_2340.py @@ -1,11 +1,11 @@ # Generated by Django 3.1.4 on 2021-01-01 23:40 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("documents", "1011_auto_20210101_2340"), ("paperless_mail", "0005_help_texts"), diff --git a/src/paperless_mail/migrations/0007_auto_20210106_0138.py b/src/paperless_mail/migrations/0007_auto_20210106_0138.py index 3325a86f7..c51a4aebe 100644 --- a/src/paperless_mail/migrations/0007_auto_20210106_0138.py +++ b/src/paperless_mail/migrations/0007_auto_20210106_0138.py @@ -1,10 +1,10 @@ # Generated by Django 3.1.5 on 2021-01-06 01:38 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0006_auto_20210101_2340"), ] diff --git a/src/paperless_mail/migrations/0008_auto_20210516_0940.py b/src/paperless_mail/migrations/0008_auto_20210516_0940.py index c25852c7d..b2fc062dd 100644 --- a/src/paperless_mail/migrations/0008_auto_20210516_0940.py +++ b/src/paperless_mail/migrations/0008_auto_20210516_0940.py @@ -1,10 +1,10 @@ # Generated by Django 3.2.3 on 2021-05-16 09:40 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0007_auto_20210106_0138"), ] diff --git a/src/paperless_mail/migrations/0009_alter_mailrule_action_alter_mailrule_folder.py b/src/paperless_mail/migrations/0009_alter_mailrule_action_alter_mailrule_folder.py index 7eff52691..47fdaff12 100644 --- a/src/paperless_mail/migrations/0009_alter_mailrule_action_alter_mailrule_folder.py +++ b/src/paperless_mail/migrations/0009_alter_mailrule_action_alter_mailrule_folder.py @@ -1,10 +1,10 @@ # Generated by Django 4.0.3 on 2022-03-28 17:40 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0008_auto_20210516_0940"), ] diff --git a/src/paperless_mail/migrations/0009_mailrule_assign_tags.py b/src/paperless_mail/migrations/0009_mailrule_assign_tags.py index fe2447e62..fbe359814 100644 --- a/src/paperless_mail/migrations/0009_mailrule_assign_tags.py +++ b/src/paperless_mail/migrations/0009_mailrule_assign_tags.py @@ -1,10 +1,10 @@ # Generated by Django 3.2.12 on 2022-03-11 15:00 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0008_auto_20210516_0940"), ] diff --git a/src/paperless_mail/migrations/0010_auto_20220311_1602.py b/src/paperless_mail/migrations/0010_auto_20220311_1602.py index 77a00d3c8..0511608ca 100644 --- a/src/paperless_mail/migrations/0010_auto_20220311_1602.py +++ b/src/paperless_mail/migrations/0010_auto_20220311_1602.py @@ -30,7 +30,6 @@ def migrate_tags_to_tag(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0009_mailrule_assign_tags"), ] diff --git a/src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py b/src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py index ce3c24932..16cec8710 100644 --- a/src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py +++ b/src/paperless_mail/migrations/0011_remove_mailrule_assign_tag.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0010_auto_20220311_1602"), ] diff --git a/src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py b/src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py index 610237f3b..83ece3bba 100644 --- a/src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py +++ b/src/paperless_mail/migrations/0012_alter_mailrule_assign_tags.py @@ -1,10 +1,10 @@ # Generated by Django 3.2.12 on 2022-03-11 16:21 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0011_remove_mailrule_assign_tag"), ] diff --git a/src/paperless_mail/migrations/0013_merge_20220412_1051.py b/src/paperless_mail/migrations/0013_merge_20220412_1051.py index 2fdb17ff5..0310fd083 100644 --- a/src/paperless_mail/migrations/0013_merge_20220412_1051.py +++ b/src/paperless_mail/migrations/0013_merge_20220412_1051.py @@ -4,7 +4,6 @@ from django.db import migrations class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0009_alter_mailrule_action_alter_mailrule_folder"), ("paperless_mail", "0012_alter_mailrule_assign_tags"), diff --git a/src/paperless_mail/migrations/0014_alter_mailrule_action.py b/src/paperless_mail/migrations/0014_alter_mailrule_action.py index 6d3a369cd..6be3ddf69 100644 --- a/src/paperless_mail/migrations/0014_alter_mailrule_action.py +++ b/src/paperless_mail/migrations/0014_alter_mailrule_action.py @@ -1,10 +1,10 @@ # Generated by Django 4.0.4 on 2022-04-18 22:57 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0013_merge_20220412_1051"), ] diff --git a/src/paperless_mail/migrations/0015_alter_mailrule_action.py b/src/paperless_mail/migrations/0015_alter_mailrule_action.py index 62a44e6aa..80de9b2b1 100644 --- a/src/paperless_mail/migrations/0015_alter_mailrule_action.py +++ b/src/paperless_mail/migrations/0015_alter_mailrule_action.py @@ -1,10 +1,10 @@ # Generated by Django 4.0.4 on 2022-05-29 13:21 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0014_alter_mailrule_action"), ] diff --git a/src/paperless_mail/migrations/0016_mailrule_consumption_scope.py b/src/paperless_mail/migrations/0016_mailrule_consumption_scope.py index ea54bce1b..d4a0ba590 100644 --- a/src/paperless_mail/migrations/0016_mailrule_consumption_scope.py +++ b/src/paperless_mail/migrations/0016_mailrule_consumption_scope.py @@ -1,10 +1,10 @@ # Generated by Django 4.0.4 on 2022-07-11 22:02 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ ("paperless_mail", "0015_alter_mailrule_action"), ] diff --git a/src/paperless_mail/migrations/0017_mailaccount_owner_mailrule_owner.py b/src/paperless_mail/migrations/0017_mailaccount_owner_mailrule_owner.py index 5eeccf440..98cfef014 100644 --- a/src/paperless_mail/migrations/0017_mailaccount_owner_mailrule_owner.py +++ b/src/paperless_mail/migrations/0017_mailaccount_owner_mailrule_owner.py @@ -1,12 +1,12 @@ # Generated by Django 4.1.3 on 2022-12-06 04:48 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("paperless_mail", "0016_mailrule_consumption_scope"), diff --git a/src/paperless_mail/migrations/0018_processedmail.py b/src/paperless_mail/migrations/0018_processedmail.py index 93ca64dda..3307f7579 100644 --- a/src/paperless_mail/migrations/0018_processedmail.py +++ b/src/paperless_mail/migrations/0018_processedmail.py @@ -1,13 +1,13 @@ # Generated by Django 4.1.5 on 2023-03-03 18:38 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion import django.utils.timezone +from django.conf import settings +from django.db import migrations +from django.db import models class Migration(migrations.Migration): - dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), ("paperless_mail", "0017_mailaccount_owner_mailrule_owner"), diff --git a/src/paperless_mail/migrations/0019_mailrule_filter_to.py b/src/paperless_mail/migrations/0019_mailrule_filter_to.py index 5089670b0..8951be290 100644 --- a/src/paperless_mail/migrations/0019_mailrule_filter_to.py +++ b/src/paperless_mail/migrations/0019_mailrule_filter_to.py @@ -1,6 +1,7 @@ # Generated by Django 4.1.7 on 2023-03-11 21:08 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): diff --git a/src/paperless_mail/migrations/0020_mailaccount_is_token.py b/src/paperless_mail/migrations/0020_mailaccount_is_token.py index 98a48ec51..81ce50a19 100644 --- a/src/paperless_mail/migrations/0020_mailaccount_is_token.py +++ b/src/paperless_mail/migrations/0020_mailaccount_is_token.py @@ -1,6 +1,7 @@ # Generated by Django 4.1.7 on 2023-03-22 17:51 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): diff --git a/src/paperless_mail/migrations/0021_alter_mailaccount_password.py b/src/paperless_mail/migrations/0021_alter_mailaccount_password.py index 2c0f68065..0c012b98b 100644 --- a/src/paperless_mail/migrations/0021_alter_mailaccount_password.py +++ b/src/paperless_mail/migrations/0021_alter_mailaccount_password.py @@ -1,6 +1,7 @@ # Generated by Django 4.1.7 on 2023-04-20 15:03 -from django.db import migrations, models +from django.db import migrations +from django.db import models class Migration(migrations.Migration): diff --git a/src/paperless_mail/models.py b/src/paperless_mail/models.py index a9165b248..3ea0619d9 100644 --- a/src/paperless_mail/models.py +++ b/src/paperless_mail/models.py @@ -1,8 +1,9 @@ -import documents.models as document_models from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ +import documents.models as document_models + class MailAccount(document_models.ModelWithOwner): class Meta: @@ -229,7 +230,6 @@ class MailRule(document_models.ModelWithOwner): class ProcessedMail(document_models.ModelWithOwner): - rule = models.ForeignKey( MailRule, null=False, diff --git a/src/paperless_mail/parsers.py b/src/paperless_mail/parsers.py index 4ce8a6019..b838290cb 100644 --- a/src/paperless_mail/parsers.py +++ b/src/paperless_mail/parsers.py @@ -10,13 +10,14 @@ from bleach import linkify from django.conf import settings from django.utils.timezone import is_naive from django.utils.timezone import make_aware -from documents.parsers import DocumentParser -from documents.parsers import make_thumbnail_from_pdf -from documents.parsers import ParseError from humanfriendly import format_size from imap_tools import MailMessage from tika import parser +from documents.parsers import DocumentParser +from documents.parsers import ParseError +from documents.parsers import make_thumbnail_from_pdf + class MailDocumentParser(DocumentParser): """ @@ -249,14 +250,12 @@ class MailDocumentParser(DocumentParser): return html def generate_pdf_from_mail(self, mail): - url = self.gotenberg_server + "/forms/chromium/convert/html" self.log("info", "Converting mail to PDF") css_file = os.path.join(os.path.dirname(__file__), "templates/output.css") with open(css_file, "rb") as css_handle: - files = { "html": ("index.html", self.mail_to_html(mail)), "css": ("output.css", css_handle), diff --git a/src/paperless_mail/serialisers.py b/src/paperless_mail/serialisers.py index e04a5c066..41dea9033 100644 --- a/src/paperless_mail/serialisers.py +++ b/src/paperless_mail/serialisers.py @@ -1,10 +1,11 @@ +from rest_framework import serializers + from documents.serialisers import CorrespondentField from documents.serialisers import DocumentTypeField from documents.serialisers import OwnedObjectSerializer from documents.serialisers import TagsField from paperless_mail.models import MailAccount from paperless_mail.models import MailRule -from rest_framework import serializers class ObfuscatedPasswordField(serializers.Field): diff --git a/src/paperless_mail/tasks.py b/src/paperless_mail/tasks.py index 5c92233de..ab013a41e 100644 --- a/src/paperless_mail/tasks.py +++ b/src/paperless_mail/tasks.py @@ -1,6 +1,7 @@ import logging from celery import shared_task + from paperless_mail.mail import MailAccountHandler from paperless_mail.mail import MailError from paperless_mail.models import MailAccount diff --git a/src/paperless_mail/tests/test_api.py b/src/paperless_mail/tests/test_api.py index 64b651970..28a369c6c 100644 --- a/src/paperless_mail/tests/test_api.py +++ b/src/paperless_mail/tests/test_api.py @@ -2,6 +2,9 @@ import json from unittest import mock from django.contrib.auth.models import User +from rest_framework import status +from rest_framework.test import APITestCase + from documents.models import Correspondent from documents.models import DocumentType from documents.models import Tag @@ -9,8 +12,6 @@ from documents.tests.utils import DirectoriesMixin from paperless_mail.models import MailAccount from paperless_mail.models import MailRule from paperless_mail.tests.test_mail import BogusMailBox -from rest_framework import status -from rest_framework.test import APITestCase class TestAPIMailAccounts(DirectoriesMixin, APITestCase): diff --git a/src/paperless_mail/tests/test_live_mail.py b/src/paperless_mail/tests/test_live_mail.py index 757bc5f4f..6de2a6770 100644 --- a/src/paperless_mail/tests/test_live_mail.py +++ b/src/paperless_mail/tests/test_live_mail.py @@ -2,11 +2,13 @@ import os import pytest from django.test import TestCase + from paperless_mail.mail import MailAccountHandler from paperless_mail.mail import MailError from paperless_mail.models import MailAccount from paperless_mail.models import MailRule + # Only run if the environment is setup # And the environment is not empty (forks, I think) @pytest.mark.skipif( @@ -16,7 +18,6 @@ from paperless_mail.models import MailRule ) class TestMailLiveServer(TestCase): def setUp(self) -> None: - self.mail_account_handler = MailAccountHandler() self.account = MailAccount.objects.create( name="test", @@ -33,7 +34,6 @@ class TestMailLiveServer(TestCase): return super().tearDown() def test_process_non_gmail_server_flag(self): - try: rule1 = MailRule.objects.create( name="testrule", @@ -51,9 +51,7 @@ class TestMailLiveServer(TestCase): pass def test_process_non_gmail_server_tag(self): - try: - rule2 = MailRule.objects.create( name="testrule", account=self.account, diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 1f482f336..0a3cb5b2c 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -12,21 +12,22 @@ from unittest import mock from django.core.management import call_command from django.db import DatabaseError from django.test import TestCase -from documents.models import Correspondent -from documents.tests.utils import DirectoriesMixin -from documents.tests.utils import FileSystemAssertsMixin +from imap_tools import NOT from imap_tools import EmailAddress from imap_tools import FolderInfo from imap_tools import MailboxFolderSelectError from imap_tools import MailboxLoginError from imap_tools import MailMessage from imap_tools import MailMessageFlags -from imap_tools import NOT + +from documents.models import Correspondent +from documents.tests.utils import DirectoriesMixin +from documents.tests.utils import FileSystemAssertsMixin from paperless_mail import tasks -from paperless_mail.mail import apply_mail_action from paperless_mail.mail import MailAccountHandler from paperless_mail.mail import MailError from paperless_mail.mail import TagMailAction +from paperless_mail.mail import apply_mail_action from paperless_mail.models import MailAccount from paperless_mail.models import MailRule @@ -77,7 +78,6 @@ class BogusClient: class BogusMailBox(ContextManager): - # Common values so tests don't need to remember an accepted login USERNAME: str = "admin" ASCII_PASSWORD: str = "secret" @@ -229,7 +229,6 @@ class TestMail( flagged: bool = False, processed: bool = False, ) -> MailMessage: - if to is None: to = ["tosomeone@somewhere.com"] @@ -545,7 +544,7 @@ class TestMail( ("*.png", ["f2.png"]), ] - for (pattern, matches) in tests: + for pattern, matches in tests: with self.subTest(msg=pattern): self._queue_consumption_tasks_mock.reset_mock() account = MailAccount(name=str(uuid.uuid4())) @@ -566,7 +565,6 @@ class TestMail( ) def test_handle_mail_account_mark_read(self): - account = MailAccount.objects.create( name="test", imap_server="", @@ -590,7 +588,6 @@ class TestMail( self.assertEqual(len(self.bogus_mailbox.messages), 3) def test_handle_mail_account_delete(self): - account = MailAccount.objects.create( name="test", imap_server="", @@ -612,6 +609,28 @@ class TestMail( self.assertEqual(len(self.bogus_mailbox.messages), 1) + def test_handle_mail_account_delete_no_filters(self): + account = MailAccount.objects.create( + name="test", + imap_server="", + username="admin", + password="secret", + ) + + _ = MailRule.objects.create( + name="testrule", + account=account, + action=MailRule.MailAction.DELETE, + maximum_age=0, + ) + + self.assertEqual(len(self.bogus_mailbox.messages), 3) + + self.mail_account_handler.handle_mail_account(account) + self.apply_mail_actions() + + self.assertEqual(len(self.bogus_mailbox.messages), 0) + def test_handle_mail_account_flag(self): account = MailAccount.objects.create( name="test", @@ -714,7 +733,6 @@ class TestMail( self.assertEqual(len(self.bogus_mailbox.messages), 3) def test_tag_mail_action_applemail_wrong_input(self): - self.assertRaises( MailError, TagMailAction, @@ -798,7 +816,6 @@ class TestMail( self.assertEqual(len(self.bogus_mailbox.messages_spam), 1) def test_error_skip_rule(self): - account = MailAccount.objects.create( name="test2", imap_server="", @@ -929,7 +946,6 @@ class TestMail( self.assertEqual(self.bogus_mailbox.messages[0].from_, "amazon@amazon.de") def test_error_create_correspondent(self): - account = MailAccount.objects.create( name="test2", imap_server="", @@ -975,7 +991,6 @@ class TestMail( ) def test_filters(self): - account = MailAccount.objects.create( name="test3", imap_server="", @@ -983,7 +998,7 @@ class TestMail( password="secret", ) - for (f_body, f_from, f_to, f_subject, expected_mail_count) in [ + for f_body, f_from, f_to, f_subject, expected_mail_count in [ (None, None, None, "Claim", 1), ("electronic", None, None, None, 1), (None, "amazon", None, None, 2), diff --git a/src/paperless_mail/tests/test_parsers.py b/src/paperless_mail/tests/test_parsers.py index 3515cfbf4..e6cae1121 100644 --- a/src/paperless_mail/tests/test_parsers.py +++ b/src/paperless_mail/tests/test_parsers.py @@ -3,6 +3,7 @@ import os from unittest import mock from django.test import TestCase + from documents.parsers import ParseError from documents.tests.utils import FileSystemAssertsMixin from paperless_mail.parsers import MailDocumentParser diff --git a/src/paperless_mail/tests/test_parsers_live.py b/src/paperless_mail/tests/test_parsers_live.py index 1ff42b6dd..1b911453d 100644 --- a/src/paperless_mail/tests/test_parsers_live.py +++ b/src/paperless_mail/tests/test_parsers_live.py @@ -6,13 +6,14 @@ from urllib.request import urlopen import pytest from django.test import TestCase -from documents.parsers import run_convert -from documents.tests.utils import FileSystemAssertsMixin from imagehash import average_hash -from paperless_mail.parsers import MailDocumentParser from pdfminer.high_level import extract_text from PIL import Image +from documents.parsers import run_convert +from documents.tests.utils import FileSystemAssertsMixin +from paperless_mail.parsers import MailDocumentParser + class TestParserLive(FileSystemAssertsMixin, TestCase): SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "samples") diff --git a/src/paperless_mail/views.py b/src/paperless_mail/views.py index e2bee34ff..15346b920 100644 --- a/src/paperless_mail/views.py +++ b/src/paperless_mail/views.py @@ -2,20 +2,21 @@ import datetime import logging from django.http import HttpResponseBadRequest -from documents.views import PassUserMixin -from paperless.views import StandardPagination -from paperless_mail.mail import get_mailbox -from paperless_mail.mail import mailbox_login -from paperless_mail.mail import MailError -from paperless_mail.models import MailAccount -from paperless_mail.models import MailRule -from paperless_mail.serialisers import MailAccountSerializer -from paperless_mail.serialisers import MailRuleSerializer from rest_framework.generics import GenericAPIView from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from documents.views import PassUserMixin +from paperless.views import StandardPagination +from paperless_mail.mail import MailError +from paperless_mail.mail import get_mailbox +from paperless_mail.mail import mailbox_login +from paperless_mail.models import MailAccount +from paperless_mail.models import MailRule +from paperless_mail.serialisers import MailAccountSerializer +from paperless_mail.serialisers import MailRuleSerializer + class MailAccountViewSet(ModelViewSet, PassUserMixin): model = MailAccount @@ -36,7 +37,6 @@ class MailRuleViewSet(ModelViewSet, PassUserMixin): class MailAccountTestView(GenericAPIView): - permission_classes = (IsAuthenticated,) serializer_class = MailAccountSerializer diff --git a/src/paperless_tesseract/apps.py b/src/paperless_tesseract/apps.py index 02045758b..f634349fb 100644 --- a/src/paperless_tesseract/apps.py +++ b/src/paperless_tesseract/apps.py @@ -1,13 +1,12 @@ from django.apps import AppConfig + from paperless_tesseract.signals import tesseract_consumer_declaration class PaperlessTesseractConfig(AppConfig): - name = "paperless_tesseract" def ready(self): - from documents.signals import document_consumer_declaration document_consumer_declaration.connect(tesseract_consumer_declaration) diff --git a/src/paperless_tesseract/checks.py b/src/paperless_tesseract/checks.py index ed5725d36..82d255005 100644 --- a/src/paperless_tesseract/checks.py +++ b/src/paperless_tesseract/checks.py @@ -3,8 +3,8 @@ import subprocess from django.conf import settings from django.core.checks import Error -from django.core.checks import register from django.core.checks import Warning +from django.core.checks import register def get_tesseract_langs(): diff --git a/src/paperless_tesseract/parsers.py b/src/paperless_tesseract/parsers.py index f3e8e21fd..7657cb7e2 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless_tesseract/parsers.py @@ -7,11 +7,12 @@ from pathlib import Path from typing import Optional from django.conf import settings -from documents.parsers import DocumentParser -from documents.parsers import make_thumbnail_from_pdf -from documents.parsers import ParseError from PIL import Image +from documents.parsers import DocumentParser +from documents.parsers import ParseError +from documents.parsers import make_thumbnail_from_pdf + class NoTextFoundException(Exception): pass @@ -30,7 +31,6 @@ class RasterisedDocumentParser(DocumentParser): logging_name = "paperless.parsing.tesseract" def extract_metadata(self, document_path, mime_type): - result = [] if mime_type == "application/pdf": import pikepdf @@ -264,7 +264,6 @@ class RasterisedDocumentParser(DocumentParser): # Convert pixels to mega-pixels and provide to ocrmypdf max_pixels_mpixels = settings.OCR_MAX_IMAGE_PIXELS / 1_000_000.0 if max_pixels_mpixels > 0: - self.log( "debug", f"Calculated {max_pixels_mpixels} megapixels for OCR", @@ -310,7 +309,8 @@ class RasterisedDocumentParser(DocumentParser): # text located via OCR import ocrmypdf - from ocrmypdf import InputFileError, EncryptedPdfError + from ocrmypdf import EncryptedPdfError + from ocrmypdf import InputFileError archive_path = Path(os.path.join(self.tempdir, "archive.pdf")) sidecar_file = Path(os.path.join(self.tempdir, "sidecar.txt")) diff --git a/src/paperless_tesseract/tests/test_checks.py b/src/paperless_tesseract/tests/test_checks.py index fdcbf7656..79991bab1 100644 --- a/src/paperless_tesseract/tests/test_checks.py +++ b/src/paperless_tesseract/tests/test_checks.py @@ -1,8 +1,9 @@ from unittest import mock from django.core.checks import ERROR -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from paperless_tesseract import check_default_language_available diff --git a/src/paperless_tesseract/tests/test_parser.py b/src/paperless_tesseract/tests/test_parser.py index 5cbbc4d55..23cff29b7 100644 --- a/src/paperless_tesseract/tests/test_parser.py +++ b/src/paperless_tesseract/tests/test_parser.py @@ -5,14 +5,15 @@ import uuid from typing import ContextManager from unittest import mock -from django.test import override_settings from django.test import TestCase +from django.test import override_settings + from documents.parsers import ParseError from documents.parsers import run_convert from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin -from paperless_tesseract.parsers import post_process_text from paperless_tesseract.parsers import RasterisedDocumentParser +from paperless_tesseract.parsers import post_process_text image_to_string_calls = [] @@ -38,7 +39,6 @@ class FakeImageFile(ContextManager): class TestParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase): - SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "samples") def assertContainsStrings(self, content, strings): @@ -52,7 +52,6 @@ class TestParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertListEqual(indices, sorted(indices)) def test_post_process_text(self): - text_cases = [ ("simple string", "simple string"), ("simple newline\n testing string", "simple newline\ntesting string"), @@ -829,7 +828,6 @@ class TestParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase): class TestParserFileTypes(DirectoriesMixin, FileSystemAssertsMixin, TestCase): - SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "samples") def test_bmp(self): diff --git a/src/paperless_text/apps.py b/src/paperless_text/apps.py index 61f5eb7ef..0dd7b2206 100644 --- a/src/paperless_text/apps.py +++ b/src/paperless_text/apps.py @@ -1,13 +1,12 @@ from django.apps import AppConfig + from paperless_text.signals import text_consumer_declaration class PaperlessTextConfig(AppConfig): - name = "paperless_text" def ready(self): - from documents.signals import document_consumer_declaration document_consumer_declaration.connect(text_consumer_declaration) diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py index 4889c54df..37e4ca1a6 100644 --- a/src/paperless_text/parsers.py +++ b/src/paperless_text/parsers.py @@ -1,11 +1,12 @@ import os from django.conf import settings -from documents.parsers import DocumentParser from PIL import Image from PIL import ImageDraw from PIL import ImageFont +from documents.parsers import DocumentParser + class TextDocumentParser(DocumentParser): """ diff --git a/src/paperless_text/tests/test_parser.py b/src/paperless_text/tests/test_parser.py index 76a8b8498..869a3a8ef 100644 --- a/src/paperless_text/tests/test_parser.py +++ b/src/paperless_text/tests/test_parser.py @@ -1,6 +1,7 @@ import os from django.test import TestCase + from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from paperless_text.parsers import TextDocumentParser @@ -8,7 +9,6 @@ from paperless_text.parsers import TextDocumentParser class TestTextParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase): def test_thumbnail(self): - parser = TextDocumentParser(None) # just make sure that it does not crash @@ -19,7 +19,6 @@ class TestTextParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertIsFile(f) def test_parse(self): - parser = TextDocumentParser(None) parser.parse( diff --git a/src/paperless_tika/apps.py b/src/paperless_tika/apps.py index 012986543..6fad68df8 100644 --- a/src/paperless_tika/apps.py +++ b/src/paperless_tika/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig from django.conf import settings + from paperless_tika.signals import tika_consumer_declaration diff --git a/src/paperless_tika/parsers.py b/src/paperless_tika/parsers.py index ea6a83f6c..779abbe59 100644 --- a/src/paperless_tika/parsers.py +++ b/src/paperless_tika/parsers.py @@ -4,11 +4,12 @@ from pathlib import Path import dateutil.parser import requests from django.conf import settings -from documents.parsers import DocumentParser -from documents.parsers import make_thumbnail_from_pdf -from documents.parsers import ParseError from tika import parser +from documents.parsers import DocumentParser +from documents.parsers import ParseError +from documents.parsers import make_thumbnail_from_pdf + class TikaDocumentParser(DocumentParser): """ diff --git a/src/paperless_tika/tests/test_live_tika.py b/src/paperless_tika/tests/test_live_tika.py index 60c2b96a4..766517965 100644 --- a/src/paperless_tika/tests/test_live_tika.py +++ b/src/paperless_tika/tests/test_live_tika.py @@ -5,6 +5,7 @@ from typing import Final import pytest from django.test import TestCase + from paperless_tika.parsers import TikaDocumentParser diff --git a/src/paperless_tika/tests/test_tika_parser.py b/src/paperless_tika/tests/test_tika_parser.py index 20e846850..4f540f3c0 100644 --- a/src/paperless_tika/tests/test_tika_parser.py +++ b/src/paperless_tika/tests/test_tika_parser.py @@ -3,13 +3,14 @@ import os from pathlib import Path from unittest import mock -from django.test import override_settings from django.test import TestCase -from documents.parsers import ParseError -from paperless_tika.parsers import TikaDocumentParser +from django.test import override_settings from requests import Response from rest_framework import status +from documents.parsers import ParseError +from paperless_tika.parsers import TikaDocumentParser + class TestTikaParser(TestCase): def setUp(self) -> None: