From c324a71e910b519d3cbc39922b03dd01d7855e86 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 15 Nov 2024 07:42:10 -0800 Subject: [PATCH 1/9] Fix: include db_index caveat in squashed migrations (#8292) --- ...dd_insensitive_to_match_squashed_0018_auto_20170715_1712.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/documents/migrations/0015_add_insensitive_to_match_squashed_0018_auto_20170715_1712.py b/src/documents/migrations/0015_add_insensitive_to_match_squashed_0018_auto_20170715_1712.py index 4463a953d..bfa897601 100644 --- a/src/documents/migrations/0015_add_insensitive_to_match_squashed_0018_auto_20170715_1712.py +++ b/src/documents/migrations/0015_add_insensitive_to_match_squashed_0018_auto_20170715_1712.py @@ -1,6 +1,7 @@ # Generated by Django 4.2.13 on 2024-06-28 17:57 import django.db.models.deletion +from django.conf import settings from django.db import migrations from django.db import models @@ -43,7 +44,7 @@ class Migration(migrations.Migration): name="content", field=models.TextField( blank=True, - db_index=True, + db_index=("mysql" not in settings.DATABASES["default"]["ENGINE"]), help_text="The raw, text-only data of the document. This field is primarily used for searching.", ), ), From 6c3d6d562d9fcc1dc04ae02d115cf8eb24f0cccc Mon Sep 17 00:00:00 2001 From: ftibi93 Date: Sat, 16 Nov 2024 05:19:18 +0100 Subject: [PATCH 2/9] Documentation: fix docker-compose.portainer.yml GID (#8273) --- docker/compose/docker-compose.portainer.yml | 26 ++++----------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/docker/compose/docker-compose.portainer.yml b/docker/compose/docker-compose.portainer.yml index 621908ff8..588a93c84 100644 --- a/docker/compose/docker-compose.portainer.yml +++ b/docker/compose/docker-compose.portainer.yml @@ -19,6 +19,8 @@ # # - Open portainer Stacks list and click 'Add stack' # - Paste the contents of this file and assign a name, e.g. 'paperless' +# - Upload 'docker-compose.env' by clicking on 'Load variables from .env file' +# - Modify the environment variables as needed # - Click 'Deploy the stack' and wait for it to be deployed # - Open the list of containers, select paperless_webserver_1 # - Click 'Console' and then 'Connect' to open the command line inside the container @@ -61,28 +63,8 @@ services: environment: PAPERLESS_REDIS: redis://broker:6379 PAPERLESS_DBHOST: db -# The UID and GID of the user used to run paperless in the container. Set this -# to your UID and GID on the host so that you have write access to the -# consumption directory. - USERMAP_UID: 1000 - USERMAP_GID: 100 -# Additional languages to install for text recognition, separated by a -# whitespace. Note that this is -# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the -# language used for OCR. -# The container installs English, German, Italian, Spanish and French by -# default. -# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster -# for available languages. - #PAPERLESS_OCR_LANGUAGES: tur ces -# Adjust this key if you plan to make paperless available publicly. It should -# be a very long sequence of random characters. You don't need to remember it. - #PAPERLESS_SECRET_KEY: change-me -# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. - #PAPERLESS_TIME_ZONE: America/Los_Angeles -# The default language to use for OCR. Set this to the language most of your -# documents are written in. - #PAPERLESS_OCR_LANGUAGE: eng + env_file: + - stack.env volumes: data: From e94a92ed595f5d8035c88673d083f60f875a66a7 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 18 Nov 2024 10:34:46 -0800 Subject: [PATCH 3/9] Feature: two-factor authentication (#8012) --- Pipfile | 2 +- Pipfile.lock | 17 +- docs/usage.md | 6 + src-ui/messages.xlf | 227 ++++++++++++++--- .../app-frame/app-frame.component.spec.ts | 1 + .../app-frame/app-frame.component.ts | 1 + .../user-edit-dialog.component.html | 14 + .../user-edit-dialog.component.spec.ts | 43 +++- .../user-edit-dialog.component.ts | 33 ++- .../profile-edit-dialog.component.html | 241 ++++++++++++------ .../profile-edit-dialog.component.spec.ts | 81 ++++++ .../profile-edit-dialog.component.ts | 93 ++++++- src-ui/src/app/data/user-profile.ts | 7 + src-ui/src/app/data/user.ts | 1 + .../app/services/permissions.service.spec.ts | 21 ++ .../src/app/services/permissions.service.ts | 4 + .../src/app/services/profile.service.spec.ts | 28 ++ src-ui/src/app/services/profile.service.ts | 27 ++ .../app/services/rest/user.service.spec.ts | 28 +- src-ui/src/app/services/rest/user.service.ts | 10 +- .../management/commands/document_exporter.py | 2 + src/documents/templates/mfa/authenticate.html | 35 +++ src/documents/tests/test_api_permissions.py | 54 ++++ src/documents/tests/test_api_profile.py | 80 ++++++ src/locale/en_US/LC_MESSAGES/django.po | 89 ++++--- src/paperless/serialisers.py | 13 + src/paperless/settings.py | 3 + src/paperless/urls.py | 45 +++- src/paperless/views.py | 97 +++++++ 29 files changed, 1128 insertions(+), 175 deletions(-) create mode 100644 src/documents/templates/mfa/authenticate.html diff --git a/Pipfile b/Pipfile index 794af014d..1e79bb604 100644 --- a/Pipfile +++ b/Pipfile @@ -8,7 +8,7 @@ dateparser = "~=1.2" # WARNING: django does not use semver. # Only patch versions are guaranteed to not introduce breaking changes. django = "~=5.1.3" -django-allauth = {extras = ["socialaccount"], version = "*"} +django-allauth = {extras = ["mfa", "socialaccount"], version = "*"} django-auditlog = "*" django-celery-results = "*" django-compression-middleware = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 9377e3575..063ec2389 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "dccf58aea1ba4c0aa4aa93c1cc13881229889db25bc6e5b2384413a7e7e85182" + "sha256": "5a7cb70103e8f3931682c73432290f2f4ec2ba06395c8ec076d2d5449c4ff0dd" }, "pipfile-spec": 6, "requires": {}, @@ -522,6 +522,7 @@ }, "django-allauth": { "extras": [ + "mfa", "socialaccount" ], "hashes": [ @@ -641,6 +642,13 @@ "markers": "python_version >= '3.7'", "version": "==1.2.2" }, + "fido2": { + "hashes": [ + "sha256:26100f226d12ced621ca6198528ce17edf67b78df4287aee1285fee3cd5aa9fc", + "sha256:6be34c0b9fe85e4911fd2d103cce7ae8ce2f064384a7a2a3bd970b3ef7702931" + ], + "version": "==1.1.3" + }, "filelock": { "hashes": [ "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0", @@ -1776,6 +1784,13 @@ "index": "pypi", "version": "==0.1.9" }, + "qrcode": { + "hashes": [ + "sha256:025ce2b150f7fe4296d116ee9bad455a6643ab4f6e7dce541613a4758cbce347", + "sha256:9fc05f03305ad27a709eb742cf3097fa19e6f6f93bb9e2f039c0979190f6f1b1" + ], + "version": "==8.0" + }, "rapidfuzz": { "hashes": [ "sha256:00d02cbd75d283c287471b5b3738b3e05c9096150f93f2d2dfa10b3d700f2db9", diff --git a/docs/usage.md b/docs/usage.md index f853bb7f5..8f22ec3eb 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -299,6 +299,12 @@ In order to enable the password reset feature you will need to setup an SMTP bac [`PAPERLESS_EMAIL_HOST`](configuration.md#PAPERLESS_EMAIL_HOST). If your installation does not have [`PAPERLESS_URL`](configuration.md#PAPERLESS_URL) set, the reset link included in emails will use the server host. +### Two-factor authentication + +Users can enable two-factor authentication (2FA) for their accounts from the 'My Profile' dialog. Opening the dropdown reveals a QR code that can be scanned by a 2FA app (e.g. Google Authenticator) to generate a code. The code must then be entered in the dialog to enable 2FA. If the code is accepted and 2FA is enabled, the user will be shown a set of 10 recovery codes that can be used to login in the event that the 2FA device is lost or unavailable. These codes should be stored securely and cannot be retrieved again. Once enabled, users will be required to enter a code from their 2FA app when logging in. + +Should a user lose access to their 2FA device and all recovery codes, a superuser can disable 2FA for the user from the 'Users & Groups' management screen. + ## Workflows !!! note diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index b9aa4e03e..e988a39cb 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -520,6 +520,10 @@ src/app/components/admin/config/config.component.html 34 + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 124 + Discard @@ -576,7 +580,7 @@ src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html - 43 + 57 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html @@ -584,7 +588,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 99 + 184 src/app/components/document-detail/document-detail.component.html @@ -712,6 +716,14 @@ src/app/components/common/permissions-dialog/permissions-dialog.component.html 23 + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 111 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 127 + src/app/components/common/system-status-dialog/system-status-dialog.component.html 10 @@ -1095,7 +1107,7 @@ src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html - 37 + 51 src/app/components/common/input/permissions/permissions-form/permissions-form.component.html @@ -1707,7 +1719,7 @@ src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html - 42 + 56 src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html @@ -1719,7 +1731,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 98 + 183 src/app/components/common/select-dialog/select-dialog.component.html @@ -2514,7 +2526,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 159 + 173 @@ -2917,21 +2929,21 @@ Sidebar views updated src/app/components/app-frame/app-frame.component.ts - 208 + 209 Error updating sidebar views src/app/components/app-frame/app-frame.component.ts - 211 + 212 An error occurred while saving update checking settings. src/app/components/app-frame/app-frame.component.ts - 232 + 233 @@ -3720,7 +3732,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 18 + 20 @@ -4263,7 +4275,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 8 + 10 @@ -4274,7 +4286,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 28 + 30 @@ -4285,7 +4297,7 @@ src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 29 + 31 @@ -4323,18 +4335,70 @@ 30 + + Two-factor Authentication + + src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html + 37 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 104 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 138 + + + + Disable Two-factor Authentication + + src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html + 39 + + + src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html + 41 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 169 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 171 + + Create new user account src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts - 44 + 49 Edit user account src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts - 48 + 53 + + + + Totp deactivated + + src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts + 109 + + + + Totp deactivation failed + + src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts + 112 + + + src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts + 117 @@ -5151,32 +5215,36 @@ Confirm Email src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 13 + 15 Confirm Password src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 23 + 25 API Auth Token src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 31 + 33 Copy src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 35 + 37 src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 42 + 44 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 156 src/app/components/common/share-links-dropdown/share-links-dropdown.component.html @@ -5207,14 +5275,18 @@ Regenerate auth token src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 45 + 47 Copied! src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 53 + 55 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 163 src/app/components/common/share-links-dropdown/share-links-dropdown.component.html @@ -5225,91 +5297,176 @@ Warning: changing the token cannot be undone src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 55 + 57 Connected social accounts src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 59 + 63 Set a password before disconnecting social account. src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 63 + 67 Disconnect src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 69 + 73 Disconnect social account src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 71 + 75 Warning: disconnecting social accounts cannot be undone src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 81 + 85 Connect new social account src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html - 86 + 90 + + + + Scan the QR code with your authenticator app and then enter the code below + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 115 + + + + Authenticator secret + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 118 + + + + You can store this secret and use it to reinstall your authenticator app at a later time. + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 119 + + + + Code + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 122 + + + + Recovery codes will not be shown again, make sure to save them. + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 141 + + + + Copy codes + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html + 159 Emails must match src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 108 + 121 Passwords must match src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 136 + 149 Profile updated successfully src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 156 + 170 Error saving profile src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 168 + 182 Error generating auth token src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 185 + 199 Error disconnecting social account src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts - 210 + 224 + + + + Error fetching TOTP settings + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 243 + + + + TOTP activated successfully + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 263 + + + + Error activating TOTP + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 265 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 271 + + + + TOTP deactivated successfully + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 287 + + + + Error deactivating TOTP + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 289 + + + src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts + 294 diff --git a/src-ui/src/app/components/app-frame/app-frame.component.spec.ts b/src-ui/src/app/components/app-frame/app-frame.component.spec.ts index 1354a187e..f440946da 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.spec.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.spec.ts @@ -343,6 +343,7 @@ describe('AppFrameComponent', () => { component.editProfile() expect(modalSpy).toHaveBeenCalledWith(ProfileEditDialogComponent, { backdrop: 'static', + size: 'xl', }) }) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.ts b/src-ui/src/app/components/app-frame/app-frame.component.ts index df6ac65a2..83d927562 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.ts +++ b/src-ui/src/app/components/app-frame/app-frame.component.ts @@ -136,6 +136,7 @@ export class AppFrameComponent editProfile() { this.modalService.open(ProfileEditDialogComponent, { backdrop: 'static', + size: 'xl', }) this.closeMenu() } diff --git a/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html b/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html index ca834a3ad..a2b3db67d 100644 --- a/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html +++ b/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html @@ -32,6 +32,20 @@ + + @if (object?.is_mfa_enabled && currentUserIsSuperUser) { + + + + }
diff --git a/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.spec.ts b/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.spec.ts index 96a0044fe..5adaf3388 100644 --- a/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.spec.ts +++ b/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.spec.ts @@ -7,7 +7,7 @@ import { } from '@angular/forms' import { NgbActiveModal, NgbModule } from '@ng-bootstrap/ng-bootstrap' import { NgSelectModule } from '@ng-select/ng-select' -import { of } from 'rxjs' +import { of, throwError } from 'rxjs' import { IfOwnerDirective } from 'src/app/directives/if-owner.directive' import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive' import { GroupService } from 'src/app/services/rest/group.service' @@ -21,10 +21,15 @@ import { EditDialogMode } from '../edit-dialog.component' import { UserEditDialogComponent } from './user-edit-dialog.component' import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http' import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' +import { ToastService } from 'src/app/services/toast.service' +import { UserService } from 'src/app/services/rest/user.service' +import { PermissionsService } from 'src/app/services/permissions.service' describe('UserEditDialogComponent', () => { let component: UserEditDialogComponent let settingsService: SettingsService + let permissionsService: PermissionsService + let toastService: ToastService let fixture: ComponentFixture beforeEach(async () => { @@ -71,6 +76,8 @@ describe('UserEditDialogComponent', () => { fixture = TestBed.createComponent(UserEditDialogComponent) settingsService = TestBed.inject(SettingsService) settingsService.currentUser = { id: 99, username: 'user99' } + permissionsService = TestBed.inject(PermissionsService) + toastService = TestBed.inject(ToastService) component = fixture.componentInstance fixture.detectChanges() @@ -121,4 +128,38 @@ describe('UserEditDialogComponent', () => { component.save() expect(component.passwordIsSet).toBeTruthy() }) + + it('should support deactivation of TOTP', () => { + component.object = { id: 99, username: 'user99' } + const deactivateSpy = jest.spyOn( + component['service'] as UserService, + 'deactivateTotp' + ) + const toastErrorSpy = jest.spyOn(toastService, 'showError') + const toastInfoSpy = jest.spyOn(toastService, 'showInfo') + deactivateSpy.mockReturnValueOnce(throwError(() => new Error('error'))) + component.deactivateTotp() + expect(deactivateSpy).toHaveBeenCalled() + expect(toastErrorSpy).toHaveBeenCalled() + + deactivateSpy.mockReturnValueOnce(of(false)) + component.deactivateTotp() + expect(deactivateSpy).toHaveBeenCalled() + expect(toastErrorSpy).toHaveBeenCalled() + + deactivateSpy.mockReturnValueOnce(of(true)) + component.deactivateTotp() + expect(deactivateSpy).toHaveBeenCalled() + expect(toastInfoSpy).toHaveBeenCalled() + }) + + it('should check superuser status of current user', () => { + expect(component.currentUserIsSuperUser).toBeFalsy() + permissionsService.initialize([], { + id: 99, + username: 'user99', + is_superuser: true, + }) + expect(component.currentUserIsSuperUser).toBeTruthy() + }) }) diff --git a/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts b/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts index baadfa541..acd327d3a 100644 --- a/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts +++ b/src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.ts @@ -5,9 +5,11 @@ import { first } from 'rxjs' import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component' import { Group } from 'src/app/data/group' import { User } from 'src/app/data/user' +import { PermissionsService } from 'src/app/services/permissions.service' import { GroupService } from 'src/app/services/rest/group.service' import { UserService } from 'src/app/services/rest/user.service' import { SettingsService } from 'src/app/services/settings.service' +import { ToastService } from 'src/app/services/toast.service' @Component({ selector: 'pngx-user-edit-dialog', @@ -20,12 +22,15 @@ export class UserEditDialogComponent { groups: Group[] passwordIsSet: boolean = false + public totpLoading: boolean = false constructor( service: UserService, activeModal: NgbActiveModal, groupsService: GroupService, - settingsService: SettingsService + settingsService: SettingsService, + private toastService: ToastService, + private permissionsService: PermissionsService ) { super(service, activeModal, service, settingsService) @@ -87,4 +92,30 @@ export class UserEditDialogComponent .length > 0 super.save() } + + get currentUserIsSuperUser(): boolean { + return this.permissionsService.isSuperUser() + } + + deactivateTotp() { + this.totpLoading = true + ;(this.service as UserService) + .deactivateTotp(this.object) + .pipe(first()) + .subscribe({ + next: (result) => { + this.totpLoading = false + if (result) { + this.toastService.showInfo($localize`Totp deactivated`) + this.object.is_mfa_enabled = false + } else { + this.toastService.showError($localize`Totp deactivation failed`) + } + }, + error: (e) => { + this.totpLoading = false + this.toastService.showError($localize`Totp deactivation failed`, e) + }, + }) + } } diff --git a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html index 713d68864..f9d57baf3 100644 --- a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html +++ b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.html @@ -5,94 +5,179 @@
- -
-
-
-
- +
+
+ +
+
+
+
+ +
+
-
-
- -
-
-
-
- + +
+
+
+
+ +
+
-
-
- - -
- -
-
- - - - -
- Copied! -
-
Warning: changing the token cannot be undone
-
- @if (socialAccounts?.length > 0) { + +
-

Connected social accounts

-
    - @for (account of socialAccounts; track account.id) { -
  • - {{account.name}} ({{account.provider}}) + +
    +
    + + + (confirm)="generateAuthToken()"> -
    • - } -
-
Warning: disconnecting social accounts cannot be undone
-
- } - @if (socialAccountProviders?.length > 0) { -
-

Connect new social account

-
- @for (provider of socialAccountProviders; track provider.name) { - - {{provider.name}}  - - } +
+ Copied! +
+
Warning: changing the token cannot be undone
-
- } +
+
+ @if (socialAccounts?.length > 0) { +
+

Connected social accounts

+
    + @for (account of socialAccounts; track account.id) { +
  • + {{account.name}} ({{account.provider}}) + + +
    • + } +
+
Warning: disconnecting social accounts cannot be undone
+
+ } + @if (socialAccountProviders?.length > 0) { +
+

Connect new social account

+
+ @for (provider of socialAccountProviders; track provider.name) { + + {{provider.name}}  + + } +
+
+ } + @if (!isTotpEnabled) { +
+
+

+ +

+
+
+ + @if (totpSettingsLoading) { + + + } @else if (totpSettings) { +
+
+
Scan the QR code with your authenticator app and then enter the code below
+
+

+ Authenticator secret: {{totpSettings.secret}}. + You can store this secret and use it to reinstall your authenticator app at a later time. +

+
+ + +
+ } + +
+
+
+
+ } @else { + + @if (recoveryCodes) { + +
+
    + @for (code of recoveryCodes; track code; let i = $index) { + @if (i % 2 === 0) { +
  • + {{code}} + @if (recoveryCodes[i + 1]) { + {{recoveryCodes[i + 1]}} + } +
    • + } + } +
+ +
+ } + + + } +
+
diff --git a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts index d0af0f2ad..aa793941e 100644 --- a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts +++ b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.spec.ts @@ -294,4 +294,85 @@ describe('ProfileEditDialogComponent', () => { expect(disconnectSpy).toHaveBeenCalled() expect(component.socialAccounts).not.toContainEqual(socialAccount) }) + + it('should get totp settings', () => { + const settings = { + url: 'http://localhost/', + qr_svg: 'svg', + secret: 'secret', + } + const getSpy = jest.spyOn(profileService, 'getTotpSettings') + const toastSpy = jest.spyOn(toastService, 'showError') + getSpy.mockReturnValueOnce( + throwError(() => new Error('failed to get settings')) + ) + component.gettotpSettings() + expect(getSpy).toHaveBeenCalled() + expect(toastSpy).toHaveBeenCalled() + + getSpy.mockReturnValue(of(settings)) + component.gettotpSettings() + expect(getSpy).toHaveBeenCalled() + expect(component.totpSettings).toEqual(settings) + }) + + it('should activate totp', () => { + const activateSpy = jest.spyOn(profileService, 'activateTotp') + const toastErrorSpy = jest.spyOn(toastService, 'showError') + const toastInfoSpy = jest.spyOn(toastService, 'showInfo') + const error = new Error('failed to activate totp') + activateSpy.mockReturnValueOnce(throwError(() => error)) + component.totpSettings = { + url: 'http://localhost/', + qr_svg: 'svg', + secret: 'secret', + } + component.form.get('totp_code').patchValue('123456') + component.activateTotp() + expect(activateSpy).toHaveBeenCalledWith( + component.totpSettings.secret, + component.form.get('totp_code').value + ) + expect(toastErrorSpy).toHaveBeenCalled() + + activateSpy.mockReturnValueOnce(of({ success: false, recovery_codes: [] })) + component.activateTotp() + expect(toastErrorSpy).toHaveBeenCalledWith('Error activating TOTP', error) + + activateSpy.mockReturnValueOnce( + of({ success: true, recovery_codes: ['1', '2', '3'] }) + ) + component.activateTotp() + expect(toastInfoSpy).toHaveBeenCalled() + expect(component.isTotpEnabled).toBeTruthy() + expect(component.recoveryCodes).toEqual(['1', '2', '3']) + }) + + it('should deactivate totp', () => { + const deactivateSpy = jest.spyOn(profileService, 'deactivateTotp') + const toastErrorSpy = jest.spyOn(toastService, 'showError') + const toastInfoSpy = jest.spyOn(toastService, 'showInfo') + const error = new Error('failed to deactivate totp') + deactivateSpy.mockReturnValueOnce(throwError(() => error)) + component.deactivateTotp() + expect(deactivateSpy).toHaveBeenCalled() + expect(toastErrorSpy).toHaveBeenCalled() + + deactivateSpy.mockReturnValueOnce(of(false)) + component.deactivateTotp() + expect(toastErrorSpy).toHaveBeenCalledWith('Error deactivating TOTP', error) + + deactivateSpy.mockReturnValueOnce(of(true)) + component.deactivateTotp() + expect(toastInfoSpy).toHaveBeenCalled() + expect(component.isTotpEnabled).toBeFalsy() + }) + + it('should copy recovery codes', fakeAsync(() => { + const copySpy = jest.spyOn(clipboard, 'copy') + component.recoveryCodes = ['1', '2', '3'] + component.copyRecoveryCodes() + expect(copySpy).toHaveBeenCalledWith('1\n2\n3') + tick(3000) + })) }) diff --git a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts index 77cba4505..a4dbaf7d6 100644 --- a/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts +++ b/src-ui/src/app/components/common/profile-edit-dialog/profile-edit-dialog.component.ts @@ -2,7 +2,11 @@ import { Component, OnDestroy, OnInit } from '@angular/core' import { FormControl, FormGroup } from '@angular/forms' import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' import { ProfileService } from 'src/app/services/profile.service' -import { SocialAccount, SocialAccountProvider } from 'src/app/data/user-profile' +import { + TotpSettings, + SocialAccount, + SocialAccountProvider, +} from 'src/app/data/user-profile' import { ToastService } from 'src/app/services/toast.service' import { Subject, takeUntil } from 'rxjs' import { Clipboard } from '@angular/cdk/clipboard' @@ -25,6 +29,7 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { first_name: new FormControl(''), last_name: new FormControl(''), auth_token: new FormControl(''), + totp_code: new FormControl(''), }) private currentPassword: string @@ -38,7 +43,14 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { private emailConfirm: string public showEmailConfirm: boolean = false + public isTotpEnabled: boolean = false + public totpSettings: TotpSettings + public totpSettingsLoading: boolean = false + public totpLoading: boolean = false + public recoveryCodes: string[] + public copied: boolean = false + public codesCopied: boolean = false public socialAccounts: SocialAccount[] = [] public socialAccountProviders: SocialAccountProvider[] = [] @@ -70,6 +82,7 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { this.onPasswordChange() }) this.socialAccounts = profile.social_accounts + this.isTotpEnabled = profile.is_mfa_enabled }) this.profileService @@ -147,6 +160,7 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { const passwordChanged = this.newPassword && this.currentPassword !== this.newPassword const profile = Object.assign({}, this.form.value) + delete profile.totp_code this.networkActive = true this.profileService .update(profile) @@ -213,4 +227,81 @@ export class ProfileEditDialogComponent implements OnInit, OnDestroy { }, }) } + + public gettotpSettings(): void { + this.totpSettingsLoading = true + this.profileService + .getTotpSettings() + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe({ + next: (totpSettings) => { + this.totpSettingsLoading = false + this.totpSettings = totpSettings + }, + error: (error) => { + this.toastService.showError( + $localize`Error fetching TOTP settings`, + error + ) + this.totpSettingsLoading = false + }, + }) + } + + public activateTotp(): void { + this.totpLoading = true + this.form.get('totp_code').disable() + this.profileService + .activateTotp(this.totpSettings.secret, this.form.get('totp_code').value) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe({ + next: (activationResponse) => { + this.totpLoading = false + this.isTotpEnabled = activationResponse.success + this.recoveryCodes = activationResponse.recovery_codes + this.form.get('totp_code').enable() + if (activationResponse.success) { + this.toastService.showInfo($localize`TOTP activated successfully`) + } else { + this.toastService.showError($localize`Error activating TOTP`) + } + }, + error: (error) => { + this.totpLoading = false + this.form.get('totp_code').enable() + this.toastService.showError($localize`Error activating TOTP`, error) + }, + }) + } + + public deactivateTotp(): void { + this.totpLoading = true + this.profileService + .deactivateTotp() + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe({ + next: (success) => { + this.totpLoading = false + this.isTotpEnabled = !success + this.recoveryCodes = null + if (success) { + this.toastService.showInfo($localize`TOTP deactivated successfully`) + } else { + this.toastService.showError($localize`Error deactivating TOTP`) + } + }, + error: (error) => { + this.totpLoading = false + this.toastService.showError($localize`Error deactivating TOTP`, error) + }, + }) + } + + public copyRecoveryCodes(): void { + this.clipboard.copy(this.recoveryCodes.join('\n')) + this.codesCopied = true + setTimeout(() => { + this.codesCopied = false + }, 3000) + } } diff --git a/src-ui/src/app/data/user-profile.ts b/src-ui/src/app/data/user-profile.ts index 554f6f0e1..a07163233 100644 --- a/src-ui/src/app/data/user-profile.ts +++ b/src-ui/src/app/data/user-profile.ts @@ -17,4 +17,11 @@ export interface PaperlessUserProfile { auth_token?: string social_accounts?: SocialAccount[] has_usable_password?: boolean + is_mfa_enabled?: boolean +} + +export interface TotpSettings { + url: string + qr_svg: string + secret: string } diff --git a/src-ui/src/app/data/user.ts b/src-ui/src/app/data/user.ts index 49216a274..6bf051c25 100644 --- a/src-ui/src/app/data/user.ts +++ b/src-ui/src/app/data/user.ts @@ -11,4 +11,5 @@ export interface User extends ObjectWithId { groups?: number[] // Group[] user_permissions?: string[] inherited_permissions?: string[] + is_mfa_enabled?: boolean } diff --git a/src-ui/src/app/services/permissions.service.spec.ts b/src-ui/src/app/services/permissions.service.spec.ts index f4e01945e..ecbdb6f1b 100644 --- a/src-ui/src/app/services/permissions.service.spec.ts +++ b/src-ui/src/app/services/permissions.service.spec.ts @@ -439,4 +439,25 @@ describe('PermissionsService', () => { expect(permissionsService.isAdmin()).toBeFalsy() }) + + it('correctly checks superuser status', () => { + permissionsService.initialize([], { + username: 'testuser', + last_name: 'User', + first_name: 'Test', + id: 1, + is_superuser: true, + }) + + expect(permissionsService.isSuperUser()).toBeTruthy() + + permissionsService.initialize([], { + username: 'testuser', + last_name: 'User', + first_name: 'Test', + id: 1, + }) + + expect(permissionsService.isSuperUser()).toBeFalsy() + }) }) diff --git a/src-ui/src/app/services/permissions.service.ts b/src-ui/src/app/services/permissions.service.ts index c80bc763d..3d88b10cc 100644 --- a/src-ui/src/app/services/permissions.service.ts +++ b/src-ui/src/app/services/permissions.service.ts @@ -56,6 +56,10 @@ export class PermissionsService { return this.currentUser?.is_staff } + public isSuperUser(): boolean { + return this.currentUser?.is_superuser + } + public currentUserOwnsObject(object: ObjectWithPermissions): boolean { return ( !object || diff --git a/src-ui/src/app/services/profile.service.spec.ts b/src-ui/src/app/services/profile.service.spec.ts index beb7e9ad5..b7b85ee35 100644 --- a/src-ui/src/app/services/profile.service.spec.ts +++ b/src-ui/src/app/services/profile.service.spec.ts @@ -72,4 +72,32 @@ describe('ProfileService', () => { ) expect(req.request.method).toEqual('GET') }) + + it('calls get totp settings endpoint', () => { + service.getTotpSettings().subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}profile/totp/` + ) + expect(req.request.method).toEqual('GET') + }) + + it('calls activate totp endpoint', () => { + service.activateTotp('secret', 'code').subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}profile/totp/` + ) + expect(req.request.method).toEqual('POST') + expect(req.request.body).toEqual({ + secret: 'secret', + code: 'code', + }) + }) + + it('calls deactivate totp endpoint', () => { + service.deactivateTotp().subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}profile/totp/` + ) + expect(req.request.method).toEqual('DELETE') + }) }) diff --git a/src-ui/src/app/services/profile.service.ts b/src-ui/src/app/services/profile.service.ts index 32e06cce0..09839d012 100644 --- a/src-ui/src/app/services/profile.service.ts +++ b/src-ui/src/app/services/profile.service.ts @@ -2,6 +2,7 @@ import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { Observable } from 'rxjs' import { + TotpSettings, PaperlessUserProfile, SocialAccountProvider, } from '../data/user-profile' @@ -47,4 +48,30 @@ export class ProfileService { `${environment.apiBaseUrl}${this.endpoint}/social_account_providers/` ) } + + getTotpSettings(): Observable { + return this.http.get( + `${environment.apiBaseUrl}${this.endpoint}/totp/` + ) + } + + activateTotp( + totpSecret: string, + totpCode: string + ): Observable<{ success: boolean; recovery_codes: string[] }> { + return this.http.post<{ success: boolean; recovery_codes: string[] }>( + `${environment.apiBaseUrl}${this.endpoint}/totp/`, + { + secret: totpSecret, + code: totpCode, + } + ) + } + + deactivateTotp(): Observable { + return this.http.delete( + `${environment.apiBaseUrl}${this.endpoint}/totp/`, + {} + ) + } } diff --git a/src-ui/src/app/services/rest/user.service.spec.ts b/src-ui/src/app/services/rest/user.service.spec.ts index acf66340a..3fd682d4e 100644 --- a/src-ui/src/app/services/rest/user.service.spec.ts +++ b/src-ui/src/app/services/rest/user.service.spec.ts @@ -160,6 +160,18 @@ const user = { commonAbstractNameFilterPaperlessServiceTests(endpoint, UserService) describe('Additional service tests for UserService', () => { + beforeEach(() => { + // Dont need to setup again + + httpTestingController = TestBed.inject(HttpTestingController) + service = TestBed.inject(UserService) + }) + + afterEach(() => { + subscription?.unsubscribe() + httpTestingController.verify() + }) + it('should retain permissions on update', () => { subscription = service.listAll().subscribe() let req = httpTestingController.expectOne( @@ -179,15 +191,11 @@ describe('Additional service tests for UserService', () => { ) }) - beforeEach(() => { - // Dont need to setup again - - httpTestingController = TestBed.inject(HttpTestingController) - service = TestBed.inject(UserService) - }) - - afterEach(() => { - subscription?.unsubscribe() - httpTestingController.verify() + it('should deactivate totp', () => { + subscription = service.deactivateTotp(user).subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}${endpoint}/${user.id}/deactivate_totp/` + ) + expect(req.request.method).toEqual('POST') }) }) diff --git a/src-ui/src/app/services/rest/user.service.ts b/src-ui/src/app/services/rest/user.service.ts index 4fb02b1f7..ded7ae248 100644 --- a/src-ui/src/app/services/rest/user.service.ts +++ b/src-ui/src/app/services/rest/user.service.ts @@ -5,6 +5,7 @@ import { User } from 'src/app/data/user' import { PermissionsService } from '../permissions.service' import { AbstractNameFilterService } from './abstract-name-filter-service' +const endpoint = 'users' @Injectable({ providedIn: 'root', }) @@ -13,7 +14,7 @@ export class UserService extends AbstractNameFilterService { http: HttpClient, private permissionService: PermissionsService ) { - super(http, 'users') + super(http, endpoint) } update(o: User): Observable { @@ -31,4 +32,11 @@ export class UserService extends AbstractNameFilterService { }) ) } + + deactivateTotp(u: User): Observable { + return this.http.post( + `${this.getResourceUrl(u.id, 'deactivate_totp')}`, + null + ) + } } diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index 79d7cca6f..f49008cc7 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import TYPE_CHECKING import tqdm +from allauth.mfa.models import Authenticator from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.models import SocialApp from allauth.socialaccount.models import SocialToken @@ -270,6 +271,7 @@ class Command(CryptMixin, BaseCommand): "social_accounts": SocialAccount.objects.all(), "social_apps": SocialApp.objects.all(), "social_tokens": SocialToken.objects.all(), + "authenticators": Authenticator.objects.all(), } if settings.AUDIT_LOG_ENABLED: diff --git a/src/documents/templates/mfa/authenticate.html b/src/documents/templates/mfa/authenticate.html new file mode 100644 index 000000000..e6d54b8fb --- /dev/null +++ b/src/documents/templates/mfa/authenticate.html @@ -0,0 +1,35 @@ +{% extends "paperless-ngx/base.html" %} +{% load i18n %} +{% load allauth %} +{% load allauth static %} + +{% block head_title %} + {% trans "Paperless-ngx Two-Factor Authentication" %} +{% endblock head_title %} + +{% block form_top_content %} +

+ {% blocktranslate %}Your account is protected by two-factor authentication. Please enter an authenticator code:{% endblocktranslate %} +

+{% endblock form_top_content %} + +{% block form_content %} +{% translate "Code" as i18n_code %} +
+ + +
+
+ + +
+{% endblock form_content %} + +{% block after_form_content %} +
+ +{% csrf_token %} +
+{% endblock after_form_content %} diff --git a/src/documents/tests/test_api_permissions.py b/src/documents/tests/test_api_permissions.py index 7708b8541..eeea830cb 100644 --- a/src/documents/tests/test_api_permissions.py +++ b/src/documents/tests/test_api_permissions.py @@ -1,5 +1,6 @@ import json +from allauth.mfa.models import Authenticator from django.contrib.auth.models import Group from django.contrib.auth.models import Permission from django.contrib.auth.models import User @@ -601,6 +602,59 @@ class TestApiUser(DirectoriesMixin, APITestCase): self.assertEqual(returned_user2.first_name, "Updated Name 2") self.assertNotEqual(returned_user2.password, initial_password) + def test_deactivate_totp(self): + """ + GIVEN: + - Existing user account with TOTP enabled + WHEN: + - API request by a superuser is made to deactivate TOTP + - API request by a regular user is made to deactivate TOTP + THEN: + - TOTP is deactivated, if exists + - Regular user is forbidden from deactivating TOTP + """ + + user1 = User.objects.create( + username="testuser", + password="test", + first_name="Test", + last_name="User", + ) + Authenticator.objects.create( + user=user1, + type=Authenticator.Type.TOTP, + data={}, + ) + + response = self.client.post( + f"{self.ENDPOINT}{user1.pk}/deactivate_totp/", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(Authenticator.objects.filter(user=user1).count(), 0) + + # fail if already deactivated + response = self.client.post( + f"{self.ENDPOINT}{user1.pk}/deactivate_totp/", + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + regular_user = User.objects.create_user(username="regular_user") + regular_user.user_permissions.add( + *Permission.objects.all(), + ) + self.client.force_authenticate(regular_user) + Authenticator.objects.create( + user=user1, + type=Authenticator.Type.TOTP, + data={}, + ) + + response = self.client.post( + f"{self.ENDPOINT}{user1.pk}/deactivate_totp/", + ) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + class TestApiGroup(DirectoriesMixin, APITestCase): ENDPOINT = "/api/groups/" diff --git a/src/documents/tests/test_api_profile.py b/src/documents/tests/test_api_profile.py index eede0d2b0..1075a0af8 100644 --- a/src/documents/tests/test_api_profile.py +++ b/src/documents/tests/test_api_profile.py @@ -1,5 +1,6 @@ from unittest import mock +from allauth.mfa.models import Authenticator from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.models import SocialApp from django.contrib.auth.models import User @@ -299,3 +300,82 @@ class TestApiProfile(DirectoriesMixin, APITestCase): len(self.user.socialaccount_set.filter(pk=social_account_id)), 0, ) + + +class TestApiTOTPViews(APITestCase): + ENDPOINT = "/api/profile/totp/" + + def setUp(self): + super().setUp() + + self.user = User.objects.create_superuser(username="temp_admin") + self.client.force_authenticate(user=self.user) + + def test_get_totp(self): + """ + GIVEN: + - Existing user account + WHEN: + - API request is made to TOTP endpoint + THEN: + - TOTP is generated + """ + response = self.client.get( + self.ENDPOINT, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("qr_svg", response.data) + self.assertIn("secret", response.data) + + @mock.patch("allauth.mfa.totp.internal.auth.validate_totp_code") + def test_activate_totp(self, mock_validate_totp_code): + """ + GIVEN: + - Existing user account + WHEN: + - API request is made to activate TOTP + THEN: + - TOTP is activated, recovery codes are returned + """ + mock_validate_totp_code.return_value = True + + response = self.client.post( + self.ENDPOINT, + data={ + "secret": "123", + "code": "456", + }, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertTrue(Authenticator.objects.filter(user=self.user).exists()) + self.assertIn("recovery_codes", response.data) + + def test_deactivate_totp(self): + """ + GIVEN: + - Existing user account with TOTP enabled + WHEN: + - API request is made to deactivate TOTP + THEN: + - TOTP is deactivated + """ + Authenticator.objects.create( + user=self.user, + type=Authenticator.Type.TOTP, + data={}, + ) + + response = self.client.delete( + self.ENDPOINT, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(Authenticator.objects.filter(user=self.user).count(), 0) + + # test fails + response = self.client.delete( + self.ENDPOINT, + ) + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index 265682f91..0b7b65ab1 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-10-19 22:56-0700\n" +"POT-Creation-Date: 2024-10-19 23:22-0700\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1039,6 +1039,7 @@ msgid "Password" msgstr "" #: documents/templates/account/login.html:30 +#: documents/templates/mfa/authenticate.html:23 msgid "Sign in" msgstr "" @@ -1161,6 +1162,24 @@ msgstr "" msgid "Here's a link to the docs." msgstr "" +#: documents/templates/mfa/authenticate.html:7 +msgid "Paperless-ngx Two-Factor Authentication" +msgstr "" + +#: documents/templates/mfa/authenticate.html:12 +msgid "" +"Your account is protected by two-factor authentication. Please enter an " +"authenticator code:" +msgstr "" + +#: documents/templates/mfa/authenticate.html:17 +msgid "Code" +msgstr "" + +#: documents/templates/mfa/authenticate.html:24 +msgid "Cancel" +msgstr "" + #: documents/templates/paperless-ngx/base.html:58 msgid "Share link was not found." msgstr "" @@ -1366,139 +1385,139 @@ msgstr "" msgid "paperless application settings" msgstr "" -#: paperless/settings.py:684 +#: paperless/settings.py:687 msgid "English (US)" msgstr "" -#: paperless/settings.py:685 +#: paperless/settings.py:688 msgid "Arabic" msgstr "" -#: paperless/settings.py:686 +#: paperless/settings.py:689 msgid "Afrikaans" msgstr "" -#: paperless/settings.py:687 +#: paperless/settings.py:690 msgid "Belarusian" msgstr "" -#: paperless/settings.py:688 +#: paperless/settings.py:691 msgid "Bulgarian" msgstr "" -#: paperless/settings.py:689 +#: paperless/settings.py:692 msgid "Catalan" msgstr "" -#: paperless/settings.py:690 +#: paperless/settings.py:693 msgid "Czech" msgstr "" -#: paperless/settings.py:691 +#: paperless/settings.py:694 msgid "Danish" msgstr "" -#: paperless/settings.py:692 +#: paperless/settings.py:695 msgid "German" msgstr "" -#: paperless/settings.py:693 +#: paperless/settings.py:696 msgid "Greek" msgstr "" -#: paperless/settings.py:694 +#: paperless/settings.py:697 msgid "English (GB)" msgstr "" -#: paperless/settings.py:695 +#: paperless/settings.py:698 msgid "Spanish" msgstr "" -#: paperless/settings.py:696 +#: paperless/settings.py:699 msgid "Finnish" msgstr "" -#: paperless/settings.py:697 +#: paperless/settings.py:700 msgid "French" msgstr "" -#: paperless/settings.py:698 +#: paperless/settings.py:701 msgid "Hungarian" msgstr "" -#: paperless/settings.py:699 +#: paperless/settings.py:702 msgid "Italian" msgstr "" -#: paperless/settings.py:700 +#: paperless/settings.py:703 msgid "Japanese" msgstr "" -#: paperless/settings.py:701 +#: paperless/settings.py:704 msgid "Korean" msgstr "" -#: paperless/settings.py:702 +#: paperless/settings.py:705 msgid "Luxembourgish" msgstr "" -#: paperless/settings.py:703 +#: paperless/settings.py:706 msgid "Norwegian" msgstr "" -#: paperless/settings.py:704 +#: paperless/settings.py:707 msgid "Dutch" msgstr "" -#: paperless/settings.py:705 +#: paperless/settings.py:708 msgid "Polish" msgstr "" -#: paperless/settings.py:706 +#: paperless/settings.py:709 msgid "Portuguese (Brazil)" msgstr "" -#: paperless/settings.py:707 +#: paperless/settings.py:710 msgid "Portuguese" msgstr "" -#: paperless/settings.py:708 +#: paperless/settings.py:711 msgid "Romanian" msgstr "" -#: paperless/settings.py:709 +#: paperless/settings.py:712 msgid "Russian" msgstr "" -#: paperless/settings.py:710 +#: paperless/settings.py:713 msgid "Slovak" msgstr "" -#: paperless/settings.py:711 +#: paperless/settings.py:714 msgid "Slovenian" msgstr "" -#: paperless/settings.py:712 +#: paperless/settings.py:715 msgid "Serbian" msgstr "" -#: paperless/settings.py:713 +#: paperless/settings.py:716 msgid "Swedish" msgstr "" -#: paperless/settings.py:714 +#: paperless/settings.py:717 msgid "Turkish" msgstr "" -#: paperless/settings.py:715 +#: paperless/settings.py:718 msgid "Ukrainian" msgstr "" -#: paperless/settings.py:716 +#: paperless/settings.py:719 msgid "Chinese Simplified" msgstr "" -#: paperless/urls.py:254 +#: paperless/urls.py:268 msgid "Paperless-ngx administration" msgstr "" diff --git a/src/paperless/serialisers.py b/src/paperless/serialisers.py index 52f9e2b33..d5acfe465 100644 --- a/src/paperless/serialisers.py +++ b/src/paperless/serialisers.py @@ -1,5 +1,6 @@ import logging +from allauth.mfa.adapter import get_adapter as get_mfa_adapter from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import Group from django.contrib.auth.models import Permission @@ -32,6 +33,11 @@ class UserSerializer(serializers.ModelSerializer): required=False, ) inherited_permissions = serializers.SerializerMethodField() + is_mfa_enabled = serializers.SerializerMethodField() + + def get_is_mfa_enabled(self, user: User): + mfa_adapter = get_mfa_adapter() + return mfa_adapter.is_mfa_enabled(user) class Meta: model = User @@ -49,6 +55,7 @@ class UserSerializer(serializers.ModelSerializer): "groups", "user_permissions", "inherited_permissions", + "is_mfa_enabled", ) def get_inherited_permissions(self, obj): @@ -130,6 +137,11 @@ class ProfileSerializer(serializers.ModelSerializer): read_only=True, source="socialaccount_set", ) + is_mfa_enabled = serializers.SerializerMethodField() + + def get_is_mfa_enabled(self, user: User): + mfa_adapter = get_mfa_adapter() + return mfa_adapter.is_mfa_enabled(user) class Meta: model = User @@ -141,6 +153,7 @@ class ProfileSerializer(serializers.ModelSerializer): "auth_token", "social_accounts", "has_usable_password", + "is_mfa_enabled", ) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index d6489fa81..e5f31800f 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -316,6 +316,7 @@ INSTALLED_APPS = [ "allauth", "allauth.account", "allauth.socialaccount", + "allauth.mfa", *env_apps, ] @@ -458,6 +459,8 @@ SOCIALACCOUNT_PROVIDERS = json.loads( os.getenv("PAPERLESS_SOCIALACCOUNT_PROVIDERS", "{}"), ) +MFA_TOTP_ISSUER = "Paperless-ngx" + ACCOUNT_EMAIL_SUBJECT_PREFIX = "[Paperless-ngx] " DISABLE_REGULAR_LOGIN = __get_boolean("PAPERLESS_DISABLE_REGULAR_LOGIN") diff --git a/src/paperless/urls.py b/src/paperless/urls.py index a024c925a..2ebd7e739 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -1,6 +1,7 @@ import os from allauth.account import views as allauth_account_views +from allauth.mfa.base import views as allauth_mfa_views from allauth.socialaccount import views as allauth_social_account_views from allauth.urls import build_provider_urlpatterns from django.conf import settings @@ -54,6 +55,7 @@ from paperless.views import GenerateAuthTokenView from paperless.views import GroupViewSet from paperless.views import ProfileView from paperless.views import SocialAccountProvidersView +from paperless.views import TOTPView from paperless.views import UserViewSet from paperless_mail.views import MailAccountTestView from paperless_mail.views import MailAccountViewSet @@ -146,19 +148,34 @@ urlpatterns = [ BulkEditObjectsView.as_view(), name="bulk_edit_objects", ), - path("profile/generate_auth_token/", GenerateAuthTokenView.as_view()), - path( - "profile/disconnect_social_account/", - DisconnectSocialAccountView.as_view(), - ), - path( - "profile/social_account_providers/", - SocialAccountProvidersView.as_view(), - ), re_path( "^profile/", - ProfileView.as_view(), - name="profile_view", + include( + [ + path( + "generate_auth_token/", + GenerateAuthTokenView.as_view(), + ), + path( + "disconnect_social_account/", + DisconnectSocialAccountView.as_view(), + ), + path( + "social_account_providers/", + SocialAccountProvidersView.as_view(), + ), + re_path( + "^$", + ProfileView.as_view(), + name="profile_view", + ), + path( + "totp/", + TOTPView.as_view(), + name="totp_view", + ), + ], + ), ), re_path( "^status/", @@ -296,6 +313,12 @@ urlpatterns = [ ), ), *build_provider_urlpatterns(), + # mfa, see allauth/mfa/base/urls.py + path( + "2fa/authenticate/", + allauth_mfa_views.authenticate, + name="mfa_authenticate", + ), ], ), ), diff --git a/src/paperless/views.py b/src/paperless/views.py index 974830d83..b5142ed62 100644 --- a/src/paperless/views.py +++ b/src/paperless/views.py @@ -1,6 +1,12 @@ import os from collections import OrderedDict +from allauth.mfa import signals +from allauth.mfa.adapter import get_adapter as get_mfa_adapter +from allauth.mfa.base.internal.flows import delete_and_cleanup +from allauth.mfa.models import Authenticator +from allauth.mfa.recovery_codes.internal.flows import auto_generate_recovery_codes +from allauth.mfa.totp.internal import auth as totp_auth from allauth.socialaccount.adapter import get_adapter from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import Group @@ -8,9 +14,12 @@ from django.contrib.auth.models import User from django.db.models.functions import Lower from django.http import HttpResponse from django.http import HttpResponseBadRequest +from django.http import HttpResponseForbidden +from django.http import HttpResponseNotFound from django.views.generic import View from django_filters.rest_framework import DjangoFilterBackend from rest_framework.authtoken.models import Token +from rest_framework.decorators import action from rest_framework.filters import OrderingFilter from rest_framework.generics import GenericAPIView from rest_framework.pagination import PageNumberPagination @@ -100,6 +109,24 @@ class UserViewSet(ModelViewSet): filterset_class = UserFilterSet ordering_fields = ("username",) + @action(detail=True, methods=["post"]) + def deactivate_totp(self, request, pk=None): + request_user = request.user + user = User.objects.get(pk=pk) + if not request_user.is_superuser and request_user != user: + return HttpResponseForbidden( + "You do not have permission to deactivate TOTP for this user", + ) + authenticator = Authenticator.objects.filter( + user=user, + type=Authenticator.Type.TOTP, + ).first() + if authenticator is not None: + delete_and_cleanup(request, authenticator) + return Response(True) + else: + return HttpResponseNotFound("TOTP not found") + class GroupViewSet(ModelViewSet): model = Group @@ -145,6 +172,76 @@ class ProfileView(GenericAPIView): return Response(serializer.to_representation(user)) +class TOTPView(GenericAPIView): + """ + TOTP views + """ + + permission_classes = [IsAuthenticated] + + def get(self, request, *args, **kwargs): + """ + Generates a new TOTP secret and returns the URL and SVG + """ + user = self.request.user + mfa_adapter = get_mfa_adapter() + secret = totp_auth.get_totp_secret(regenerate=True) + url = mfa_adapter.build_totp_url(user, secret) + svg = mfa_adapter.build_totp_svg(url) + return Response( + { + "url": url, + "qr_svg": svg, + "secret": secret, + }, + ) + + def post(self, request, *args, **kwargs): + """ + Validates a TOTP code and activates the TOTP authenticator + """ + valid = totp_auth.validate_totp_code( + request.data["secret"], + request.data["code"], + ) + recovery_codes = None + if valid: + auth = totp_auth.TOTP.activate( + request.user, + request.data["secret"], + ).instance + signals.authenticator_added.send( + sender=Authenticator, + request=request, + user=request.user, + authenticator=auth, + ) + rc_auth: Authenticator = auto_generate_recovery_codes(request) + if rc_auth: + recovery_codes = rc_auth.wrap().get_unused_codes() + return Response( + { + "success": valid, + "recovery_codes": recovery_codes, + }, + ) + + def delete(self, request, *args, **kwargs): + """ + Deactivates the TOTP authenticator + """ + user = self.request.user + authenticator = Authenticator.objects.filter( + user=user, + type=Authenticator.Type.TOTP, + ).first() + if authenticator is not None: + delete_and_cleanup(request, authenticator) + return Response(True) + else: + return HttpResponseNotFound("TOTP not found") + + class GenerateAuthTokenView(GenericAPIView): """ Generates (or re-generates) an auth token, requires a logged in user From e4578b45895654a31fb40cdf1afc735f2ee21160 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 18 Nov 2024 11:03:56 -0800 Subject: [PATCH 4/9] Chore: change doc detail icon --- src-ui/src/app/app.module.ts | 4 ++-- .../app-frame/global-search/global-search.component.html | 4 ++-- .../document-card-large/document-card-large.component.html | 2 +- .../document-card-small/document-card-small.component.html | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index 08b3348e7..a0e23193d 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -145,7 +145,6 @@ import { asterisk, braces, bodyText, - boxArrowInRight, boxArrowUp, boxArrowUpRight, boxes, @@ -186,6 +185,7 @@ import { fileEarmarkFill, fileEarmarkLock, fileEarmarkMinus, + fileEarmarkRichtext, files, fileText, filter, @@ -253,7 +253,6 @@ const icons = { asterisk, braces, bodyText, - boxArrowInRight, boxArrowUp, boxArrowUpRight, boxes, @@ -294,6 +293,7 @@ const icons = { fileEarmarkFill, fileEarmarkLock, fileEarmarkMinus, + fileEarmarkRichtext, files, fileText, filter, diff --git a/src-ui/src/app/components/app-frame/global-search/global-search.component.html b/src-ui/src/app/components/app-frame/global-search/global-search.component.html index 2f5104547..3399b4fad 100644 --- a/src-ui/src/app/components/app-frame/global-search/global-search.component.html +++ b/src-ui/src/app/components/app-frame/global-search/global-search.component.html @@ -49,7 +49,7 @@ [disabled]="disablePrimaryButton(type, item)" (mouseenter)="onButtonHover($event)"> @if (type === DataType.Document) { - +  Open } @else if (type === DataType.SavedView) { @@ -72,7 +72,7 @@  Download } @else { - +  Open } diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html index a433a6198..04f3a236a 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html @@ -54,7 +54,7 @@  More like this -  Open +  Open
- + Date: Mon, 18 Nov 2024 11:05:00 -0800 Subject: [PATCH 5/9] Enhancement: next / previous shortcuts for document list (#8309) --- src-ui/messages.xlf | 18 +++++++++++-- .../document-list.component.spec.ts | 26 +++++++++++++++++++ .../document-list/document-list.component.ts | 24 +++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index e988a39cb..3eee47eb8 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -7405,18 +7405,32 @@ 264 + + Previous page + + src/app/components/document-list/document-list.component.ts + 280 + + + + Next page + + src/app/components/document-list/document-list.component.ts + 292 + + View "" saved successfully. src/app/components/document-list/document-list.component.ts - 300 + 324 View "" created successfully. src/app/components/document-list/document-list.component.ts - 343 + 367 diff --git a/src-ui/src/app/components/document-list/document-list.component.spec.ts b/src-ui/src/app/components/document-list/document-list.component.spec.ts index 0a8faa4d3..823fb6e6c 100644 --- a/src-ui/src/app/components/document-list/document-list.component.spec.ts +++ b/src-ui/src/app/components/document-list/document-list.component.spec.ts @@ -698,5 +698,31 @@ describe('DocumentListComponent', () => { fixture.detectChanges() document.dispatchEvent(new KeyboardEvent('keydown', { key: 'o' })) expect(detailSpy).toHaveBeenCalledWith(docs[1].id) + + const lotsOfDocs: Document[] = Array.from({ length: 100 }, (_, i) => ({ + id: i + 1, + title: `Doc${i + 1}`, + notes: [], + tags$: new Subject(), + content: `document content ${i + 1}`, + })) + jest + .spyOn(documentListService, 'documents', 'get') + .mockReturnValue(lotsOfDocs) + jest + .spyOn(documentService, 'listAllFilteredIds') + .mockReturnValue(of(lotsOfDocs.map((d) => d.id))) + jest.spyOn(documentListService, 'getLastPage').mockReturnValue(4) + fixture.detectChanges() + + expect(component.list.currentPage).toEqual(1) + document.dispatchEvent( + new KeyboardEvent('keydown', { key: 'ArrowRight', ctrlKey: true }) + ) + expect(component.list.currentPage).toEqual(2) + document.dispatchEvent( + new KeyboardEvent('keydown', { key: 'ArrowLeft', ctrlKey: true }) + ) + expect(component.list.currentPage).toEqual(1) }) }) diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index 1c559da05..c21c78fc8 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -273,6 +273,30 @@ export class DocumentListComponent } } }) + + this.hotKeyService + .addShortcut({ + keys: 'control.arrowleft', + description: $localize`Previous page`, + }) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(() => { + if (this.list.currentPage > 1) { + this.list.currentPage-- + } + }) + + this.hotKeyService + .addShortcut({ + keys: 'control.arrowright', + description: $localize`Next page`, + }) + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe(() => { + if (this.list.currentPage < this.list.getLastPage()) { + this.list.currentPage++ + } + }) } ngOnDestroy() { From 827121808a412ca5d30015789e1c6f0f41de444d Mon Sep 17 00:00:00 2001 From: Kevin Doren Date: Tue, 19 Nov 2024 07:20:24 -0800 Subject: [PATCH 6/9] Enhancement: Add --compare-json option to document_exporter to write json files only if changed (#8261) --- docs/administration.md | 4 +- .../management/commands/document_exporter.py | 72 ++++++++++++++----- .../tests/test_management_exporter.py | 7 ++ 3 files changed, 63 insertions(+), 20 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 7624de41b..8204352d8 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -241,6 +241,7 @@ document_exporter target [-c] [-d] [-f] [-na] [-nt] [-p] [-sm] [-z] optional arguments: -c, --compare-checksums +-cj, --compare-json -d, --delete -f, --use-filename-format -na, --no-archive @@ -269,7 +270,8 @@ only export changed and added files. Paperless determines whether a file has changed by inspecting the file attributes "date/time modified" and "size". If that does not work out for you, specify `-c` or `--compare-checksums` and paperless will attempt to compare file -checksums instead. This is slower. +checksums instead. This is slower. The manifest and metadata json files +are always updated, unless `cj` or `--compare-json` is specified. Paperless will not remove any existing files in the export directory. If you want paperless to also remove files that do not belong to the diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index f49008cc7..84275507d 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -82,6 +82,18 @@ class Command(CryptMixin, BaseCommand): ), ) + parser.add_argument( + "-cj", + "--compare-json", + default=False, + action="store_true", + help=( + "Compare json file checksums when determining whether to " + "export a json file or not (manifest or metadata). " + "If not specified, the file is always exported." + ), + ) + parser.add_argument( "-d", "--delete", @@ -178,6 +190,7 @@ class Command(CryptMixin, BaseCommand): self.target = Path(options["target"]).resolve() self.split_manifest: bool = options["split_manifest"] self.compare_checksums: bool = options["compare_checksums"] + self.compare_json: bool = options["compare_json"] self.use_filename_format: bool = options["use_filename_format"] self.use_folder_prefix: bool = options["use_folder_prefix"] self.delete: bool = options["delete"] @@ -343,12 +356,11 @@ class Command(CryptMixin, BaseCommand): manifest_dict["custom_field_instances"], ), ) - manifest_name.write_text( - json.dumps(content, indent=2, ensure_ascii=False), - encoding="utf-8", + + self.check_and_write_json( + content, + manifest_name, ) - if manifest_name in self.files_in_export_dir: - self.files_in_export_dir.remove(manifest_name) # These were exported already if self.split_manifest: @@ -361,12 +373,10 @@ class Command(CryptMixin, BaseCommand): for key in manifest_dict: manifest.extend(manifest_dict[key]) manifest_path = (self.target / "manifest.json").resolve() - manifest_path.write_text( - json.dumps(manifest, indent=2, ensure_ascii=False), - encoding="utf-8", + self.check_and_write_json( + manifest, + manifest_path, ) - if manifest_path in self.files_in_export_dir: - self.files_in_export_dir.remove(manifest_path) # 4.2 write version information to target folder extra_metadata_path = (self.target / "metadata.json").resolve() @@ -378,16 +388,11 @@ class Command(CryptMixin, BaseCommand): # Django stores most of these in the field itself, we store them once here if self.passphrase: metadata.update(self.get_crypt_params()) - extra_metadata_path.write_text( - json.dumps( - metadata, - indent=2, - ensure_ascii=False, - ), - encoding="utf-8", + + self.check_and_write_json( + metadata, + extra_metadata_path, ) - if extra_metadata_path in self.files_in_export_dir: - self.files_in_export_dir.remove(extra_metadata_path) if self.delete: # 5. Remove files which we did not explicitly export in this run @@ -516,6 +521,35 @@ class Command(CryptMixin, BaseCommand): archive_target, ) + def check_and_write_json( + self, + content: list[dict] | dict, + target: Path, + ): + """ + Writes the source content to the target json file. + If --compare-json arg was used, don't write to target file if + the file exists and checksum is identical to content checksum. + This preserves the file timestamps when no changes are made. + """ + + target = target.resolve() + perform_write = True + if target in self.files_in_export_dir: + self.files_in_export_dir.remove(target) + if self.compare_json: + target_checksum = hashlib.md5(target.read_bytes()).hexdigest() + src_str = json.dumps(content, indent=2, ensure_ascii=False) + src_checksum = hashlib.md5(src_str.encode("utf-8")).hexdigest() + if src_checksum == target_checksum: + perform_write = False + + if perform_write: + target.write_text( + json.dumps(content, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + def check_and_copy( self, source: Path, diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 9697f0c03..0a79b6cd7 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -153,6 +153,7 @@ class TestExportImport( *, use_filename_format=False, compare_checksums=False, + compare_json=False, delete=False, no_archive=False, no_thumbnail=False, @@ -165,6 +166,8 @@ class TestExportImport( args += ["--use-filename-format"] if compare_checksums: args += ["--compare-checksums"] + if compare_json: + args += ["--compare-json"] if delete: args += ["--delete"] if no_archive: @@ -340,6 +343,10 @@ class TestExportImport( self.assertNotEqual(st_mtime_1, st_mtime_2) self.assertNotEqual(st_mtime_2, st_mtime_3) + self._do_export(compare_json=True) + st_mtime_4 = os.stat(os.path.join(self.target, "manifest.json")).st_mtime + self.assertEqual(st_mtime_3, st_mtime_4) + def test_update_export_changed_checksum(self): shutil.rmtree(os.path.join(self.dirs.media_dir, "documents")) shutil.copytree( From d7d3fed833da2654db43f518ca93ec69964bc288 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:50:21 -0800 Subject: [PATCH 7/9] Chore(deps): Bump watchdog in the major-versions group (#8257) Bumps the major-versions group with 1 update: [watchdog](https://github.com/gorakhargosh/watchdog). Updates `watchdog` from 5.0.3 to 6.0.0 - [Release notes](https://github.com/gorakhargosh/watchdog/releases) - [Changelog](https://github.com/gorakhargosh/watchdog/blob/master/changelog.rst) - [Commits](https://github.com/gorakhargosh/watchdog/compare/v5.0.3...v6.0.0) --- updated-dependencies: - dependency-name: watchdog dependency-type: direct:production update-type: version-update:semver-major dependency-group: major-versions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Pipfile | 2 +- Pipfile.lock | 64 ++++++++++++++++++++++++++-------------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/Pipfile b/Pipfile index 1e79bb604..335d9bff3 100644 --- a/Pipfile +++ b/Pipfile @@ -55,7 +55,7 @@ tika-client = "*" tqdm = "*" # See https://github.com/paperless-ngx/paperless-ngx/issues/5494 uvicorn = {extras = ["standard"], version = "==0.25.0"} -watchdog = "~=5.0" +watchdog = "~=6.0" whitenoise = "~=6.8" whoosh = "~=2.7" zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"} diff --git a/Pipfile.lock b/Pipfile.lock index 063ec2389..26a2c3377 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5a7cb70103e8f3931682c73432290f2f4ec2ba06395c8ec076d2d5449c4ff0dd" + "sha256": "e4cb2328c49829f56793ef25780dcc73ea8e4838e6e9bc25d1b6feb74eb3befe" }, "pipfile-spec": 6, "requires": {}, @@ -2351,40 +2351,40 @@ }, "watchdog": { "hashes": [ - "sha256:0f9332243355643d567697c3e3fa07330a1d1abf981611654a1f2bf2175612b7", - "sha256:1021223c08ba8d2d38d71ec1704496471ffd7be42cfb26b87cd5059323a389a1", - "sha256:108f42a7f0345042a854d4d0ad0834b741d421330d5f575b81cb27b883500176", - "sha256:1e9679245e3ea6498494b3028b90c7b25dbb2abe65c7d07423ecfc2d6218ff7c", - "sha256:223160bb359281bb8e31c8f1068bf71a6b16a8ad3d9524ca6f523ac666bb6a1e", - "sha256:26dd201857d702bdf9d78c273cafcab5871dd29343748524695cecffa44a8d97", - "sha256:294b7a598974b8e2c6123d19ef15de9abcd282b0fbbdbc4d23dfa812959a9e05", - "sha256:349c9488e1d85d0a58e8cb14222d2c51cbc801ce11ac3936ab4c3af986536926", - "sha256:49f4d36cb315c25ea0d946e018c01bb028048023b9e103d3d3943f58e109dd45", - "sha256:53a3f10b62c2d569e260f96e8d966463dec1a50fa4f1b22aec69e3f91025060e", - "sha256:53adf73dcdc0ef04f7735066b4a57a4cd3e49ef135daae41d77395f0b5b692cb", - "sha256:560135542c91eaa74247a2e8430cf83c4342b29e8ad4f520ae14f0c8a19cfb5b", - "sha256:720ef9d3a4f9ca575a780af283c8fd3a0674b307651c1976714745090da5a9e8", - "sha256:752fb40efc7cc8d88ebc332b8f4bcbe2b5cc7e881bccfeb8e25054c00c994ee3", - "sha256:78864cc8f23dbee55be34cc1494632a7ba30263951b5b2e8fc8286b95845f82c", - "sha256:85527b882f3facda0579bce9d743ff7f10c3e1e0db0a0d0e28170a7d0e5ce2ea", - "sha256:90a67d7857adb1d985aca232cc9905dd5bc4803ed85cfcdcfcf707e52049eda7", - "sha256:91b522adc25614cdeaf91f7897800b82c13b4b8ac68a42ca959f992f6990c490", - "sha256:9413384f26b5d050b6978e6fcd0c1e7f0539be7a4f1a885061473c5deaa57221", - "sha256:94d11b07c64f63f49876e0ab8042ae034674c8653bfcdaa8c4b32e71cfff87e8", - "sha256:950f531ec6e03696a2414b6308f5c6ff9dab7821a768c9d5788b1314e9a46ca7", - "sha256:a2e8f3f955d68471fa37b0e3add18500790d129cc7efe89971b8a4cc6fdeb0b2", - "sha256:ae6deb336cba5d71476caa029ceb6e88047fc1dc74b62b7c4012639c0b563906", - "sha256:b8ca4d854adcf480bdfd80f46fdd6fb49f91dd020ae11c89b3a79e19454ec627", - "sha256:c66f80ee5b602a9c7ab66e3c9f36026590a0902db3aea414d59a2f55188c1f49", - "sha256:d52db5beb5e476e6853da2e2d24dbbbed6797b449c8bf7ea118a4ee0d2c9040e", - "sha256:dd021efa85970bd4824acacbb922066159d0f9e546389a4743d56919b6758b91", - "sha256:e25adddab85f674acac303cf1f5835951345a56c5f7f582987d266679979c75b", - "sha256:f00b4cf737f568be9665563347a910f8bdc76f88c2970121c86243c8cfdf90e9", - "sha256:f01f4a3565a387080dc49bdd1fefe4ecc77f894991b88ef927edbfa45eb10818" + "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", + "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", + "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", + "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", + "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", + "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", + "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", + "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", + "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", + "sha256:7a0e56874cfbc4b9b05c60c8a1926fedf56324bb08cfbc188969777940aef3aa", + "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", + "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", + "sha256:90c8e78f3b94014f7aaae121e6b909674df5b46ec24d6bebc45c44c56729af2a", + "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11", + "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", + "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", + "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", + "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", + "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112", + "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", + "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881", + "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", + "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3", + "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", + "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26", + "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", + "sha256:e6439e374fc012255b4ec786ae3c4bc838cd7309a540e5fe0952d03687d8804e", + "sha256:e6f0e77c9417e7cd62af82529b10563db3423625c5fce018430b249bf977f9e8", + "sha256:e7631a77ffb1f7d2eefa4445ebbee491c720a5661ddf6df3498ebecae5ed375c", + "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==5.0.3" + "version": "==6.0.0" }, "watchfiles": { "hashes": [ From 9c1561adfbeecab3fdb30b9a2988f63ccb3f1cbe Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:01:13 -0800 Subject: [PATCH 8/9] Change: change update content to handle archive disabled (#8315) --- src/documents/bulk_edit.py | 8 +- .../management/commands/document_archiver.py | 6 +- src/documents/tasks.py | 48 ++++++++---- src/documents/tests/test_bulk_edit.py | 10 +-- src/documents/tests/test_management.py | 10 +-- src/documents/tests/test_tasks.py | 74 +++++++++++++++++++ 6 files changed, 124 insertions(+), 32 deletions(-) diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 4d03769c8..83be5eea9 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -24,7 +24,7 @@ from documents.models import StoragePath from documents.permissions import set_permissions_for_object from documents.tasks import bulk_update_documents from documents.tasks import consume_file -from documents.tasks import update_document_archive_file +from documents.tasks import update_document_content_maybe_archive_file logger: logging.Logger = logging.getLogger("paperless.bulk_edit") @@ -191,7 +191,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]: def reprocess(doc_ids: list[int]) -> Literal["OK"]: for document_id in doc_ids: - update_document_archive_file.delay( + update_document_content_maybe_archive_file.delay( document_id=document_id, ) @@ -245,7 +245,7 @@ def rotate(doc_ids: list[int], degrees: int) -> Literal["OK"]: doc.checksum = hashlib.md5(doc.source_path.read_bytes()).hexdigest() doc.save() rotate_tasks.append( - update_document_archive_file.s( + update_document_content_maybe_archive_file.s( document_id=doc.id, ), ) @@ -423,7 +423,7 @@ def delete_pages(doc_ids: list[int], pages: list[int]) -> Literal["OK"]: if doc.page_count is not None: doc.page_count = doc.page_count - len(pages) doc.save() - update_document_archive_file.delay(document_id=doc.id) + update_document_content_maybe_archive_file.delay(document_id=doc.id) logger.info(f"Deleted pages {pages} from document {doc.id}") except Exception as e: logger.exception(f"Error deleting pages from document {doc.id}: {e}") diff --git a/src/documents/management/commands/document_archiver.py b/src/documents/management/commands/document_archiver.py index 12677dd79..1aa52117a 100644 --- a/src/documents/management/commands/document_archiver.py +++ b/src/documents/management/commands/document_archiver.py @@ -9,7 +9,7 @@ from django.core.management.base import BaseCommand from documents.management.commands.mixins import MultiProcessMixin from documents.management.commands.mixins import ProgressBarMixin from documents.models import Document -from documents.tasks import update_document_archive_file +from documents.tasks import update_document_content_maybe_archive_file logger = logging.getLogger("paperless.management.archiver") @@ -77,13 +77,13 @@ class Command(MultiProcessMixin, ProgressBarMixin, BaseCommand): if self.process_count == 1: for doc_id in document_ids: - update_document_archive_file(doc_id) + update_document_content_maybe_archive_file(doc_id) else: # pragma: no cover with multiprocessing.Pool(self.process_count) as pool: list( tqdm.tqdm( pool.imap_unordered( - update_document_archive_file, + update_document_content_maybe_archive_file, document_ids, ), total=len(document_ids), diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 8f5ee51bc..e04cdb34e 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -206,9 +206,10 @@ def bulk_update_documents(document_ids): @shared_task -def update_document_archive_file(document_id): +def update_document_content_maybe_archive_file(document_id): """ - Re-creates the archive file of a document, including new OCR content and thumbnail + Re-creates OCR content and thumbnail for a document, and archive file if + it exists. """ document = Document.objects.get(id=document_id) @@ -234,8 +235,9 @@ def update_document_archive_file(document_id): document.get_public_filename(), ) - if parser.get_archive_path(): - with transaction.atomic(): + with transaction.atomic(): + oldDocument = Document.objects.get(pk=document.pk) + if parser.get_archive_path(): with open(parser.get_archive_path(), "rb") as f: checksum = hashlib.md5(f.read()).hexdigest() # I'm going to save first so that in case the file move @@ -246,7 +248,6 @@ def update_document_archive_file(document_id): document, archive_filename=True, ) - oldDocument = Document.objects.get(pk=document.pk) Document.objects.filter(pk=document.pk).update( archive_checksum=checksum, content=parser.get_text(), @@ -268,24 +269,41 @@ def update_document_archive_file(document_id): ], }, additional_data={ - "reason": "Update document archive file", + "reason": "Update document content", + }, + action=LogEntry.Action.UPDATE, + ) + else: + Document.objects.filter(pk=document.pk).update( + content=parser.get_text(), + ) + + if settings.AUDIT_LOG_ENABLED: + LogEntry.objects.log_create( + instance=oldDocument, + changes={ + "content": [oldDocument.content, parser.get_text()], + }, + additional_data={ + "reason": "Update document content", }, action=LogEntry.Action.UPDATE, ) - with FileLock(settings.MEDIA_LOCK): + with FileLock(settings.MEDIA_LOCK): + if parser.get_archive_path(): create_source_path_directory(document.archive_path) shutil.move(parser.get_archive_path(), document.archive_path) - shutil.move(thumbnail, document.thumbnail_path) + shutil.move(thumbnail, document.thumbnail_path) - document.refresh_from_db() - logger.info( - f"Updating index for document {document_id} ({document.archive_checksum})", - ) - with index.open_index_writer() as writer: - index.update_document(writer, document) + document.refresh_from_db() + logger.info( + f"Updating index for document {document_id} ({document.archive_checksum})", + ) + with index.open_index_writer() as writer: + index.update_document(writer, document) - clear_document_caches(document.pk) + clear_document_caches(document.pk) except Exception: logger.exception( diff --git a/src/documents/tests/test_bulk_edit.py b/src/documents/tests/test_bulk_edit.py index c6e846a77..bb5ebf04d 100644 --- a/src/documents/tests/test_bulk_edit.py +++ b/src/documents/tests/test_bulk_edit.py @@ -607,7 +607,7 @@ class TestPDFActions(DirectoriesMixin, TestCase): mock_consume_file.assert_not_called() @mock.patch("documents.tasks.bulk_update_documents.si") - @mock.patch("documents.tasks.update_document_archive_file.s") + @mock.patch("documents.tasks.update_document_content_maybe_archive_file.s") @mock.patch("celery.chord.delay") def test_rotate(self, mock_chord, mock_update_document, mock_update_documents): """ @@ -626,7 +626,7 @@ class TestPDFActions(DirectoriesMixin, TestCase): self.assertEqual(result, "OK") @mock.patch("documents.tasks.bulk_update_documents.si") - @mock.patch("documents.tasks.update_document_archive_file.s") + @mock.patch("documents.tasks.update_document_content_maybe_archive_file.s") @mock.patch("pikepdf.Pdf.save") def test_rotate_with_error( self, @@ -654,7 +654,7 @@ class TestPDFActions(DirectoriesMixin, TestCase): mock_update_archive_file.assert_not_called() @mock.patch("documents.tasks.bulk_update_documents.si") - @mock.patch("documents.tasks.update_document_archive_file.s") + @mock.patch("documents.tasks.update_document_content_maybe_archive_file.s") @mock.patch("celery.chord.delay") def test_rotate_non_pdf( self, @@ -680,7 +680,7 @@ class TestPDFActions(DirectoriesMixin, TestCase): mock_chord.assert_called_once() self.assertEqual(result, "OK") - @mock.patch("documents.tasks.update_document_archive_file.delay") + @mock.patch("documents.tasks.update_document_content_maybe_archive_file.delay") @mock.patch("pikepdf.Pdf.save") def test_delete_pages(self, mock_pdf_save, mock_update_archive_file): """ @@ -705,7 +705,7 @@ class TestPDFActions(DirectoriesMixin, TestCase): self.doc2.refresh_from_db() self.assertEqual(self.doc2.page_count, expected_page_count) - @mock.patch("documents.tasks.update_document_archive_file.delay") + @mock.patch("documents.tasks.update_document_content_maybe_archive_file.delay") @mock.patch("pikepdf.Pdf.save") def test_delete_pages_with_error(self, mock_pdf_save, mock_update_archive_file): """ diff --git a/src/documents/tests/test_management.py b/src/documents/tests/test_management.py index 76a0a2c74..5340035e7 100644 --- a/src/documents/tests/test_management.py +++ b/src/documents/tests/test_management.py @@ -13,7 +13,7 @@ from django.test import override_settings from documents.file_handling import generate_filename from documents.models import Document -from documents.tasks import update_document_archive_file +from documents.tasks import update_document_content_maybe_archive_file from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin @@ -46,7 +46,7 @@ class TestArchiver(DirectoriesMixin, FileSystemAssertsMixin, TestCase): os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"), ) - update_document_archive_file(doc.pk) + update_document_content_maybe_archive_file(doc.pk) doc = Document.objects.get(id=doc.id) @@ -63,7 +63,7 @@ class TestArchiver(DirectoriesMixin, FileSystemAssertsMixin, TestCase): doc.save() shutil.copy(sample_file, doc.source_path) - update_document_archive_file(doc.pk) + update_document_content_maybe_archive_file(doc.pk) doc = Document.objects.get(id=doc.id) @@ -94,8 +94,8 @@ class TestArchiver(DirectoriesMixin, FileSystemAssertsMixin, TestCase): os.path.join(self.dirs.originals_dir, "document_01.pdf"), ) - update_document_archive_file(doc2.pk) - update_document_archive_file(doc1.pk) + update_document_content_maybe_archive_file(doc2.pk) + update_document_content_maybe_archive_file(doc1.pk) doc1 = Document.objects.get(id=doc1.id) doc2 = Document.objects.get(id=doc2.id) diff --git a/src/documents/tests/test_tasks.py b/src/documents/tests/test_tasks.py index a1d21b532..0f9f8511f 100644 --- a/src/documents/tests/test_tasks.py +++ b/src/documents/tests/test_tasks.py @@ -1,5 +1,7 @@ import os +import shutil from datetime import timedelta +from pathlib import Path from unittest import mock from django.conf import settings @@ -184,3 +186,75 @@ class TestEmptyTrashTask(DirectoriesMixin, FileSystemAssertsMixin, TestCase): tasks.empty_trash() self.assertEqual(Document.global_objects.count(), 0) + + +class TestUpdateContent(DirectoriesMixin, TestCase): + def test_update_content_maybe_archive_file(self): + """ + GIVEN: + - Existing document with archive file + WHEN: + - Update content task is called + THEN: + - Document is reprocessed, content and checksum are updated + """ + sample1 = self.dirs.scratch_dir / "sample.pdf" + shutil.copy( + Path(__file__).parent + / "samples" + / "documents" + / "originals" + / "0000001.pdf", + sample1, + ) + sample1_archive = self.dirs.archive_dir / "sample_archive.pdf" + shutil.copy( + Path(__file__).parent + / "samples" + / "documents" + / "originals" + / "0000001.pdf", + sample1_archive, + ) + doc = Document.objects.create( + title="test", + content="my document", + checksum="wow", + archive_checksum="wow", + filename=sample1, + mime_type="application/pdf", + archive_filename=sample1_archive, + ) + + tasks.update_document_content_maybe_archive_file(doc.pk) + self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test") + self.assertNotEqual(Document.objects.get(pk=doc.pk).archive_checksum, "wow") + + def test_update_content_maybe_archive_file_no_archive(self): + """ + GIVEN: + - Existing document without archive file + WHEN: + - Update content task is called + THEN: + - Document is reprocessed, content is updated + """ + sample1 = self.dirs.scratch_dir / "sample.pdf" + shutil.copy( + Path(__file__).parent + / "samples" + / "documents" + / "originals" + / "0000001.pdf", + sample1, + ) + doc = Document.objects.create( + title="test", + content="my document", + checksum="wow", + filename=sample1, + mime_type="application/pdf", + ) + + tasks.update_document_content_maybe_archive_file(doc.pk) + self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test") From 8bfe68743d99b61e3798ce1cc28f325e90876969 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:25:53 -0800 Subject: [PATCH 9/9] Enhancement: support owner permissions for file tasks (#8195) --- docs/api.md | 8 +++ src-ui/src/app/data/paperless-task.ts | 2 + src-ui/src/app/services/tasks.service.spec.ts | 2 +- src-ui/src/app/services/tasks.service.ts | 2 +- src-ui/src/environments/environment.prod.ts | 2 +- src-ui/src/environments/environment.ts | 2 +- .../migrations/1057_paperlesstask_owner.py | 28 +++++++++++ src/documents/models.py | 2 +- src/documents/serialisers.py | 3 +- src/documents/signals/handlers.py | 4 +- src/documents/tests/test_api_tasks.py | 50 ++++++++++++++++++- src/documents/tests/test_task_signals.py | 7 ++- src/documents/views.py | 19 ++++--- src/paperless/settings.py | 2 +- src/paperless/urls.py | 6 --- 15 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 src/documents/migrations/1057_paperlesstask_owner.py diff --git a/docs/api.md b/docs/api.md index ccbde9b22..c5f20edd1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -556,3 +556,11 @@ Initial API version. - Consumption templates were refactored to workflows and API endpoints changed as such. + +#### Version 5 + +- Added bulk deletion methods for documents and objects. + +#### Version 6 + +- Moved acknowledge tasks endpoint to be under `/api/tasks/acknowledge/`. diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts index 08b30d44b..d15f006d7 100644 --- a/src-ui/src/app/data/paperless-task.ts +++ b/src-ui/src/app/data/paperless-task.ts @@ -30,4 +30,6 @@ export interface PaperlessTask extends ObjectWithId { result?: string related_document?: number + + owner?: number } diff --git a/src-ui/src/app/services/tasks.service.spec.ts b/src-ui/src/app/services/tasks.service.spec.ts index 41a374831..d746707b7 100644 --- a/src-ui/src/app/services/tasks.service.spec.ts +++ b/src-ui/src/app/services/tasks.service.spec.ts @@ -48,7 +48,7 @@ describe('TasksService', () => { it('calls acknowledge_tasks api endpoint on dismiss and reloads', () => { tasksService.dismissTasks(new Set([1, 2, 3])) const req = httpTestingController.expectOne( - `${environment.apiBaseUrl}acknowledge_tasks/` + `${environment.apiBaseUrl}tasks/acknowledge/` ) expect(req.request.method).toEqual('POST') expect(req.request.body).toEqual({ diff --git a/src-ui/src/app/services/tasks.service.ts b/src-ui/src/app/services/tasks.service.ts index e2c064e03..c3c8f1d2b 100644 --- a/src-ui/src/app/services/tasks.service.ts +++ b/src-ui/src/app/services/tasks.service.ts @@ -64,7 +64,7 @@ export class TasksService { public dismissTasks(task_ids: Set) { this.http - .post(`${this.baseUrl}acknowledge_tasks/`, { + .post(`${this.baseUrl}tasks/acknowledge/`, { tasks: [...task_ids], }) .pipe(first()) diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index 76ba37891..ba01ac9b0 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -3,7 +3,7 @@ const base_url = new URL(document.baseURI) export const environment = { production: true, apiBaseUrl: document.baseURI + 'api/', - apiVersion: '5', + apiVersion: '6', appTitle: 'Paperless-ngx', version: '2.13.5', webSocketHost: window.location.host, diff --git a/src-ui/src/environments/environment.ts b/src-ui/src/environments/environment.ts index 18715e90f..6256f3ae3 100644 --- a/src-ui/src/environments/environment.ts +++ b/src-ui/src/environments/environment.ts @@ -5,7 +5,7 @@ export const environment = { production: false, apiBaseUrl: 'http://localhost:8000/api/', - apiVersion: '5', + apiVersion: '6', appTitle: 'Paperless-ngx', version: 'DEVELOPMENT', webSocketHost: 'localhost:8000', diff --git a/src/documents/migrations/1057_paperlesstask_owner.py b/src/documents/migrations/1057_paperlesstask_owner.py new file mode 100644 index 000000000..e9f108d3a --- /dev/null +++ b/src/documents/migrations/1057_paperlesstask_owner.py @@ -0,0 +1,28 @@ +# Generated by Django 5.1.1 on 2024-11-04 21:56 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "1056_customfieldinstance_deleted_at_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="paperlesstask", + name="owner", + field=models.ForeignKey( + blank=True, + default=None, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + verbose_name="owner", + ), + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 4528d5127..05226b0e9 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -641,7 +641,7 @@ class UiSettings(models.Model): return self.user.username -class PaperlessTask(models.Model): +class PaperlessTask(ModelWithOwner): ALL_STATES = sorted(states.ALL_STATES) TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 45bf672d8..f960cac24 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1567,7 +1567,7 @@ class UiSettingsViewSerializer(serializers.ModelSerializer): return ui_settings -class TasksViewSerializer(serializers.ModelSerializer): +class TasksViewSerializer(OwnedObjectSerializer): class Meta: model = PaperlessTask depth = 1 @@ -1582,6 +1582,7 @@ class TasksViewSerializer(serializers.ModelSerializer): "result", "acknowledged", "related_document", + "owner", ) type = serializers.SerializerMethodField() diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 114654c64..cd2e3972e 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -939,9 +939,10 @@ def before_task_publish_handler(sender=None, headers=None, body=None, **kwargs): close_old_connections() task_args = body[0] - input_doc, _ = task_args + input_doc, overrides = task_args task_file_name = input_doc.original_file.name + user_id = overrides.owner_id if overrides else None PaperlessTask.objects.create( task_id=headers["id"], @@ -952,6 +953,7 @@ def before_task_publish_handler(sender=None, headers=None, body=None, **kwargs): date_created=timezone.now(), date_started=None, date_done=None, + owner_id=user_id, ) except Exception: # pragma: no cover # Don't let an exception in the signal handlers prevent diff --git a/src/documents/tests/test_api_tasks.py b/src/documents/tests/test_api_tasks.py index 52ffb09fe..dd5425278 100644 --- a/src/documents/tests/test_api_tasks.py +++ b/src/documents/tests/test_api_tasks.py @@ -1,6 +1,7 @@ import uuid import celery +from django.contrib.auth.models import Permission from django.contrib.auth.models import User from rest_framework import status from rest_framework.test import APITestCase @@ -11,7 +12,6 @@ from documents.tests.utils import DirectoriesMixin class TestTasks(DirectoriesMixin, APITestCase): ENDPOINT = "/api/tasks/" - ENDPOINT_ACKNOWLEDGE = "/api/acknowledge_tasks/" def setUp(self): super().setUp() @@ -125,7 +125,7 @@ class TestTasks(DirectoriesMixin, APITestCase): self.assertEqual(len(response.data), 1) response = self.client.post( - self.ENDPOINT_ACKNOWLEDGE, + self.ENDPOINT + "acknowledge/", {"tasks": [task.id]}, ) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -133,6 +133,52 @@ class TestTasks(DirectoriesMixin, APITestCase): response = self.client.get(self.ENDPOINT) self.assertEqual(len(response.data), 0) + def test_tasks_owner_aware(self): + """ + GIVEN: + - Existing PaperlessTasks with owner and with no owner + WHEN: + - API call is made to get tasks + THEN: + - Only tasks with no owner or request user are returned + """ + + regular_user = User.objects.create_user(username="test") + regular_user.user_permissions.add(*Permission.objects.all()) + self.client.logout() + self.client.force_authenticate(user=regular_user) + + task1 = PaperlessTask.objects.create( + task_id=str(uuid.uuid4()), + task_file_name="task_one.pdf", + owner=self.user, + ) + + task2 = PaperlessTask.objects.create( + task_id=str(uuid.uuid4()), + task_file_name="task_two.pdf", + ) + + task3 = PaperlessTask.objects.create( + task_id=str(uuid.uuid4()), + task_file_name="task_three.pdf", + owner=regular_user, + ) + + response = self.client.get(self.ENDPOINT) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data), 2) + self.assertEqual(response.data[0]["task_id"], task3.task_id) + self.assertEqual(response.data[1]["task_id"], task2.task_id) + + acknowledge_response = self.client.post( + self.ENDPOINT + "acknowledge/", + {"tasks": [task1.id, task2.id, task3.id]}, + ) + self.assertEqual(acknowledge_response.status_code, status.HTTP_200_OK) + self.assertEqual(acknowledge_response.data, {"result": 2}) + def test_task_result_no_error(self): """ GIVEN: diff --git a/src/documents/tests/test_task_signals.py b/src/documents/tests/test_task_signals.py index 4a54220e0..a025fb9dc 100644 --- a/src/documents/tests/test_task_signals.py +++ b/src/documents/tests/test_task_signals.py @@ -5,6 +5,7 @@ import celery from django.test import TestCase from documents.data_models import ConsumableDocument +from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource from documents.models import PaperlessTask from documents.signals.handlers import before_task_publish_handler @@ -48,7 +49,10 @@ class TestTaskSignalHandler(DirectoriesMixin, TestCase): source=DocumentSource.ConsumeFolder, original_file="/consume/hello-999.pdf", ), - None, + DocumentMetadataOverrides( + title="Hello world", + owner_id=1, + ), ), # kwargs {}, @@ -65,6 +69,7 @@ class TestTaskSignalHandler(DirectoriesMixin, TestCase): self.assertEqual(headers["id"], task.task_id) self.assertEqual("hello-999.pdf", task.task_file_name) self.assertEqual("documents.tasks.consume_file", task.task_name) + self.assertEqual(1, task.owner_id) self.assertEqual(celery.states.PENDING, task.status) def test_task_prerun_handler(self): diff --git a/src/documents/views.py b/src/documents/views.py index 2d0c030f4..332d5f64a 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -1705,6 +1705,7 @@ class RemoteVersionView(GenericAPIView): class TasksViewSet(ReadOnlyModelViewSet): permission_classes = (IsAuthenticated, PaperlessObjectPermissions) serializer_class = TasksViewSerializer + filter_backends = (ObjectOwnedOrGrantedPermissionsFilter,) def get_queryset(self): queryset = ( @@ -1719,19 +1720,17 @@ class TasksViewSet(ReadOnlyModelViewSet): queryset = PaperlessTask.objects.filter(task_id=task_id) return queryset - -class AcknowledgeTasksView(GenericAPIView): - permission_classes = (IsAuthenticated,) - serializer_class = AcknowledgeTasksViewSerializer - - def post(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) + @action(methods=["post"], detail=False) + def acknowledge(self, request): + serializer = AcknowledgeTasksViewSerializer(data=request.data) serializer.is_valid(raise_exception=True) - - tasks = serializer.validated_data.get("tasks") + task_ids = serializer.validated_data.get("tasks") try: - result = PaperlessTask.objects.filter(id__in=tasks).update( + tasks = PaperlessTask.objects.filter(id__in=task_ids) + if request.user is not None and not request.user.is_superuser: + tasks = tasks.filter(owner=request.user) | tasks.filter(owner=None) + result = tasks.update( acknowledged=True, ) return Response({"result": result}) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index e5f31800f..1a495de09 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -333,7 +333,7 @@ REST_FRAMEWORK = { "DEFAULT_VERSION": "1", # Make sure these are ordered and that the most recent version appears # last - "ALLOWED_VERSIONS": ["1", "2", "3", "4", "5"], + "ALLOWED_VERSIONS": ["1", "2", "3", "4", "5", "6"], } if DEBUG: diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 2ebd7e739..5b7327b8d 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -18,7 +18,6 @@ from django.views.static import serve from rest_framework.authtoken import views from rest_framework.routers import DefaultRouter -from documents.views import AcknowledgeTasksView from documents.views import BulkDownloadView from documents.views import BulkEditObjectsView from documents.views import BulkEditView @@ -132,11 +131,6 @@ urlpatterns = [ name="remoteversion", ), re_path("^ui_settings/", UiSettingsView.as_view(), name="ui_settings"), - re_path( - "^acknowledge_tasks/", - AcknowledgeTasksView.as_view(), - name="acknowledge_tasks", - ), re_path( "^mail_accounts/test/", MailAccountTestView.as_view(),