From 061f33fb0516da84df8aca4575a2a17b2335a4d8 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Fri, 29 Dec 2023 15:42:56 -0800 Subject: [PATCH] Feature: Allow setting backend configuration settings via the UI (#5126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Saving some start on this * At least partially working for the tesseract parser * Problems with migration testing need to figure out * Work around that error * Fixes max m_pixels * Moving the settings to main paperless application * Starting some consumer options * More fixes and work * Fixes these last tests * Fix max_length on OcrSettings.mode field * Fix all fields on Common & Ocr settings serializers * Umbrellla config view * Revert "Umbrellla config view" This reverts commit fbaf9f4be30f89afeb509099180158a3406416a5. * Updates to use a single configuration object for all settings * Squashed commit of the following: commit 8a0a49dd5766094f60462fbfbe62e9921fbd2373 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 23:02:47 2023 -0800 Fix formatting commit 66b2d90c507b8afd9507813ff555e46198ea33b9 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 22:36:35 2023 -0800 Refactor frontend data models commit 5723bd8dd823ee855625e250df39393e26709d48 Author: Adam Bogdał Date: Wed Dec 20 01:17:43 2023 +0100 Fix: speed up admin panel for installs with a large number of documents (#5052) commit 9b08ce176199bf9011a6634bb88f616846150d2b Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 15:18:51 2023 -0800 Update PULL_REQUEST_TEMPLATE.md commit a6248bec2d793b7690feed95fcaf5eb34a75bfb6 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 15:02:05 2023 -0800 Chore: Update Angular to v17 (#4980) commit b1f6f52486d5ba5c04af99b41315eb6428fd1fa8 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 13:53:56 2023 -0800 Fix: Dont allow null custom_fields property via API (#5063) commit 638d9970fd468d8c02c91d19bd28f8b0796bdcb1 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 13:43:50 2023 -0800 Enhancement: symmetric document links (#4907) commit 5e8de4c1da6eb4eb8f738b20962595c7536b30ec Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 12:45:04 2023 -0800 Enhancement: shared icon & shared by me filter (#4859) commit 088bad90306025d3f6b139cbd0ad264a1cbecfe5 Author: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue Dec 19 12:04:03 2023 -0800 Bulk updates all the backend libraries (#5061) * Saving some work on frontend config * Very basic but dynamically-generated config form * Saving work on slightly less ugly frontend config * JSON validation for user_args field * Fully dynamic config form * Adds in some additional validators for a nicer error message * Cleaning up the testing and coverage more * Reverts unintentional change * Adds documentation about the settings and the precedence * Couple more commenting and style fixes --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- docs/configuration.md | 5 + src-ui/src/app/app-routing.module.ts | 12 + src-ui/src/app/app.module.ts | 4 + .../admin/config/config.component.html | 54 ++++ .../admin/config/config.component.scss | 0 .../admin/config/config.component.spec.ts | 103 ++++++++ .../admin/config/config.component.ts | 163 ++++++++++++ .../app-frame/app-frame.component.html | 9 + .../common/input/number/number.component.html | 4 +- .../common/input/switch/switch.component.html | 27 ++ .../common/input/switch/switch.component.scss | 0 .../input/switch/switch.component.spec.ts | 39 +++ .../common/input/switch/switch.component.ts | 21 ++ .../common/input/text/text.component.html | 4 +- src-ui/src/app/data/paperless-config.ts | 183 ++++++++++++++ .../src/app/services/config.service.spec.ts | 42 ++++ src-ui/src/app/services/config.service.ts | 27 ++ src/documents/classifier.py | 4 +- src/documents/consumer.py | 2 +- .../management/commands/document_consumer.py | 2 +- .../management/commands/document_exporter.py | 5 + .../management/commands/loaddata_stdin.py | 2 +- src/documents/parsers.py | 11 +- src/documents/tests/test_consumer.py | 16 +- .../tests/test_management_exporter.py | 16 +- src/paperless/config.py | 88 +++++++ src/paperless/migrations/0001_initial.py | 180 ++++++++++++++ src/paperless/migrations/__init__.py | 0 src/paperless/models.py | 173 +++++++++++++ src/paperless/serialisers.py | 8 + src/paperless/settings.py | 70 +++--- src/paperless/urls.py | 2 + src/paperless/views.py | 13 +- src/paperless_mail/mail.py | 14 +- src/paperless_mail/parsers.py | 6 + src/paperless_tesseract/parsers.py | 90 ++++--- src/paperless_tesseract/tests/test_parser.py | 32 +-- .../tests/test_parser_custom_settings.py | 232 ++++++++++++++++++ src/paperless_text/parsers.py | 6 + src/paperless_tika/parsers.py | 19 +- src/setup.cfg | 1 + 41 files changed, 1570 insertions(+), 119 deletions(-) create mode 100644 src-ui/src/app/components/admin/config/config.component.html create mode 100644 src-ui/src/app/components/admin/config/config.component.scss create mode 100644 src-ui/src/app/components/admin/config/config.component.spec.ts create mode 100644 src-ui/src/app/components/admin/config/config.component.ts create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.html create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.scss create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.spec.ts create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.ts create mode 100644 src-ui/src/app/data/paperless-config.ts create mode 100644 src-ui/src/app/services/config.service.spec.ts create mode 100644 src-ui/src/app/services/config.service.ts create mode 100644 src/paperless/config.py create mode 100644 src/paperless/migrations/0001_initial.py create mode 100644 src/paperless/migrations/__init__.py create mode 100644 src/paperless/models.py create mode 100644 src/paperless_tesseract/tests/test_parser_custom_settings.py diff --git a/docs/configuration.md b/docs/configuration.md index 212508806..a7cf67bf9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -3,6 +3,11 @@ Paperless provides a wide range of customizations. Depending on how you run paperless, these settings have to be defined in different places. +Certain configuration options may be set via the UI. This currently includes +common [OCR](#ocr) related settings. If set, these will take preference over the +settings via environment variables. If not set, the environment setting or applicable +default will be utilized instead. + - If you run paperless on docker, `paperless.conf` is not used. Rather, configure paperless by copying necessary options to `docker-compose.env`. diff --git a/src-ui/src/app/app-routing.module.ts b/src-ui/src/app/app-routing.module.ts index b3952634c..89ed06e39 100644 --- a/src-ui/src/app/app-routing.module.ts +++ b/src-ui/src/app/app-routing.module.ts @@ -25,6 +25,7 @@ import { ConsumptionTemplatesComponent } from './components/manage/consumption-t import { MailComponent } from './components/manage/mail/mail.component' import { UsersAndGroupsComponent } from './components/admin/users-groups/users-groups.component' import { CustomFieldsComponent } from './components/manage/custom-fields/custom-fields.component' +import { ConfigComponent } from './components/admin/config/config.component' export const routes: Routes = [ { path: '', redirectTo: 'dashboard', pathMatch: 'full' }, @@ -179,6 +180,17 @@ export const routes: Routes = [ }, }, }, + { + path: 'config', + component: ConfigComponent, + canActivate: [PermissionsGuard], + data: { + requiredPermission: { + action: PermissionAction.View, + type: PermissionType.Admin, + }, + }, + }, { path: 'tasks', component: TasksComponent, diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index c3b98549a..6d8d58944 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -108,6 +108,8 @@ import { ProfileEditDialogComponent } from './components/common/profile-edit-dia import { PdfViewerComponent } from './components/common/pdf-viewer/pdf-viewer.component' import { DocumentLinkComponent } from './components/common/input/document-link/document-link.component' import { PreviewPopupComponent } from './components/common/preview-popup/preview-popup.component' +import { ConfigComponent } from './components/admin/config/config.component' +import { SwitchComponent } from './components/common/input/switch/switch.component' import localeAf from '@angular/common/locales/af' import localeAr from '@angular/common/locales/ar' @@ -263,6 +265,8 @@ function initializeApp(settings: SettingsService) { PdfViewerComponent, DocumentLinkComponent, PreviewPopupComponent, + ConfigComponent, + SwitchComponent, ], imports: [ BrowserModule, diff --git a/src-ui/src/app/components/admin/config/config.component.html b/src-ui/src/app/components/admin/config/config.component.html new file mode 100644 index 000000000..48cac6bfa --- /dev/null +++ b/src-ui/src/app/components/admin/config/config.component.html @@ -0,0 +1,54 @@ + + +
+ + +
+ +
diff --git a/src-ui/src/app/components/admin/config/config.component.scss b/src-ui/src/app/components/admin/config/config.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/admin/config/config.component.spec.ts b/src-ui/src/app/components/admin/config/config.component.spec.ts new file mode 100644 index 000000000..5d70881b6 --- /dev/null +++ b/src-ui/src/app/components/admin/config/config.component.spec.ts @@ -0,0 +1,103 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' + +import { ConfigComponent } from './config.component' +import { ConfigService } from 'src/app/services/config.service' +import { ToastService } from 'src/app/services/toast.service' +import { of, throwError } from 'rxjs' +import { OutputTypeConfig } from 'src/app/data/paperless-config' +import { HttpClientTestingModule } from '@angular/common/http/testing' +import { BrowserModule } from '@angular/platform-browser' +import { NgbModule } from '@ng-bootstrap/ng-bootstrap' +import { NgSelectModule } from '@ng-select/ng-select' +import { TextComponent } from '../../common/input/text/text.component' +import { NumberComponent } from '../../common/input/number/number.component' +import { SwitchComponent } from '../../common/input/switch/switch.component' +import { FormsModule, ReactiveFormsModule } from '@angular/forms' +import { PageHeaderComponent } from '../../common/page-header/page-header.component' +import { SelectComponent } from '../../common/input/select/select.component' + +describe('ConfigComponent', () => { + let component: ConfigComponent + let fixture: ComponentFixture + let configService: ConfigService + let toastService: ToastService + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ + ConfigComponent, + TextComponent, + SelectComponent, + NumberComponent, + SwitchComponent, + PageHeaderComponent, + ], + imports: [ + HttpClientTestingModule, + BrowserModule, + NgbModule, + NgSelectModule, + FormsModule, + ReactiveFormsModule, + ], + }).compileComponents() + + configService = TestBed.inject(ConfigService) + toastService = TestBed.inject(ToastService) + fixture = TestBed.createComponent(ConfigComponent) + component = fixture.componentInstance + fixture.detectChanges() + }) + + it('should load config on init, show error if necessary', () => { + const getSpy = jest.spyOn(configService, 'getConfig') + const errorSpy = jest.spyOn(toastService, 'showError') + getSpy.mockReturnValueOnce( + throwError(() => new Error('Error getting config')) + ) + component.ngOnInit() + expect(getSpy).toHaveBeenCalled() + expect(errorSpy).toHaveBeenCalled() + getSpy.mockReturnValueOnce( + of({ output_type: OutputTypeConfig.PDF_A } as any) + ) + component.ngOnInit() + expect(component.initialConfig).toEqual({ + output_type: OutputTypeConfig.PDF_A, + }) + }) + + it('should save config, show error if necessary', () => { + const saveSpy = jest.spyOn(configService, 'saveConfig') + const errorSpy = jest.spyOn(toastService, 'showError') + saveSpy.mockReturnValueOnce( + throwError(() => new Error('Error saving config')) + ) + component.saveConfig() + expect(saveSpy).toHaveBeenCalled() + expect(errorSpy).toHaveBeenCalled() + saveSpy.mockReturnValueOnce( + of({ output_type: OutputTypeConfig.PDF_A } as any) + ) + component.saveConfig() + expect(component.initialConfig).toEqual({ + output_type: OutputTypeConfig.PDF_A, + }) + }) + + it('should support discard changes', () => { + component.initialConfig = { output_type: OutputTypeConfig.PDF_A2 } as any + component.configForm.patchValue({ output_type: OutputTypeConfig.PDF_A }) + component.discardChanges() + expect(component.configForm.get('output_type').value).toEqual( + OutputTypeConfig.PDF_A2 + ) + }) + + it('should support JSON validation for e.g. user_args', () => { + component.configForm.patchValue({ user_args: '{ foo bar }' }) + expect(component.errors).toEqual({ user_args: 'Invalid JSON' }) + component.configForm.patchValue({ user_args: '{ "foo": "bar" }' }) + expect(component.errors).toEqual({ user_args: null }) + }) +}) diff --git a/src-ui/src/app/components/admin/config/config.component.ts b/src-ui/src/app/components/admin/config/config.component.ts new file mode 100644 index 000000000..66d7b537f --- /dev/null +++ b/src-ui/src/app/components/admin/config/config.component.ts @@ -0,0 +1,163 @@ +import { Component, OnDestroy, OnInit } from '@angular/core' +import { AbstractControl, FormControl, FormGroup } from '@angular/forms' +import { + BehaviorSubject, + Observable, + Subject, + Subscription, + first, + takeUntil, +} from 'rxjs' +import { + PaperlessConfigOptions, + ConfigCategory, + ConfigOption, + ConfigOptionType, + PaperlessConfig, +} from 'src/app/data/paperless-config' +import { ConfigService } from 'src/app/services/config.service' +import { ToastService } from 'src/app/services/toast.service' +import { ComponentWithPermissions } from '../../with-permissions/with-permissions.component' +import { DirtyComponent, dirtyCheck } from '@ngneat/dirty-check-forms' + +@Component({ + selector: 'pngx-config', + templateUrl: './config.component.html', + styleUrl: './config.component.scss', +}) +export class ConfigComponent + extends ComponentWithPermissions + implements OnInit, OnDestroy, DirtyComponent +{ + public readonly ConfigOptionType = ConfigOptionType + + // generated dynamically + public configForm = new FormGroup({}) + + public errors = {} + + get optionCategories(): string[] { + return Object.values(ConfigCategory) + } + + getCategoryOptions(category: string): ConfigOption[] { + return PaperlessConfigOptions.filter((o) => o.category === category) + } + + public loading: boolean = false + + initialConfig: PaperlessConfig + store: BehaviorSubject + storeSub: Subscription + isDirty$: Observable + + private unsubscribeNotifier: Subject = new Subject() + + constructor( + private configService: ConfigService, + private toastService: ToastService + ) { + super() + this.configForm.addControl('id', new FormControl()) + PaperlessConfigOptions.forEach((option) => { + this.configForm.addControl(option.key, new FormControl()) + }) + } + + ngOnInit(): void { + this.loading = true + this.configService + .getConfig() + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe({ + next: (config) => { + this.loading = false + this.initialize(config) + }, + error: (e) => { + this.loading = false + this.toastService.showError($localize`Error retrieving config`, e) + }, + }) + + // validate JSON inputs + PaperlessConfigOptions.filter( + (o) => o.type === ConfigOptionType.JSON + ).forEach((option) => { + this.configForm + .get(option.key) + .addValidators((control: AbstractControl) => { + if (!control.value || control.value.toString().length === 0) + return null + try { + JSON.parse(control.value) + } catch (e) { + return [ + { + user_args: e, + }, + ] + } + return null + }) + this.configForm.get(option.key).statusChanges.subscribe((status) => { + this.errors[option.key] = + status === 'INVALID' ? $localize`Invalid JSON` : null + }) + this.configForm.get(option.key).updateValueAndValidity() + }) + } + + ngOnDestroy(): void { + this.unsubscribeNotifier.next(true) + this.unsubscribeNotifier.complete() + } + + private initialize(config: PaperlessConfig) { + if (!this.store) { + this.store = new BehaviorSubject(config) + + this.store + .asObservable() + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe((state) => { + this.configForm.patchValue(state, { emitEvent: false }) + }) + + this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable()) + } + this.configForm.patchValue(config) + + this.initialConfig = config + } + + getDocsUrl(key: string) { + return `https://docs.paperless-ngx.com/configuration/#${key}` + } + + public saveConfig() { + this.loading = true + this.configService + .saveConfig(this.configForm.value as PaperlessConfig) + .pipe(takeUntil(this.unsubscribeNotifier), first()) + .subscribe({ + next: (config) => { + this.loading = false + this.initialize(config) + this.store.next(config) + this.toastService.showInfo($localize`Configuration updated`) + }, + error: (e) => { + this.loading = false + this.toastService.showError( + $localize`An error occurred updating configuration`, + e + ) + }, + }) + } + + public discardChanges() { + this.configForm.reset(this.initialConfig) + } +} diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 2ab3fe0ae..234099d60 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -271,6 +271,15 @@  Settings +
  • + + + +  Configuration + +
  • - + @if (title) { + + } @if (removable) { + } +
    + } +
    +
    + + @if (!horizontal) { + + } + @if (hint) { +
    {{hint}}
    + } +
    +
    +
    + diff --git a/src-ui/src/app/components/common/input/switch/switch.component.scss b/src-ui/src/app/components/common/input/switch/switch.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/common/input/switch/switch.component.spec.ts b/src-ui/src/app/components/common/input/switch/switch.component.spec.ts new file mode 100644 index 000000000..08a4598a3 --- /dev/null +++ b/src-ui/src/app/components/common/input/switch/switch.component.spec.ts @@ -0,0 +1,39 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' +import { SwitchComponent } from './switch.component' +import { + FormsModule, + NG_VALUE_ACCESSOR, + ReactiveFormsModule, +} from '@angular/forms' + +describe('SwitchComponent', () => { + let component: SwitchComponent + let fixture: ComponentFixture + let input: HTMLInputElement + + beforeEach(async () => { + TestBed.configureTestingModule({ + declarations: [SwitchComponent], + providers: [], + imports: [FormsModule, ReactiveFormsModule], + }).compileComponents() + + fixture = TestBed.createComponent(SwitchComponent) + fixture.debugElement.injector.get(NG_VALUE_ACCESSOR) + component = fixture.componentInstance + fixture.detectChanges() + input = component.inputField.nativeElement + }) + + it('should support use of checkbox', () => { + input.checked = true + input.dispatchEvent(new Event('change')) + fixture.detectChanges() + expect(component.value).toBeTruthy() + + input.checked = false + input.dispatchEvent(new Event('change')) + fixture.detectChanges() + expect(component.value).toBeFalsy() + }) +}) diff --git a/src-ui/src/app/components/common/input/switch/switch.component.ts b/src-ui/src/app/components/common/input/switch/switch.component.ts new file mode 100644 index 000000000..44e095baa --- /dev/null +++ b/src-ui/src/app/components/common/input/switch/switch.component.ts @@ -0,0 +1,21 @@ +import { Component, forwardRef } from '@angular/core' +import { NG_VALUE_ACCESSOR } from '@angular/forms' +import { AbstractInputComponent } from '../abstract-input' + +@Component({ + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SwitchComponent), + multi: true, + }, + ], + selector: 'pngx-input-switch', + templateUrl: './switch.component.html', + styleUrls: ['./switch.component.scss'], +}) +export class SwitchComponent extends AbstractInputComponent { + constructor() { + super() + } +} diff --git a/src-ui/src/app/components/common/input/text/text.component.html b/src-ui/src/app/components/common/input/text/text.component.html index 3b9f7e36d..4169d09db 100644 --- a/src-ui/src/app/components/common/input/text/text.component.html +++ b/src-ui/src/app/components/common/input/text/text.component.html @@ -1,7 +1,9 @@
    - + @if (title) { + + } @if (removable) {