Feature: Allow setting backend configuration settings via the UI (#5126)

* Saving some start on this

* At least partially working for the tesseract parser

* Problems with migration testing need to figure out

* Work around that error

* Fixes max m_pixels

* Moving the settings to main paperless application

* Starting some consumer options

* More fixes and work

* Fixes these last tests

* Fix max_length on OcrSettings.mode field

* Fix all fields on Common & Ocr settings serializers

* Umbrellla config view

* Revert "Umbrellla config view"

This reverts commit fbaf9f4be30f89afeb509099180158a3406416a5.

* Updates to use a single configuration object for all settings

* Squashed commit of the following:

commit 8a0a49dd57
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 23:02:47 2023 -0800

    Fix formatting

commit 66b2d90c50
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 22:36:35 2023 -0800

    Refactor frontend data models

commit 5723bd8dd8
Author: Adam Bogdał <adam@bogdal.pl>
Date:   Wed Dec 20 01:17:43 2023 +0100

    Fix: speed up admin panel for installs with a large number of documents (#5052)

commit 9b08ce1761
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 15:18:51 2023 -0800

    Update PULL_REQUEST_TEMPLATE.md

commit a6248bec2d
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 15:02:05 2023 -0800

    Chore: Update Angular to v17 (#4980)

commit b1f6f52486
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 13:53:56 2023 -0800

    Fix: Dont allow null custom_fields property via API (#5063)

commit 638d9970fd
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 13:43:50 2023 -0800

    Enhancement: symmetric document links (#4907)

commit 5e8de4c1da
Author: shamoon <4887959+shamoon@users.noreply.github.com>
Date:   Tue Dec 19 12:45:04 2023 -0800

    Enhancement: shared icon & shared by me filter (#4859)

commit 088bad9030
Author: Trenton H <797416+stumpylog@users.noreply.github.com>
Date:   Tue Dec 19 12:04:03 2023 -0800

    Bulk updates all the backend libraries (#5061)

* Saving some work on frontend config

* Very basic but dynamically-generated config form

* Saving work on slightly less ugly frontend config

* JSON validation for user_args field

* Fully dynamic config form

* Adds in some additional validators for a nicer error message

* Cleaning up the testing and coverage more

* Reverts unintentional change

* Adds documentation about the settings and the precedence

* Couple more commenting and style fixes

---------

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src-ui/src/app/components/admin/config/config.component.html

@@ -0,0 +1,54 @@
+<pngx-page-header title="Configuration" i18n-title></pngx-page-header>
+
+<form [formGroup]="configForm" (ngSubmit)="saveConfig()" class="pb-4">
+
+    <ul ngbNav #nav="ngbNav" class="nav-tabs">
+        @for (category of optionCategories; track category) {
+            <li [ngbNavItem]="category">
+                <a ngbNavLink i18n>{{category}}</a>
+                <ng-template ngbNavContent>
+                    <div class="p-3">
+                        <div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
+                            @for (option of getCategoryOptions(category); track option.key) {
+                                <div class="col">
+                                    <div class="card bg-light">
+                                        <div class="card-body">
+                                            <div class="card-title">
+                                                <h6>
+                                                    {{option.title}}
+                                                    <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
+                                                        <svg class="sidebaricon" fill="currentColor">
+                                                            <use xlink:href="assets/bootstrap-icons.svg#info-circle"/>
+                                                        </svg>
+                                                    </a>
+                                                </h6>
+                                            </div>
+                                            <div class="mb-n3">
+                                                @switch (option.type) {
+                                                    @case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
+                                                    @case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
+                                                    @case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" title="Enable" i18n-title></pngx-input-switch> }
+                                                    @case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
+                                                    @case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
+                                                }
+                                            </div>
+                                        </div>
+                                    </div>
+                                </div>
+                            }
+                        </div>
+                    </div>
+                </ng-template>
+            </li>
+        }
+    </ul>
+    <div [ngbNavOutlet]="nav" class="border-start border-end border-bottom p-3 mb-3 shadow-sm"></div>
+    <div class="btn-toolbar" role="toolbar">
+        <div class="btn-group me-2">
+            <button type="button" (click)="discardChanges()" class="btn btn-secondary" [disabled]="loading || (isDirty$ | async) === false" i18n>Discard</button>
+        </div>
+        <div class="btn-group">
+            <button type="submit" class="btn btn-primary" [disabled]="loading || !configForm.valid || (isDirty$ | async) === false" i18n>Save</button>
+        </div>
+    </div>
+</form>

src-ui/src/app/components/admin/config/config.component.spec.ts

@@ -0,0 +1,103 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing'
+
+import { ConfigComponent } from './config.component'
+import { ConfigService } from 'src/app/services/config.service'
+import { ToastService } from 'src/app/services/toast.service'
+import { of, throwError } from 'rxjs'
+import { OutputTypeConfig } from 'src/app/data/paperless-config'
+import { HttpClientTestingModule } from '@angular/common/http/testing'
+import { BrowserModule } from '@angular/platform-browser'
+import { NgbModule } from '@ng-bootstrap/ng-bootstrap'
+import { NgSelectModule } from '@ng-select/ng-select'
+import { TextComponent } from '../../common/input/text/text.component'
+import { NumberComponent } from '../../common/input/number/number.component'
+import { SwitchComponent } from '../../common/input/switch/switch.component'
+import { FormsModule, ReactiveFormsModule } from '@angular/forms'
+import { PageHeaderComponent } from '../../common/page-header/page-header.component'
+import { SelectComponent } from '../../common/input/select/select.component'
+
+describe('ConfigComponent', () => {
+  let component: ConfigComponent
+  let fixture: ComponentFixture<ConfigComponent>
+  let configService: ConfigService
+  let toastService: ToastService
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [
+        ConfigComponent,
+        TextComponent,
+        SelectComponent,
+        NumberComponent,
+        SwitchComponent,
+        PageHeaderComponent,
+      ],
+      imports: [
+        HttpClientTestingModule,
+        BrowserModule,
+        NgbModule,
+        NgSelectModule,
+        FormsModule,
+        ReactiveFormsModule,
+      ],
+    }).compileComponents()
+
+    configService = TestBed.inject(ConfigService)
+    toastService = TestBed.inject(ToastService)
+    fixture = TestBed.createComponent(ConfigComponent)
+    component = fixture.componentInstance
+    fixture.detectChanges()
+  })
+
+  it('should load config on init, show error if necessary', () => {
+    const getSpy = jest.spyOn(configService, 'getConfig')
+    const errorSpy = jest.spyOn(toastService, 'showError')
+    getSpy.mockReturnValueOnce(
+      throwError(() => new Error('Error getting config'))
+    )
+    component.ngOnInit()
+    expect(getSpy).toHaveBeenCalled()
+    expect(errorSpy).toHaveBeenCalled()
+    getSpy.mockReturnValueOnce(
+      of({ output_type: OutputTypeConfig.PDF_A } as any)
+    )
+    component.ngOnInit()
+    expect(component.initialConfig).toEqual({
+      output_type: OutputTypeConfig.PDF_A,
+    })
+  })
+
+  it('should save config, show error if necessary', () => {
+    const saveSpy = jest.spyOn(configService, 'saveConfig')
+    const errorSpy = jest.spyOn(toastService, 'showError')
+    saveSpy.mockReturnValueOnce(
+      throwError(() => new Error('Error saving config'))
+    )
+    component.saveConfig()
+    expect(saveSpy).toHaveBeenCalled()
+    expect(errorSpy).toHaveBeenCalled()
+    saveSpy.mockReturnValueOnce(
+      of({ output_type: OutputTypeConfig.PDF_A } as any)
+    )
+    component.saveConfig()
+    expect(component.initialConfig).toEqual({
+      output_type: OutputTypeConfig.PDF_A,
+    })
+  })
+
+  it('should support discard changes', () => {
+    component.initialConfig = { output_type: OutputTypeConfig.PDF_A2 } as any
+    component.configForm.patchValue({ output_type: OutputTypeConfig.PDF_A })
+    component.discardChanges()
+    expect(component.configForm.get('output_type').value).toEqual(
+      OutputTypeConfig.PDF_A2
+    )
+  })
+
+  it('should support JSON validation for e.g. user_args', () => {
+    component.configForm.patchValue({ user_args: '{ foo bar }' })
+    expect(component.errors).toEqual({ user_args: 'Invalid JSON' })
+    component.configForm.patchValue({ user_args: '{ "foo": "bar" }' })
+    expect(component.errors).toEqual({ user_args: null })
+  })
+})

src-ui/src/app/components/admin/config/config.component.ts

@@ -0,0 +1,163 @@
+import { Component, OnDestroy, OnInit } from '@angular/core'
+import { AbstractControl, FormControl, FormGroup } from '@angular/forms'
+import {
+  BehaviorSubject,
+  Observable,
+  Subject,
+  Subscription,
+  first,
+  takeUntil,
+} from 'rxjs'
+import {
+  PaperlessConfigOptions,
+  ConfigCategory,
+  ConfigOption,
+  ConfigOptionType,
+  PaperlessConfig,
+} from 'src/app/data/paperless-config'
+import { ConfigService } from 'src/app/services/config.service'
+import { ToastService } from 'src/app/services/toast.service'
+import { ComponentWithPermissions } from '../../with-permissions/with-permissions.component'
+import { DirtyComponent, dirtyCheck } from '@ngneat/dirty-check-forms'
+
+@Component({
+  selector: 'pngx-config',
+  templateUrl: './config.component.html',
+  styleUrl: './config.component.scss',
+})
+export class ConfigComponent
+  extends ComponentWithPermissions
+  implements OnInit, OnDestroy, DirtyComponent
+{
+  public readonly ConfigOptionType = ConfigOptionType
+
+  // generated dynamically
+  public configForm = new FormGroup({})
+
+  public errors = {}
+
+  get optionCategories(): string[] {
+    return Object.values(ConfigCategory)
+  }
+
+  getCategoryOptions(category: string): ConfigOption[] {
+    return PaperlessConfigOptions.filter((o) => o.category === category)
+  }
+
+  public loading: boolean = false
+
+  initialConfig: PaperlessConfig
+  store: BehaviorSubject<any>
+  storeSub: Subscription
+  isDirty$: Observable<boolean>
+
+  private unsubscribeNotifier: Subject<any> = new Subject()
+
+  constructor(
+    private configService: ConfigService,
+    private toastService: ToastService
+  ) {
+    super()
+    this.configForm.addControl('id', new FormControl())
+    PaperlessConfigOptions.forEach((option) => {
+      this.configForm.addControl(option.key, new FormControl())
+    })
+  }
+
+  ngOnInit(): void {
+    this.loading = true
+    this.configService
+      .getConfig()
+      .pipe(takeUntil(this.unsubscribeNotifier))
+      .subscribe({
+        next: (config) => {
+          this.loading = false
+          this.initialize(config)
+        },
+        error: (e) => {
+          this.loading = false
+          this.toastService.showError($localize`Error retrieving config`, e)
+        },
+      })
+
+    // validate JSON inputs
+    PaperlessConfigOptions.filter(
+      (o) => o.type === ConfigOptionType.JSON
+    ).forEach((option) => {
+      this.configForm
+        .get(option.key)
+        .addValidators((control: AbstractControl) => {
+          if (!control.value || control.value.toString().length === 0)
+            return null
+          try {
+            JSON.parse(control.value)
+          } catch (e) {
+            return [
+              {
+                user_args: e,
+              },
+            ]
+          }
+          return null
+        })
+      this.configForm.get(option.key).statusChanges.subscribe((status) => {
+        this.errors[option.key] =
+          status === 'INVALID' ? $localize`Invalid JSON` : null
+      })
+      this.configForm.get(option.key).updateValueAndValidity()
+    })
+  }
+
+  ngOnDestroy(): void {
+    this.unsubscribeNotifier.next(true)
+    this.unsubscribeNotifier.complete()
+  }
+
+  private initialize(config: PaperlessConfig) {
+    if (!this.store) {
+      this.store = new BehaviorSubject(config)
+
+      this.store
+        .asObservable()
+        .pipe(takeUntil(this.unsubscribeNotifier))
+        .subscribe((state) => {
+          this.configForm.patchValue(state, { emitEvent: false })
+        })
+
+      this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable())
+    }
+    this.configForm.patchValue(config)
+
+    this.initialConfig = config
+  }
+
+  getDocsUrl(key: string) {
+    return `https://docs.paperless-ngx.com/configuration/#${key}`
+  }
+
+  public saveConfig() {
+    this.loading = true
+    this.configService
+      .saveConfig(this.configForm.value as PaperlessConfig)
+      .pipe(takeUntil(this.unsubscribeNotifier), first())
+      .subscribe({
+        next: (config) => {
+          this.loading = false
+          this.initialize(config)
+          this.store.next(config)
+          this.toastService.showInfo($localize`Configuration updated`)
+        },
+        error: (e) => {
+          this.loading = false
+          this.toastService.showError(
+            $localize`An error occurred updating configuration`,
+            e
+          )
+        },
+      })
+  }
+
+  public discardChanges() {
+    this.configForm.reset(this.initialConfig)
+  }
+}

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -271,6 +271,15 @@
               </svg><span>&nbsp;<ng-container i18n>Settings</ng-container></span>
             </a>
           </li>
+          <li class="nav-item" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Admin }">
+            <a class="nav-link" routerLink="config" routerLinkActive="active" (click)="closeMenu()"
+              ngbPopover="Configuration" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end"
+              container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
+              <svg class="sidebaricon" fill="currentColor">
+                <use xlink:href="assets/bootstrap-icons.svg#sliders2-vertical" />
+              </svg><span>&nbsp;<ng-container i18n>Configuration</ng-container></span>
+            </a>
+          </li>
           <li class="nav-item" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.User }">
             <a class="nav-link" routerLink="usersgroups" routerLinkActive="active" (click)="closeMenu()"
               ngbPopover="Users & Groups" i18n-ngbPopover [disablePopover]="!slimSidebarEnabled" placement="end"

src-ui/src/app/components/common/input/number/number.component.html

@@ -1,7 +1,9 @@
 <div class="mb-3" [class.pb-3]="error">
   <div class="row">
     <div class="d-flex align-items-center position-relative hidden-button-container" [class.col-md-3]="horizontal">
-      <label class="form-label" [class.mb-md-0]="horizontal" [for]="inputId">{{title}}</label>
+      @if (title) {
+        <label class="form-label" [class.mb-md-0]="horizontal" [for]="inputId">{{title}}</label>
+      }
       @if (removable) {
         <button type="button" class="btn btn-sm btn-danger position-absolute left-0" (click)="removed.emit(this)">
           <svg class="sidebaricon" fill="currentColor">

src-ui/src/app/components/common/input/switch/switch.component.html

@@ -0,0 +1,27 @@
+<div class="mb-3">
+  <div class="row">
+    @if (horizontal) {
+      <div class="d-flex align-items-center position-relative hidden-button-container col-md-3">
+        <label class="form-label" [class.mb-md-0]="horizontal" [for]="inputId">{{title}}</label>
+        @if (removable) {
+          <button type="button" class="btn btn-sm btn-danger position-absolute left-0" (click)="removed.emit(this)">
+            <svg class="sidebaricon" fill="currentColor">
+              <use xlink:href="assets/bootstrap-icons.svg#x"/>
+              </svg>&nbsp;<ng-container i18n>Remove</ng-container>
+            </button>
+          }
+        </div>
+      }
+      <div [ngClass]="{'col-md-9': horizontal, 'align-items-center': horizontal, 'd-flex': horizontal}">
+        <div class="form-check form-switch">
+          <input #inputField type="checkbox" class="form-check-input" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)" (blur)="onTouched()" [disabled]="disabled">
+          @if (!horizontal) {
+            <label class="form-check-label" [for]="inputId">{{title}}</label>
+          }
+          @if (hint) {
+            <div class="form-text text-muted">{{hint}}</div>
+          }
+        </div>
+      </div>
+    </div>
+  </div>

src-ui/src/app/components/common/input/switch/switch.component.spec.ts

@@ -0,0 +1,39 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing'
+import { SwitchComponent } from './switch.component'
+import {
+  FormsModule,
+  NG_VALUE_ACCESSOR,
+  ReactiveFormsModule,
+} from '@angular/forms'
+
+describe('SwitchComponent', () => {
+  let component: SwitchComponent
+  let fixture: ComponentFixture<SwitchComponent>
+  let input: HTMLInputElement
+
+  beforeEach(async () => {
+    TestBed.configureTestingModule({
+      declarations: [SwitchComponent],
+      providers: [],
+      imports: [FormsModule, ReactiveFormsModule],
+    }).compileComponents()
+
+    fixture = TestBed.createComponent(SwitchComponent)
+    fixture.debugElement.injector.get(NG_VALUE_ACCESSOR)
+    component = fixture.componentInstance
+    fixture.detectChanges()
+    input = component.inputField.nativeElement
+  })
+
+  it('should support use of checkbox', () => {
+    input.checked = true
+    input.dispatchEvent(new Event('change'))
+    fixture.detectChanges()
+    expect(component.value).toBeTruthy()
+
+    input.checked = false
+    input.dispatchEvent(new Event('change'))
+    fixture.detectChanges()
+    expect(component.value).toBeFalsy()
+  })
+})

src-ui/src/app/components/common/input/switch/switch.component.ts

@@ -0,0 +1,21 @@
+import { Component, forwardRef } from '@angular/core'
+import { NG_VALUE_ACCESSOR } from '@angular/forms'
+import { AbstractInputComponent } from '../abstract-input'
+
+@Component({
+  providers: [
+    {
+      provide: NG_VALUE_ACCESSOR,
+      useExisting: forwardRef(() => SwitchComponent),
+      multi: true,
+    },
+  ],
+  selector: 'pngx-input-switch',
+  templateUrl: './switch.component.html',
+  styleUrls: ['./switch.component.scss'],
+})
+export class SwitchComponent extends AbstractInputComponent<boolean> {
+  constructor() {
+    super()
+  }
+}

src-ui/src/app/components/common/input/text/text.component.html

@@ -1,7 +1,9 @@
 <div class="mb-3" [class.pb-3]="error">
   <div class="row">
     <div class="d-flex align-items-center position-relative hidden-button-container" [class.col-md-3]="horizontal">
-      <label class="form-label" [class.mb-md-0]="horizontal" [for]="inputId">{{title}}</label>
+      @if (title) {
+        <label class="form-label" [class.mb-md-0]="horizontal" [for]="inputId">{{title}}</label>
+      }
       @if (removable) {
         <button type="button" class="btn btn-sm btn-danger position-absolute left-0" (click)="removed.emit(this)">
           <svg class="sidebaricon" fill="currentColor">