Feature: consumption templates (#4196)

* Initial implementation of consumption templates

* Frontend implementation of consumption templates

Testing

* Support consumption template source

* order templates, automatically add permissions

* Support title assignment in consumption templates

* Refactoring, filters to and, show sources on list

Show sources on template list, update some translation strings

Make filters and

minor testing

* Update strings

* Only update django-multiselectfield

* Basic docs, document some methods

* Improve testing coverage, template multi-assignment merges

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.html

@@ -0,0 +1,92 @@
+<form [formGroup]="objectForm" (ngSubmit)="save()">
+    <div class="modal-header">
+      <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
+      <button type="button" [disabled]="!closeEnabled" class="btn-close" aria-label="Close" (click)="cancel()">
+      </button>
+    </div>
+    <div class="modal-body">
+      <div class="row">
+        <div class="col-md-8">
+          <pngx-input-text i18n-title title="Name" formControlName="name" [error]="error?.name"></pngx-input-text>
+        </div>
+        <div class="col">
+          <pngx-input-number i18n-title title="Sort order" formControlName="order" [showAdd]="false" [error]="error?.order"></pngx-input-number>
+        </div>
+      </div>
+      <div class="row">
+        <div class="col-md-4">
+          <h5 class="border-bottom pb-2" i18n>Filters</h5>
+          <p class="small" i18n>Process documents that match <em>all</em> filters specified below.</p>
+          <pngx-input-select i18n-title title="Filter sources" [items]="sourceOptions" [multiple]="true" formControlName="sources" [error]="error?.filter_filename"></pngx-input-select>
+          <pngx-input-text i18n-title title="Filter filename" formControlName="filter_filename" i18n-hint hint="Apply to documents that match this filename. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_filename"></pngx-input-text>
+          <pngx-input-text i18n-title title="Filter path" formControlName="filter_path" i18n-hint hint="Apply to documents that match this path. Wildcards specified as * are allowed. Case insensitive.</a>" [error]="error?.filter_path"></pngx-input-text>
+          <pngx-input-select i18n-title title="Filter mail rule" [items]="mailRules" [allowNull]="true" formControlName="filter_mailrule" i18n-hint hint="Apply to documents consumed via this mail rule." [error]="error?.filter_mailrule"></pngx-input-select>
+        </div>
+        <div class="col">
+          <div class="row">
+            <div class="col">
+              <h5 class="border-bottom pb-2" i18n>Assignments</h5>
+            </div>
+          </div>
+          <div class="row">
+            <div class="col">
+              <pngx-input-text i18n-title title="Assign title" formControlName="assign_title" i18n-hint hint="Can include some placeholders, see <a target='_blank' href='https://docs.paperless-ngx.com/usage/#consumption-templates'>documentation</a>." [error]="error?.assign_title"></pngx-input-text>
+              <pngx-input-tags [allowCreate]="false" i18n-title title="Assign tags" formControlName="assign_tags"></pngx-input-tags>
+              <pngx-input-select i18n-title title="Assign document type" [items]="documentTypes" [allowNull]="true" formControlName="assign_document_type"></pngx-input-select>
+              <pngx-input-select i18n-title title="Assign correspondent" [items]="correspondents" [allowNull]="true" formControlName="assign_correspondent"></pngx-input-select>
+              <pngx-input-select i18n-title title="Assign storage path" [items]="storagePaths" [allowNull]="true" formControlName="assign_storage_path"></pngx-input-select>
+          </div>
+          <div class="col">
+            <pngx-input-select i18n-title title="Assign owner" [items]="users" bindLabel="username" formControlName="assign_owner" [allowNull]="true"></pngx-input-select>
+              <div>
+                <label class="form-label" i18n>Assign view permissions</label>
+                <div class="mb-2">
+                  <div class="row mb-1">
+                    <div class="col-lg-3">
+                      <label class="form-label d-block my-2" i18n>Users:</label>
+                    </div>
+                    <div class="col-lg-9">
+                      <pngx-permissions-user type="view" formControlName="assign_view_users"></pngx-permissions-user>
+                    </div>
+                  </div>
+                  <div class="row">
+                    <div class="col-lg-3">
+                      <label class="form-label d-block my-2" i18n>Groups:</label>
+                    </div>
+                    <div class="col-lg-9">
+                      <pngx-permissions-group type="view" formControlName="assign_view_groups"></pngx-permissions-group>
+                    </div>
+                  </div>
+                </div>
+                <label class="form-label" i18n>Assign edit permissions</label>
+                <div>
+                  <div class="row mb-1">
+                    <div class="col-lg-3">
+                      <label class="form-label d-block my-2" i18n>Users:</label>
+                    </div>
+                    <div class="col-lg-9">
+                      <pngx-permissions-user type="change" formControlName="assign_change_users"></pngx-permissions-user>
+                    </div>
+                  </div>
+                  <div class="row">
+                    <div class="col-lg-3">
+                      <label class="form-label d-block my-2" i18n>Groups:</label>
+                    </div>
+                    <div class="col-lg-9">
+                      <pngx-permissions-group type="change" formControlName="assign_change_groups"></pngx-permissions-group>
+                    </div>
+                  </div>
+                  <small class="form-text text-muted text-end d-block" i18n>Edit permissions also grant viewing permissions</small>
+                </div>
+              </div>
+          </div>
+        </div>
+      </div>
+      </div>
+    </div>
+    <div class="modal-footer">
+      <span class="text-danger" *ngIf="error?.non_field_errors"><ng-container i18n>Error</ng-container>: {{error.non_field_errors}}</span>
+      <button type="button" class="btn btn-outline-secondary" (click)="cancel()" i18n [disabled]="networkActive">Cancel</button>
+      <button type="submit" class="btn btn-primary" i18n [disabled]="networkActive">Save</button>
+    </div>
+  </form>

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.spec.ts

@@ -0,0 +1,125 @@
+import { HttpClientTestingModule } from '@angular/common/http/testing'
+import { ComponentFixture, TestBed } from '@angular/core/testing'
+import { FormsModule, ReactiveFormsModule } from '@angular/forms'
+import { NgbActiveModal, NgbModule } from '@ng-bootstrap/ng-bootstrap'
+import { NgSelectModule } from '@ng-select/ng-select'
+import { of } from 'rxjs'
+import { IfOwnerDirective } from 'src/app/directives/if-owner.directive'
+import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
+import { SafeHtmlPipe } from 'src/app/pipes/safehtml.pipe'
+import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
+import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
+import { MailRuleService } from 'src/app/services/rest/mail-rule.service'
+import { StoragePathService } from 'src/app/services/rest/storage-path.service'
+import { SettingsService } from 'src/app/services/settings.service'
+import { NumberComponent } from '../../input/number/number.component'
+import { PermissionsGroupComponent } from '../../input/permissions/permissions-group/permissions-group.component'
+import { PermissionsUserComponent } from '../../input/permissions/permissions-user/permissions-user.component'
+import { SelectComponent } from '../../input/select/select.component'
+import { TagsComponent } from '../../input/tags/tags.component'
+import { TextComponent } from '../../input/text/text.component'
+import { EditDialogMode } from '../edit-dialog.component'
+import { ConsumptionTemplateEditDialogComponent } from './consumption-template-edit-dialog.component'
+
+describe('ConsumptionTemplateEditDialogComponent', () => {
+  let component: ConsumptionTemplateEditDialogComponent
+  let settingsService: SettingsService
+  let fixture: ComponentFixture<ConsumptionTemplateEditDialogComponent>
+
+  beforeEach(() => {
+    TestBed.configureTestingModule({
+      declarations: [
+        ConsumptionTemplateEditDialogComponent,
+        IfPermissionsDirective,
+        IfOwnerDirective,
+        SelectComponent,
+        TextComponent,
+        NumberComponent,
+        TagsComponent,
+        PermissionsUserComponent,
+        PermissionsGroupComponent,
+        SafeHtmlPipe,
+      ],
+      providers: [
+        NgbActiveModal,
+        {
+          provide: CorrespondentService,
+          useValue: {
+            listAll: () =>
+              of({
+                results: [
+                  {
+                    id: 1,
+                    username: 'c1',
+                  },
+                ],
+              }),
+          },
+        },
+        {
+          provide: DocumentTypeService,
+          useValue: {
+            listAll: () =>
+              of({
+                results: [
+                  {
+                    id: 1,
+                    username: 'dt1',
+                  },
+                ],
+              }),
+          },
+        },
+        {
+          provide: StoragePathService,
+          useValue: {
+            listAll: () =>
+              of({
+                results: [
+                  {
+                    id: 1,
+                    username: 'sp1',
+                  },
+                ],
+              }),
+          },
+        },
+        {
+          provide: MailRuleService,
+          useValue: {
+            listAll: () =>
+              of({
+                results: [],
+              }),
+          },
+        },
+      ],
+      imports: [
+        HttpClientTestingModule,
+        FormsModule,
+        ReactiveFormsModule,
+        NgSelectModule,
+        NgbModule,
+      ],
+    }).compileComponents()
+
+    fixture = TestBed.createComponent(ConsumptionTemplateEditDialogComponent)
+    settingsService = TestBed.inject(SettingsService)
+    settingsService.currentUser = { id: 99, username: 'user99' }
+    component = fixture.componentInstance
+
+    fixture.detectChanges()
+  })
+
+  it('should support create and edit modes', () => {
+    component.dialogMode = EditDialogMode.CREATE
+    const createTitleSpy = jest.spyOn(component, 'getCreateTitle')
+    const editTitleSpy = jest.spyOn(component, 'getEditTitle')
+    fixture.detectChanges()
+    expect(createTitleSpy).toHaveBeenCalled()
+    expect(editTitleSpy).not.toHaveBeenCalled()
+    component.dialogMode = EditDialogMode.EDIT
+    fixture.detectChanges()
+    expect(editTitleSpy).toHaveBeenCalled()
+  })
+})

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.ts

@@ -0,0 +1,115 @@
+import { Component } from '@angular/core'
+import { FormGroup, FormControl } from '@angular/forms'
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
+import { first } from 'rxjs'
+import {
+  DocumentSource,
+  PaperlessConsumptionTemplate,
+} from 'src/app/data/paperless-consumption-template'
+import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
+import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
+import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
+import { ConsumptionTemplateService } from 'src/app/services/rest/consumption-template.service'
+import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
+import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
+import { StoragePathService } from 'src/app/services/rest/storage-path.service'
+import { UserService } from 'src/app/services/rest/user.service'
+import { SettingsService } from 'src/app/services/settings.service'
+import { EditDialogComponent } from '../edit-dialog.component'
+import { MailRuleService } from 'src/app/services/rest/mail-rule.service'
+import { PaperlessMailRule } from 'src/app/data/paperless-mail-rule'
+
+export const DOCUMENT_SOURCE_OPTIONS = [
+  {
+    id: DocumentSource.ConsumeFolder,
+    name: $localize`Consume Folder`,
+  },
+  {
+    id: DocumentSource.ApiUpload,
+    name: $localize`API Upload`,
+  },
+  {
+    id: DocumentSource.MailFetch,
+    name: $localize`Mail Fetch`,
+  },
+]
+
+@Component({
+  selector: 'pngx-consumption-template-edit-dialog',
+  templateUrl: './consumption-template-edit-dialog.component.html',
+  styleUrls: ['./consumption-template-edit-dialog.component.scss'],
+})
+export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<PaperlessConsumptionTemplate> {
+  templates: PaperlessConsumptionTemplate[]
+  correspondents: PaperlessCorrespondent[]
+  documentTypes: PaperlessDocumentType[]
+  storagePaths: PaperlessStoragePath[]
+  mailRules: PaperlessMailRule[]
+
+  constructor(
+    service: ConsumptionTemplateService,
+    activeModal: NgbActiveModal,
+    correspondentService: CorrespondentService,
+    documentTypeService: DocumentTypeService,
+    storagePathService: StoragePathService,
+    mailRuleService: MailRuleService,
+    userService: UserService,
+    settingsService: SettingsService
+  ) {
+    super(service, activeModal, userService, settingsService)
+
+    correspondentService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.correspondents = result.results))
+
+    documentTypeService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.documentTypes = result.results))
+
+    storagePathService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.storagePaths = result.results))
+
+    mailRuleService
+      .listAll()
+      .pipe(first())
+      .subscribe((result) => (this.mailRules = result.results))
+  }
+
+  getCreateTitle() {
+    return $localize`Create new consumption template`
+  }
+
+  getEditTitle() {
+    return $localize`Edit consumption template`
+  }
+
+  getForm(): FormGroup {
+    return new FormGroup({
+      name: new FormControl(null),
+      account: new FormControl(null),
+      filter_filename: new FormControl(null),
+      filter_path: new FormControl(null),
+      filter_mailrule: new FormControl(null),
+      order: new FormControl(null),
+      sources: new FormControl([]),
+      assign_title: new FormControl(null),
+      assign_tags: new FormControl([]),
+      assign_owner: new FormControl(null),
+      assign_document_type: new FormControl(null),
+      assign_correspondent: new FormControl(null),
+      assign_storage_path: new FormControl(null),
+      assign_view_users: new FormControl([]),
+      assign_view_groups: new FormControl([]),
+      assign_change_users: new FormControl([]),
+      assign_change_groups: new FormControl([]),
+    })
+  }
+
+  get sourceOptions() {
+    return DOCUMENT_SOURCE_OPTIONS
+  }
+}

src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html

@@ -26,11 +26,13 @@
       <div class="col-md-4">
         <pngx-input-select i18n-title title="Action" [items]="actionOptions" formControlName="action" i18n-hint hint="Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched."></pngx-input-select>
         <pngx-input-text i18n-title title="Action parameter" *ngIf="showActionParamField" formControlName="action_parameter" [error]="error?.action_parameter"></pngx-input-text>
+        <p class="small fst-italic mt-5" i18n>Assignments specified here will supersede any consumption templates.</p>
         <pngx-input-select i18n-title title="Assign title from" [items]="metadataTitleOptions" formControlName="assign_title_from"></pngx-input-select>
         <pngx-input-tags [allowCreate]="false" formControlName="assign_tags"></pngx-input-tags>
         <pngx-input-select i18n-title title="Assign document type" [items]="documentTypes" [allowNull]="true" formControlName="assign_document_type"></pngx-input-select>
         <pngx-input-select i18n-title title="Assign correspondent from" [items]="metadataCorrespondentOptions" formControlName="assign_correspondent_from"></pngx-input-select>
         <pngx-input-select *ngIf="showCorrespondentField" i18n-title title="Assign correspondent" [items]="correspondents" [allowNull]="true" formControlName="assign_correspondent"></pngx-input-select>
+        <pngx-input-check i18n-title title="Assign owner from rule" formControlName="assign_owner_from_rule"></pngx-input-check>
       </div>
     </div>
   </div>

src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.spec.ts

@@ -15,6 +15,7 @@ import { CorrespondentService } from 'src/app/services/rest/correspondent.servic
 import { DocumentTypeService } from 'src/app/services/rest/document-type.service'
 import { MailAccountService } from 'src/app/services/rest/mail-account.service'
 import { SettingsService } from 'src/app/services/settings.service'
+import { CheckComponent } from '../../input/check/check.component'
 import { NumberComponent } from '../../input/number/number.component'
 import { PermissionsFormComponent } from '../../input/permissions/permissions-form/permissions-form.component'
 import { SelectComponent } from '../../input/select/select.component'
@@ -27,9 +28,6 @@ describe('MailRuleEditDialogComponent', () => {
   let component: MailRuleEditDialogComponent
   let settingsService: SettingsService
   let fixture: ComponentFixture<MailRuleEditDialogComponent>
-  let accountService: MailAccountService
-  let correspondentService: CorrespondentService
-  let documentTypeService: DocumentTypeService
 
   beforeEach(async () => {
     TestBed.configureTestingModule({
@@ -43,6 +41,7 @@ describe('MailRuleEditDialogComponent', () => {
         NumberComponent,
         TagsComponent,
         SafeHtmlPipe,
+        CheckComponent,
       ],
       providers: [
         NgbActiveModal,

src-ui/src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.ts

@@ -79,6 +79,10 @@ const METADATA_TITLE_OPTIONS = [
     id: MailMetadataTitleOption.FromFilename,
     name: $localize`Use attachment filename as title`,
   },
+  {
+    id: MailMetadataTitleOption.None,
+    name: $localize`Do not assign title from this rule`,
+  },
 ]
 
 const METADATA_CORRESPONDENT_OPTIONS = [
@@ -168,6 +172,7 @@ export class MailRuleEditDialogComponent extends EditDialogComponent<PaperlessMa
         MailMetadataCorrespondentOption.FromNothing
       ),
       assign_correspondent: new FormControl(null),
+      assign_owner_from_rule: new FormControl(true),
     })
   }
 

src-ui/src/app/components/common/edit-dialog/user-edit-dialog/user-edit-dialog.component.html

@@ -6,7 +6,7 @@
     </div>
     <div class="modal-body">
       <div class="row">
-        <div class="col">
+        <div class="col-md-4">
           <pngx-input-text i18n-title title="Username" formControlName="username" [error]="error?.username"></pngx-input-text>
           <pngx-input-text i18n-title title="Email" formControlName="email" [error]="error?.email"></pngx-input-text>
           <pngx-input-password i18n-title title="Password" formControlName="password" [error]="error?.password"></pngx-input-password>

src-ui/src/app/components/common/input/tags/tags.component.html

@@ -1,5 +1,5 @@
 <div class="mb-3 paperless-input-select paperless-input-tags" [class.disabled]="disabled">
-  <label class="form-label" for="tags" i18n>Tags</label>
+  <label class="form-label" for="tags" i18n>{{title}}</label>
 
   <div class="input-group flex-nowrap">
     <ng-select #tagSelect name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="value"
@@ -7,7 +7,7 @@
       [multiple]="true"
       [closeOnSelect]="false"
       [clearSearchOnAdd]="true"
-      [hideSelected]="true"
+      [hideSelected]="tags.length > 0"
       [addTag]="allowCreate ? createTagRef : false"
       addTagText="Add tag"
       i18n-addTagText

src-ui/src/app/components/common/input/tags/tags.component.ts

@@ -59,6 +59,9 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
     })
   }
 
+  @Input()
+  title = $localize`Tags`
+
   @Input()
   disabled = false