Compare commits

..

24 Commits

Author SHA1 Message Date
shamoon
a0a9e0c6c8 Update views.py
[ci ckip]
2025-02-25 09:50:27 -08:00
shamoon
1c7c703e5f Merge migrations 2025-02-21 08:35:36 -08:00
shamoon
53e9e910d8 Merge branch 'dev' into feature-improve-paperless-task 2025-02-21 08:33:40 -08:00
shamoon
9fe611a24c Update views.py 2025-02-20 12:11:55 -08:00
shamoon
31e71aab83 Fix migrations merge 2025-02-17 08:19:11 -08:00
shamoon
7e7ce97d10 merge migrations 2025-02-17 08:19:11 -08:00
shamoon
e06adc58c7 Update tasks.service.ts 2025-02-17 08:19:11 -08:00
shamoon
7170ac31b7 Update test_api_tasks.py 2025-02-17 08:19:11 -08:00
shamoon
a0aa78c788 Translations 2025-02-17 08:19:11 -08:00
shamoon
f3438914cc Support acknowledged param 2025-02-17 08:19:11 -08:00
shamoon
e1b944ce6b Use choices for task name, rework task type 2025-02-17 08:19:11 -08:00
shamoon
0add5aab0e Styling, celery url 2025-02-17 08:19:11 -08:00
shamoon
c9adc74fa9 Styling, 4th column 2025-02-17 08:19:11 -08:00
shamoon
32abfbfc0a Health 2025-02-17 08:19:11 -08:00
shamoon
7f02f782f4 Fix warning 2025-02-17 08:19:11 -08:00
shamoon
7c3f011e84 Couple more test fixes 2025-02-17 08:19:11 -08:00
shamoon
5c68177960 Update tasks.py 2025-02-17 08:19:11 -08:00
shamoon
7a4666783e Fix tests, warning 2025-02-17 08:19:11 -08:00
shamoon
372825c271 Update translation strings 2025-02-17 08:19:11 -08:00
shamoon
abfddd6931 Fix tests 2025-02-17 08:19:11 -08:00
shamoon
b3d49dbf12 Add sanity check to system status 2025-02-17 08:19:11 -08:00
shamoon
673839265d Update system status to use classifier paperlesstask 2025-02-17 08:19:11 -08:00
shamoon
f31df22ab6 Revert "Tweak: more accurate classifier last trained time (#9004)"
This reverts commit 3314c59828.
2025-02-17 08:19:11 -08:00
shamoon
f897447a65 Create paperlesstasks for sanity, classifier
[ci skip]
2025-02-17 08:19:11 -08:00
63 changed files with 1101 additions and 1394 deletions

View File

@@ -123,13 +123,13 @@ RUN set -eux \
WORKDIR /usr/src/paperless/src/docker/
COPY [ \
"docker/rootfs/etc/ImageMagick-6/paperless-policy.xml", \
"docker/imagemagick-policy.xml", \
"./" \
]
RUN set -eux \
&& echo "Configuring ImageMagick" \
&& mv paperless-policy.xml /etc/ImageMagick-6/policy.xml
&& mv imagemagick-policy.xml /etc/ImageMagick-6/policy.xml
# Packages needed only for building a few quick Python
# dependencies

View File

@@ -65,7 +65,7 @@ services:
command: /bin/sh -c "chown -R paperless:paperless /usr/src/paperless/paperless-ngx/src/documents/static/frontend && chown -R paperless:paperless /usr/src/paperless/paperless-ngx/.ruff_cache && while sleep 1000; do :; done"
gotenberg:
image: docker.io/gotenberg/gotenberg:8.17
image: docker.io/gotenberg/gotenberg:7.10
restart: unless-stopped
# The Gotenberg Chromium route is used to convert .eml files. We do not

View File

@@ -5,7 +5,7 @@
services:
gotenberg:
image: docker.io/gotenberg/gotenberg:8.17
image: docker.io/gotenberg/gotenberg:8.7
hostname: gotenberg
container_name: gotenberg
network_mode: host

View File

@@ -77,7 +77,7 @@ services:
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
gotenberg:
image: docker.io/gotenberg/gotenberg:8.17
image: docker.io/gotenberg/gotenberg:8.7
restart: unless-stopped
# The gotenberg chromium route is used to convert .eml files. We do not
# want to allow external content like tracking pixels or even javascript.

View File

@@ -71,7 +71,7 @@ services:
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
gotenberg:
image: docker.io/gotenberg/gotenberg:8.17
image: docker.io/gotenberg/gotenberg:8.7
restart: unless-stopped
# The gotenberg chromium route is used to convert .eml files. We do not

View File

@@ -59,7 +59,7 @@ services:
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
gotenberg:
image: docker.io/gotenberg/gotenberg:8.17
image: docker.io/gotenberg/gotenberg:8.7
restart: unless-stopped
# The gotenberg chromium route is used to convert .eml files. We do not

View File

@@ -1,18 +1,10 @@
#!/command/with-contenv /usr/bin/bash
# shellcheck shell=bash
cd ${PAPERLESS_SRC_DIR}
if [[ -n "${PAPERLESS_CONSUMER_DISABLE}" ]]; then
echo "[svc-consumer] Consumer is disabled, exiting"
# https://skarnet.org/software/s6/s6-svc.html
s6-svc -Od .
if [[ -n "${USER_IS_NON_ROOT}" ]]; then
exec python3 manage.py document_consumer
else
cd ${PAPERLESS_SRC_DIR}
if [[ -n "${USER_IS_NON_ROOT}" ]]; then
exec python3 manage.py document_consumer
else
exec s6-setuidgid paperless python3 manage.py document_consumer
fi
exec s6-setuidgid paperless python3 manage.py document_consumer
fi

View File

@@ -1030,11 +1030,6 @@ be used with caution!
## Document Consumption {#consume_config}
#### [`PAPERLESS_CONSUMER_DISABLE=<bool>`](#PAPERLESS_CONSUMER_DISABLE) {#PAPERLESS_CONSUMER_DISABLE}
: Completely disable the directory-based consumer in docker. If you don't plan to consume documents
via the consumption directory, you can disable the consumer to save resources.
#### [`PAPERLESS_CONSUMER_DELETE_DUPLICATES=<bool>`](#PAPERLESS_CONSUMER_DELETE_DUPLICATES) {#PAPERLESS_CONSUMER_DELETE_DUPLICATES}
: When the consumer detects a duplicate document, it will not touch

View File

@@ -714,8 +714,6 @@ the Pi and configuring some options in paperless can help improve
performance immensely:
- Stick with SQLite to save some resources.
- If you do not need the filesystem-based consumer, consider disabling it
entirely by setting [`PAPERLESS_CONSUMER_DISABLE`](configuration.md#PAPERLESS_CONSUMER_DISABLE) to `true`.
- Consider setting [`PAPERLESS_OCR_PAGES`](configuration.md#PAPERLESS_OCR_PAGES) to 1, so that paperless will
only OCR the first page of your documents. In most cases, this page
contains enough information to be able to find it.

File diff suppressed because it is too large Load Diff

View File

@@ -303,12 +303,17 @@ describe('SettingsComponent', () => {
redis_error:
'Error 61 connecting to localhost:6379. Connection refused.',
celery_status: SystemStatusItemStatus.ERROR,
celery_url: 'celery@localhost',
celery_error: 'Error connecting to celery@localhost',
index_status: SystemStatusItemStatus.OK,
index_last_modified: new Date().toISOString(),
index_error: null,
classifier_status: SystemStatusItemStatus.OK,
classifier_last_trained: new Date().toISOString(),
classifier_error: null,
sanity_check_status: SystemStatusItemStatus.ERROR,
sanity_check_last_run: new Date().toISOString(),
sanity_check_error: 'Error running sanity check.',
},
}
jest.spyOn(systemStatusService, 'get').mockReturnValue(of(status))

View File

@@ -19,6 +19,7 @@ import { allIcons, NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
import { routes } from 'src/app/app-routing.module'
import {
PaperlessTask,
PaperlessTaskName,
PaperlessTaskStatus,
PaperlessTaskType,
} from 'src/app/data/paperless-task'
@@ -39,7 +40,8 @@ const tasks: PaperlessTask[] = [
task_file_name: 'test.pdf',
date_created: new Date('2023-03-01T10:26:03.093116Z'),
date_done: new Date('2023-03-01T10:26:07.223048Z'),
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Failed,
result: 'test.pd: Not consuming test.pdf: It is a duplicate of test (#100)',
acknowledged: false,
@@ -51,7 +53,8 @@ const tasks: PaperlessTask[] = [
task_file_name: '191092.pdf',
date_created: new Date('2023-03-01T09:26:03.093116Z'),
date_done: new Date('2023-03-01T09:26:07.223048Z'),
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Failed,
result:
'191092.pd: Not consuming 191092.pdf: It is a duplicate of 191092 (#311)',
@@ -64,7 +67,8 @@ const tasks: PaperlessTask[] = [
task_file_name: 'Scan Jun 6, 2023 at 3.19 PM.pdf',
date_created: new Date('2023-06-06T15:22:05.722323-07:00'),
date_done: new Date('2023-06-06T15:22:14.564305-07:00'),
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Pending,
result: null,
acknowledged: false,
@@ -76,7 +80,8 @@ const tasks: PaperlessTask[] = [
task_file_name: 'paperless-mail-l4dkg8ir',
date_created: new Date('2023-06-04T11:24:32.898089-07:00'),
date_done: new Date('2023-06-04T11:24:44.678605-07:00'),
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Complete,
result: 'Success. New document id 422 created',
acknowledged: false,
@@ -88,7 +93,8 @@ const tasks: PaperlessTask[] = [
task_file_name: 'onlinePaymentSummary.pdf',
date_created: new Date('2023-06-01T13:49:51.631305-07:00'),
date_done: new Date('2023-06-01T13:49:54.190220-07:00'),
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Complete,
result: 'Success. New document id 421 created',
acknowledged: false,
@@ -100,7 +106,8 @@ const tasks: PaperlessTask[] = [
task_file_name: 'paperless-mail-_rrpmqk6',
date_created: new Date('2023-06-07T02:54:35.694916Z'),
date_done: null,
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Started,
result: null,
acknowledged: false,
@@ -155,7 +162,9 @@ describe('TasksComponent', () => {
jest.useFakeTimers()
fixture.detectChanges()
httpTestingController
.expectOne(`${environment.apiBaseUrl}tasks/`)
.expectOne(
`${environment.apiBaseUrl}tasks/?task_name=consume_file&acknowledged=false`
)
.flush(tasks)
})

View File

@@ -84,7 +84,7 @@ export class SplitConfirmDialogComponent
addSplit() {
if (this.page === this.totalPages) return
this.pages.add(this.page)
this.pages = new Set(Array.from(this.pages).sort((a, b) => a - b))
this.pages = new Set(Array.from(this.pages).sort())
this.confirmButtonEnabled = this.pages.size > 0
}

View File

@@ -9,24 +9,19 @@
</div>
<div class="modal-body">
<div class="row">
<div class="col-md-6">
<div class="col-md-4">
<pngx-input-text [horizontal]="true" i18n-title title="Name" formControlName="name" [error]="error?.name" autocomplete="off"></pngx-input-text>
</div>
<div class="col-md-4">
<div class="col-md-3">
<pngx-input-select [horizontal]="true" i18n-title title="Account" [items]="accounts" formControlName="account"></pngx-input-select>
</div>
<div class="col-md-3">
<pngx-input-number [horizontal]="true" i18n-title title="Order" formControlName="order" [showAdd]="false" [error]="error?.order"></pngx-input-number>
</div>
<div class="col-md-2 pt-2">
<pngx-input-switch [horizontal]="true" i18n-title title="Enabled" formControlName="enabled"></pngx-input-switch>
</div>
</div>
<div class="row">
<div class="col-md-6">
<pngx-input-number [horizontal]="true" i18n-title title="Order" formControlName="order" [showAdd]="false" [error]="error?.order"></pngx-input-number>
</div>
<div class="col-md-6">
<pngx-input-switch [horizontal]="true" i18n-title title="Stop further processing" formControlName="stop_processing" i18n-hint hint="Stop processing further rules if this rule queues any document(s)."></pngx-input-switch>
</div>
</div>
<hr class="mt-0"/>
<div class="row">
<p class="small" i18n>Paperless will only process mails that match <em>all</em> of the criteria specified below.</p>

View File

@@ -221,7 +221,6 @@ export class MailRuleEditDialogComponent extends EditDialogComponent<MailRule> {
),
assign_correspondent: new FormControl(null),
assign_owner_from_rule: new FormControl(true),
stop_processing: new FormControl(false),
})
}

View File

@@ -1,32 +0,0 @@
<div class="modal-header">
<h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
<button type="button" class="btn-close" aria-label="Close" (click)="close()"></button>
</div>
<div class="modal-body">
<div class="mb-1">
<label for="email" class="form-label" i18n>Email address(es)</label>
<input type="email" class="form-control" id="email" [(ngModel)]="emailAddress">
</div>
<div class="mb-1">
<label for="email" class="form-label" i18n>Subject</label>
<input type="email" class="form-control" id="subject" [(ngModel)]="emailSubject">
</div>
<div>
<label for="message" class="form-label" i18n>Message</label>
<textarea class="form-control" id="message" rows="3" [(ngModel)]="emailMessage"></textarea>
</div>
</div>
<div class="modal-footer">
<div class="input-group">
<div class="input-group-text flex-grow-1">
<input class="form-check-input mt-0 me-2" type="checkbox" role="switch" id="useArchiveVersion" [disabled]="!hasArchiveVersion" [(ngModel)]="useArchiveVersion">
<label class="form-check-label w-100 text-start" for="useArchiveVersion" i18n>Use archive version</label>
</div>
<button type="submit" class="btn btn-outline-primary" (click)="emailDocument()" [disabled]="loading || emailAddress.length === 0 || emailMessage.length === 0 || emailSubject.length === 0">
@if (loading) {
<div class="spinner-border spinner-border-sm me-2" role="status"></div>
}
<ng-container i18n>Send email</ng-container>
</button>
</div>
</div>

View File

@@ -1,72 +0,0 @@
import { ComponentFixture, TestBed } from '@angular/core/testing'
import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http'
import { provideHttpClientTesting } from '@angular/common/http/testing'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
import { of, throwError } from 'rxjs'
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
import { PermissionsService } from 'src/app/services/permissions.service'
import { DocumentService } from 'src/app/services/rest/document.service'
import { ToastService } from 'src/app/services/toast.service'
import { EmailDocumentDialogComponent } from './email-document-dialog.component'
describe('EmailDocumentDialogComponent', () => {
let component: EmailDocumentDialogComponent
let fixture: ComponentFixture<EmailDocumentDialogComponent>
let documentService: DocumentService
let permissionsService: PermissionsService
let toastService: ToastService
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [
EmailDocumentDialogComponent,
IfPermissionsDirective,
NgxBootstrapIconsModule.pick(allIcons),
],
providers: [
provideHttpClient(withInterceptorsFromDi()),
provideHttpClientTesting(),
NgbActiveModal,
],
}).compileComponents()
fixture = TestBed.createComponent(EmailDocumentDialogComponent)
documentService = TestBed.inject(DocumentService)
toastService = TestBed.inject(ToastService)
component = fixture.componentInstance
fixture.detectChanges()
})
it('should set hasArchiveVersion and useArchiveVersion', () => {
expect(component.hasArchiveVersion).toBeTruthy()
component.hasArchiveVersion = false
expect(component.hasArchiveVersion).toBeFalsy()
expect(component.useArchiveVersion).toBeFalsy()
})
it('should support sending document via email, showing error if needed', () => {
const toastErrorSpy = jest.spyOn(toastService, 'showError')
const toastSuccessSpy = jest.spyOn(toastService, 'showInfo')
component.emailAddress = 'hello@paperless-ngx.com'
component.emailSubject = 'Hello'
component.emailMessage = 'World'
jest
.spyOn(documentService, 'emailDocument')
.mockReturnValue(throwError(() => new Error('Unable to email document')))
component.emailDocument()
expect(toastErrorSpy).toHaveBeenCalled()
jest.spyOn(documentService, 'emailDocument').mockReturnValue(of(true))
component.emailDocument()
expect(toastSuccessSpy).toHaveBeenCalled()
})
it('should close the dialog', () => {
const activeModal = TestBed.inject(NgbActiveModal)
const closeSpy = jest.spyOn(activeModal, 'close')
component.close()
expect(closeSpy).toHaveBeenCalled()
})
})

View File

@@ -1,77 +0,0 @@
import { Component, Input } from '@angular/core'
import { FormsModule } from '@angular/forms'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
import { DocumentService } from 'src/app/services/rest/document.service'
import { ToastService } from 'src/app/services/toast.service'
import { LoadingComponentWithPermissions } from '../../loading-component/loading.component'
@Component({
selector: 'pngx-email-document-dialog',
templateUrl: './email-document-dialog.component.html',
styleUrl: './email-document-dialog.component.scss',
imports: [FormsModule, NgxBootstrapIconsModule],
})
export class EmailDocumentDialogComponent extends LoadingComponentWithPermissions {
@Input()
title = $localize`Email Document`
@Input()
documentId: number
private _hasArchiveVersion: boolean = true
@Input()
set hasArchiveVersion(value: boolean) {
this._hasArchiveVersion = value
this.useArchiveVersion = value
}
get hasArchiveVersion(): boolean {
return this._hasArchiveVersion
}
public useArchiveVersion: boolean = true
public emailAddress: string = ''
public emailSubject: string = ''
public emailMessage: string = ''
constructor(
private activeModal: NgbActiveModal,
private documentService: DocumentService,
private toastService: ToastService
) {
super()
this.loading = false
}
public emailDocument() {
this.loading = true
this.documentService
.emailDocument(
this.documentId,
this.emailAddress,
this.emailSubject,
this.emailMessage,
this.useArchiveVersion
)
.subscribe({
next: () => {
this.loading = false
this.emailAddress = ''
this.emailSubject = ''
this.emailMessage = ''
this.toastService.showInfo($localize`Email sent`)
},
error: (e) => {
this.loading = false
this.toastService.showError($localize`Error emailing document`, e)
},
})
}
public close() {
this.activeModal.close()
}
}

View File

@@ -1,68 +0,0 @@
<div class="modal-header">
<h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
<button type="button" class="btn-close" aria-label="Close" (click)="close()"></button>
</div>
<div class="modal-body p-0">
<ul class="list-group list-group-flush">
@if (!shareLinks || shareLinks.length === 0) {
<li class="list-group-item fst-italic small text-center text-secondary" i18n>
No existing links
</li>
}
@for (link of shareLinks; track link) {
<li class="list-group-item">
<div class="input-group w-100">
<input type="text" class="form-control" aria-label="Share link" [value]="getShareUrl(link)" readonly>
@if (link.expiration) {
<span class="input-group-text">
{{ getDaysRemaining(link) }}
</span>
}
<button type="button" class="btn btn-outline-primary" (click)="copy(link)">
@if (copied !== link.id) {
<i-bs width="1.2em" height="1.2em" name="clipboard-fill"></i-bs>
}
@if (copied === link.id) {
<i-bs width="1.2em" height="1.2em" name="clipboard-check-fill"></i-bs>
}
<span class="visually-hidden" i18n>Copy</span>
</button>
@if (canShare(link)) {
<button type="button" class="btn btn-outline-primary" (click)="share(link)">
<i-bs width="1.2em" height="1.2em" name="box-arrow-up"></i-bs><span class="visually-hidden" i18n>Share</span>
</button>
}
<button type="button" class="btn btn-outline-danger" (click)="delete(link)">
<i-bs width="1.2em" height="1.2em" name="trash"></i-bs><span class="visually-hidden" i18n>Delete</span>
</button>
</div>
<span class="badge copied-badge bg-primary small fade ms-4 position-absolute top-50 translate-middle-y pe-none z-3" [class.show]="copied === link.id" i18n>Copied!</span>
</li>
}
</ul>
</div>
<div class="modal-footer">
<div class="input-group w-100">
<div class="form-check form-switch ms-auto">
<input class="form-check-input" type="checkbox" role="switch" id="versionSwitch" [disabled]="!hasArchiveVersion" [(ngModel)]="useArchiveVersion">
<label class="form-check-label" for="versionSwitch" i18n>Share archive version</label>
</div>
</div>
<div class="input-group w-100 mt-2">
<label class="input-group-text" for="addLink"><ng-container i18n>Expires</ng-container>:</label>
<select class="form-select fs-6" [(ngModel)]="expirationDays">
@for (option of EXPIRATION_OPTIONS; track option) {
<option [ngValue]="option.value">{{ option.label }}</option>
}
</select>
<button class="btn btn-outline-primary ms-auto" type="button" (click)="createLink()" [disabled]="loading">
@if (loading) {
<div class="spinner-border spinner-border-sm me-2" role="status"></div>
}
@if (!loading) {
<i-bs name="plus"></i-bs>
}
<ng-container i18n>Create</ng-container>
</button>
</div>
</div>

View File

@@ -1,3 +0,0 @@
.copied-badge {
right: 15em;
}

View File

@@ -0,0 +1,70 @@
<div ngbDropdown>
<button class="btn btn-sm btn-outline-primary" id="shareLinksDropdown" [disabled]="disabled" ngbDropdownToggle>
<i-bs name="link"></i-bs>
<div class="d-none d-sm-inline">&nbsp;<ng-container i18n>Share Links</ng-container></div>
</button>
<div ngbDropdownMenu aria-labelledby="shareLinksDropdown" class="shadow share-links-dropdown">
<ul class="list-group list-group-flush">
@if (!shareLinks || shareLinks.length === 0) {
<li class="list-group-item fst-italic small text-center text-secondary" i18n>
No existing links
</li>
}
@for (link of shareLinks; track link) {
<li class="list-group-item">
<div class="input-group input-group-sm w-100">
<input type="text" class="form-control" aria-label="Share link" [value]="getShareUrl(link)" readonly>
@if (link.expiration) {
<span class="input-group-text">
{{ getDaysRemaining(link) }}
</span>
}
<button type="button" class="btn btn-sm btn-outline-primary" (click)="copy(link)">
@if (copied !== link.id) {
<i-bs width="1.2em" height="1.2em" name="clipboard-fill"></i-bs>
}
@if (copied === link.id) {
<i-bs width="1.2em" height="1.2em" name="clipboard-check-fill"></i-bs>
}
<span class="visually-hidden" i18n>Copy</span>
</button>
@if (canShare(link)) {
<button type="button" class="btn btn-sm btn-outline-primary" (click)="share(link)">
<i-bs width="1.2em" height="1.2em" name="box-arrow-up"></i-bs><span class="visually-hidden" i18n>Share</span>
</button>
}
<button type="button" class="btn btn-sm btn-outline-danger" (click)="delete(link)">
<i-bs width="1.2em" height="1.2em" name="trash"></i-bs><span class="visually-hidden" i18n>Delete</span>
</button>
</div>
<span class="badge copied-badge bg-primary small fade ms-4 position-absolute top-50 translate-middle-y pe-none z-3" [class.show]="copied === link.id" i18n>Copied!</span>
</li>
}
<li class="list-group-item pt-3 pb-2">
<div class="input-group input-group-sm w-100">
<div class="form-check form-switch ms-auto small">
<input class="form-check-input" type="checkbox" role="switch" id="versionSwitch" [disabled]="!hasArchiveVersion" [(ngModel)]="useArchiveVersion">
<label class="form-check-label" for="versionSwitch" i18n>Share archive version</label>
</div>
</div>
<div class="input-group input-group-sm w-100 mt-2">
<label class="input-group-text" for="addLink"><ng-container i18n>Expires</ng-container>:</label>
<select class="form-select form-select-sm" [(ngModel)]="expirationDays">
@for (option of EXPIRATION_OPTIONS; track option) {
<option [ngValue]="option.value">{{ option.label }}</option>
}
</select>
<button class="btn btn-sm btn-outline-primary ms-auto" type="button" (click)="createLink()" [disabled]="loading">
@if (loading) {
<div class="spinner-border spinner-border-sm me-2" role="status"></div>
}
@if (!loading) {
<i-bs name="plus"></i-bs>
}
<ng-container i18n>Create</ng-container>
</button>
</div>
</li>
</ul>
</div>
</div>

View File

@@ -0,0 +1,14 @@
.share-links-dropdown {
min-width: 350px;
// correct position on mobile
@media (max-width: 575.98px) {
&.show {
margin-left: -175px !important;
}
}
}
.copied-badge {
right: 7.5em;
}

View File

@@ -11,18 +11,17 @@ import {
tick,
} from '@angular/core/testing'
import { By } from '@angular/platform-browser'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
import { of, throwError } from 'rxjs'
import { FileVersion, ShareLink } from 'src/app/data/share-link'
import { ShareLinkService } from 'src/app/services/rest/share-link.service'
import { ToastService } from 'src/app/services/toast.service'
import { environment } from 'src/environments/environment'
import { ShareLinksDialogComponent } from './share-links-dialog.component'
import { ShareLinksDropdownComponent } from './share-links-dropdown.component'
describe('ShareLinksDialogComponent', () => {
let component: ShareLinksDialogComponent
let fixture: ComponentFixture<ShareLinksDialogComponent>
describe('ShareLinksDropdownComponent', () => {
let component: ShareLinksDropdownComponent
let fixture: ComponentFixture<ShareLinksDropdownComponent>
let shareLinkService: ShareLinkService
let toastService: ToastService
let httpController: HttpTestingController
@@ -31,17 +30,16 @@ describe('ShareLinksDialogComponent', () => {
beforeEach(() => {
TestBed.configureTestingModule({
imports: [
ShareLinksDialogComponent,
ShareLinksDropdownComponent,
NgxBootstrapIconsModule.pick(allIcons),
],
providers: [
provideHttpClient(withInterceptorsFromDi()),
provideHttpClientTesting(),
NgbActiveModal,
],
})
fixture = TestBed.createComponent(ShareLinksDialogComponent)
fixture = TestBed.createComponent(ShareLinksDropdownComponent)
shareLinkService = TestBed.inject(ShareLinkService)
toastService = TestBed.inject(ToastService)
httpController = TestBed.inject(HttpTestingController)
@@ -234,11 +232,4 @@ describe('ShareLinksDialogComponent', () => {
]
).toBeTruthy()
})
it('should support close', () => {
const activeModal = TestBed.inject(NgbActiveModal)
const closeSpy = jest.spyOn(activeModal, 'close')
component.close()
expect(closeSpy).toHaveBeenCalled()
})
})

View File

@@ -1,7 +1,7 @@
import { Clipboard } from '@angular/cdk/clipboard'
import { Component, Input, OnInit } from '@angular/core'
import { FormsModule, ReactiveFormsModule } from '@angular/forms'
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
import { first } from 'rxjs'
import { FileVersion, ShareLink } from 'src/app/data/share-link'
@@ -10,12 +10,17 @@ import { ToastService } from 'src/app/services/toast.service'
import { environment } from 'src/environments/environment'
@Component({
selector: 'pngx-share-links-dialog',
templateUrl: './share-links-dialog.component.html',
styleUrls: ['./share-links-dialog.component.scss'],
imports: [FormsModule, ReactiveFormsModule, NgxBootstrapIconsModule],
selector: 'pngx-share-links-dropdown',
templateUrl: './share-links-dropdown.component.html',
styleUrls: ['./share-links-dropdown.component.scss'],
imports: [
FormsModule,
ReactiveFormsModule,
NgbDropdownModule,
NgxBootstrapIconsModule,
],
})
export class ShareLinksDialogComponent implements OnInit {
export class ShareLinksDropdownComponent implements OnInit {
EXPIRATION_OPTIONS = [
{ label: $localize`1 day`, value: 1 },
{ label: $localize`7 days`, value: 7 },
@@ -36,6 +41,9 @@ export class ShareLinksDialogComponent implements OnInit {
}
}
@Input()
disabled: boolean = false
private _hasArchiveVersion: boolean = true
@Input()
@@ -59,7 +67,6 @@ export class ShareLinksDialogComponent implements OnInit {
useArchiveVersion: boolean = true
constructor(
private activeModal: NgbActiveModal,
private shareLinkService: ShareLinkService,
private toastService: ToastService,
private clipboard: Clipboard
@@ -162,8 +169,4 @@ export class ShareLinksDialogComponent implements OnInit {
},
})
}
close() {
this.activeModal.close()
}
}

View File

@@ -1,5 +1,5 @@
<div class="modal-header">
<h5 class="modal-title" id="modal-basic-title" i18n>System Status</h5>
<h6 class="modal-title" id="modal-basic-title" i18n>System Status</h6>
<button type="button" class="btn-close" aria-label="Close" (click)="close()"></button>
</div>
<div class="modal-body">
@@ -11,11 +11,11 @@
</div>
</div>
} @else {
<div class="row row-cols-1 row-cols-md-3 g-3">
<div class="col">
<div class="row row-cols-1 row-cols-md-4 g-3">
<div class="col-4">
<div class="card bg-light h-100">
<div class="card-header">
<h5 class="card-title mb-0" i18n>Environment</h5>
<h6 class="card-title mb-0" i18n>Environment</h6>
</div>
<div class="card-body">
<dl class="card-text">
@@ -38,37 +38,96 @@
<div class="col">
<div class="card bg-light h-100">
<div class="card-header">
<h5 class="card-title mb-0" i18n>Database</h5>
<h6 class="card-title mb-0" i18n>Database</h6>
</div>
<div class="card-body">
<dl class="card-text">
<dt i18n>Type</dt>
<dd>{{status.database.type}}</dd>
<dt i18n>Status</dt>
<dd class="d-flex align-items-center">
{{status.database.status}}
@if (status.database.status === 'OK') {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1" ngbPopover="{{status.database.url}}" triggers="mouseenter:mouseleave"></i-bs>
} @else {
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1" ngbPopover="{{status.database.url}}: {{status.database.error}}" triggers="mouseenter:mouseleave"></i-bs>
}
<dd>
<div class="badge text-uppercase bg-info text-dark" [ngbPopover]="databaseStatus" triggers="mouseenter:mouseleave">
{{status.database.status}}
@if (status.database.status === 'OK') {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
} @else {
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1"></i-bs>
}
</div>
<ng-template #databaseStatus>
@if (status.database.status === 'OK') {
{{status.database.url}}
} @else {
{{status.database.url}}: {{status.database.error}}
}
</ng-template>
</dd>
<dt i18n>Migration Status</dt>
<dd class="d-flex align-items-center">
@if (status.database.migration_status.unapplied_migrations.length === 0) {
<ng-container i18n>Up to date</ng-container><i-bs name="check-circle-fill" class="text-primary ms-2 lh-1" [ngbPopover]="migrationStatus" triggers="mouseenter:mouseleave"></i-bs>
} @else {
<ng-container>{{status.database.migration_status.unapplied_migrations.length}} Pending</ng-container><i-bs name="exclamation-triangle-fill" class="text-warning ms-2 lh-1" [ngbPopover]="migrationStatus" triggers="mouseenter:mouseleave"></i-bs>
}
<ng-template #migrationStatus>
<h6><ng-container i18n>Latest Migration</ng-container>:</h6> <span class="font-monospace small">{{status.database.migration_status.latest_migration}}</span>
@if (status.database.migration_status.unapplied_migrations.length > 0) {
<h6 class="mt-3"><ng-container i18n>Pending Migrations</ng-container>:</h6>
<ul>
@for (migration of status.database.migration_status.unapplied_migrations; track migration) {
<li class="font-monospace small">{{migration}}</li>
}
</ul>
<dd>
<div class="badge text-uppercase bg-info text-dark" [ngbPopover]="migrationStatus" triggers="mouseenter:mouseleave">
@if (status.database.migration_status.unapplied_migrations.length === 0) {
<ng-container i18n>Up to date</ng-container><i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
} @else {
<ng-container>{{status.database.migration_status.unapplied_migrations.length}} Pending</ng-container><i-bs name="exclamation-triangle-fill" class="text-warning ms-2 lh-1"></i-bs>
}
<ng-template #migrationStatus>
<h6><ng-container i18n>Latest Migration</ng-container>:</h6> <span class="font-monospace small">{{status.database.migration_status.latest_migration}}</span>
@if (status.database.migration_status.unapplied_migrations.length > 0) {
<h6 class="mt-3"><ng-container i18n>Pending Migrations</ng-container>:</h6>
<ul>
@for (migration of status.database.migration_status.unapplied_migrations; track migration) {
<li class="font-monospace small">{{migration}}</li>
}
</ul>
}
</ng-template>
</div>
</dd>
</dl>
</div>
</div>
</div>
<div class="col">
<div class="card bg-light h-100">
<div class="card-header">
<h6 class="card-title mb-0" i18n>Tasks Queue</h6>
</div>
<div class="card-body">
<dl class="card-text">
<dt i18n>Redis Status</dt>
<dd>
<div class="badge text-uppercase bg-info text-dark" [ngbPopover]="redisStatus" triggers="mouseenter:mouseleave">
{{status.tasks.redis_status}}
@if (status.tasks.redis_status === 'OK') {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
} @else {
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1"></i-bs>
}
</div>
<ng-template #redisStatus>
@if (status.tasks.redis_status === 'OK') {
{{status.tasks.redis_url}}
} @else {
{{status.tasks.redis_url}}: {{status.tasks.redis_error}}
}
</ng-template>
</dd>
<dt i18n>Celery Status</dt>
<dd>
<div class="badge text-uppercase bg-info text-dark" [ngbPopover]="celeryStatus" triggers="mouseenter:mouseleave">
{{status.tasks.celery_status}}
@if (status.tasks.celery_status === 'OK') {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
} @else {
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1"></i-bs>
}
</div>
<ng-template #celeryStatus>
@if (status.tasks.celery_status === 'OK') {
{{status.tasks.celery_url}}
} @else {
{{status.tasks.celery_error}}
}
</ng-template>
</dd>
@@ -80,63 +139,79 @@
<div class="col">
<div class="card bg-light h-100">
<div class="card-header">
<h5 class="card-title mb-0" i18n>Tasks</h5>
<h6 class="card-title mb-0" i18n>Health</h6>
</div>
<div class="card-body">
<dl class="card-text">
<dt i18n>Redis Status</dt>
<dd class="d-flex align-items-center">
{{status.tasks.redis_status}}
@if (status.tasks.redis_status === 'OK') {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1" ngbPopover="{{status.tasks.redis_url}}" triggers="mouseenter:mouseleave"></i-bs>
} @else {
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1" ngbPopover="{{status.tasks.redis_url}}: {{status.tasks.redis_error}}" triggers="mouseenter:mouseleave"></i-bs>
}
</dd>
<dt i18n>Celery Status</dt>
<dd class="d-flex align-items-center">
{{status.tasks.celery_status}}
@if (status.tasks.celery_status === 'OK') {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
} @else {
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1"></i-bs>
}
</dd>
<dt i18n>Search Index</dt>
<dd class="d-flex align-items-center">
{{status.tasks.index_status}}
@if (status.tasks.index_status === 'OK') {
@if (isStale(status.tasks.index_last_modified)) {
<i-bs name="exclamation-triangle-fill" class="text-warning ms-2 lh-1" [ngbPopover]="indexStatus" triggers="mouseenter:mouseleave"></i-bs>
<dd>
<div class="badge text-uppercase bg-info text-dark" [ngbPopover]="indexStatus" triggers="mouseenter:mouseleave">
{{status.tasks.index_status}}
@if (status.tasks.index_status === 'OK') {
@if (isStale(status.tasks.index_last_modified)) {
<i-bs name="exclamation-triangle-fill" class="text-warning ms-2 lh-1"></i-bs>
} @else {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
}
} @else {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1" [ngbPopover]="indexStatus" triggers="mouseenter:mouseleave"></i-bs>
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1"></i-bs>
}
} @else {
<i-bs name="exclamation-triangle-fill" class="text-danger ms-2 lh-1" ngbPopover="{{status.tasks.index_error}}" triggers="mouseenter:mouseleave"></i-bs>
}
</div>
</dd>
<ng-template #indexStatus>
<h6><ng-container i18n>Last Updated</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.index_last_modified | customDate:'medium'}}</span>
@if (status.tasks.index_status === 'OK') {
<h6><ng-container i18n>Last Updated</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.index_last_modified | customDate:'medium'}}</span>
} @else {
<h6><ng-container i18n>Error</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.index_error}}</span>
}
</ng-template>
<dt i18n>Classifier</dt>
<dd class="d-flex align-items-center">
{{status.tasks.classifier_status}}
@if (status.tasks.classifier_status === 'OK') {
@if (isStale(status.tasks.classifier_last_trained)) {
<i-bs name="exclamation-triangle-fill" class="text-warning ms-2 lh-1" [ngbPopover]="classifierStatus" triggers="mouseenter:mouseleave"></i-bs>
<dd>
<div class="badge text-uppercase bg-info text-dark" [ngbPopover]="classifierStatus" triggers="mouseenter:mouseleave">
{{status.tasks.classifier_status}}
@if (status.tasks.classifier_status === 'OK') {
@if (isStale(status.tasks.classifier_last_trained)) {
<i-bs name="exclamation-triangle-fill" class="text-warning ms-2 lh-1"></i-bs>
} @else {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
}
} @else {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1" [ngbPopover]="classifierStatus" triggers="mouseenter:mouseleave"></i-bs>
}
} @else {
<i-bs name="exclamation-triangle-fill" class="ms-2 lh-1"
[class.text-danger]="status.tasks.classifier_status === SystemStatusItemStatus.ERROR"
[class.text-warning]="status.tasks.classifier_status === SystemStatusItemStatus.WARNING"
ngbPopover="{{status.tasks.classifier_error}}"
triggers="mouseenter:mouseleave"></i-bs>
}
[class.text-warning]="status.tasks.classifier_status === SystemStatusItemStatus.WARNING"></i-bs>
}
</div>
</dd>
<ng-template #classifierStatus>
<h6><ng-container i18n>Last Trained</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.classifier_last_trained | customDate:'medium'}}</span>
@if (status.tasks.classifier_status === 'OK') {
<h6><ng-container i18n>Last Trained</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.classifier_last_trained | customDate:'medium'}}</span>
} @else {
<h6><ng-container i18n>Error</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.classifier_error}}</span>
}
</ng-template>
<dt i18n>Sanity Checker</dt>
<dd>
<div class="badge text-uppercase bg-info text-dark" [ngbPopover]="sanityCheckerStatus" triggers="mouseenter:mouseleave">
{{status.tasks.sanity_check_status}}
@if (status.tasks.sanity_check_status === 'OK') {
@if (isStale(status.tasks.sanity_check_last_run)) {
<i-bs name="exclamation-triangle-fill" class="text-warning ms-2 lh-1"></i-bs>
} @else {
<i-bs name="check-circle-fill" class="text-primary ms-2 lh-1"></i-bs>
}
} @else {
<i-bs name="exclamation-triangle-fill" class="ms-2 lh-1"
[class.text-danger]="status.tasks.sanity_check_status === SystemStatusItemStatus.ERROR"
[class.text-warning]="status.tasks.sanity_check_status === SystemStatusItemStatus.WARNING"></i-bs>
}
</div>
</dd>
<ng-template #sanityCheckerStatus>
@if (status.tasks.sanity_check_status === 'OK') {
<h6><ng-container i18n>Last Run</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.sanity_check_last_run | customDate:'medium'}}</span>
} @else {
<h6><ng-container i18n>Error</ng-container>:</h6> <span class="font-monospace small">{{status.tasks.sanity_check_error}}</span>
}
</ng-template>
</dl>
</div>

View File

@@ -0,0 +1,3 @@
.border-primary {
--bs-border-color: var(--bs-primary);
}

View File

@@ -36,12 +36,17 @@ const status: SystemStatus = {
redis_status: SystemStatusItemStatus.ERROR,
redis_error: 'Error 61 connecting to localhost:6379. Connection refused.',
celery_status: SystemStatusItemStatus.ERROR,
celery_url: 'celery@localhost',
celery_error: 'Error connecting to celery@localhost',
index_status: SystemStatusItemStatus.OK,
index_last_modified: new Date().toISOString(),
index_error: null,
classifier_status: SystemStatusItemStatus.OK,
classifier_last_trained: new Date().toISOString(),
classifier_error: null,
sanity_check_status: SystemStatusItemStatus.OK,
sanity_check_last_run: new Date().toISOString(),
sanity_check_error: null,
},
}

View File

@@ -81,24 +81,7 @@
(added)="addField($event)">
</pngx-custom-fields-dropdown>
<div class="ms-auto" ngbDropdown>
<button class="btn btn-sm btn-outline-primary" id="sendDropdown" ngbDropdownToggle>
<i-bs name="send"></i-bs>
<div class="d-none d-sm-inline">&nbsp;<ng-container i18n>Send</ng-container></div>
</button>
<div ngbDropdownMenu aria-labelledby="actionsDropdown" class="shadow">
<button ngbDropdownItem (click)="openShareLinks()" *pngxIfPermissions="{ action: PermissionAction.Add, type: PermissionType.ShareLink }">
<i-bs name="link"></i-bs>&nbsp;<span i18n>Share Links</span>
</button>
@if (emailEnabled) {
<button ngbDropdownItem (click)="openEmailDocument()">
<i-bs name="envelope"></i-bs>&nbsp;<span i18n>Email</span>
</button>
}
</div>
</div>
<pngx-share-links-dropdown [documentId]="documentId" [hasArchiveVersion]="!!document?.archived_file_name" [disabled]="!userCanEdit && !userIsOwner" *pngxIfPermissions="{ action: PermissionAction.Add, type: PermissionType.ShareLink }"></pngx-share-links-dropdown>
</pngx-page-header>
<div class="row">

View File

@@ -1330,18 +1330,4 @@ describe('DocumentDetailComponent', () => {
expect(createSpy).toHaveBeenCalledWith('a')
expect(urlRevokeSpy).toHaveBeenCalled()
})
it('should get email enabled status from settings', () => {
jest.spyOn(settingsService, 'get').mockReturnValue(true)
expect(component.emailEnabled).toBeTruthy()
})
it('should support open share links and email modals', () => {
const modalSpy = jest.spyOn(modalService, 'open')
initNormally()
component.openShareLinks()
expect(modalSpy).toHaveBeenCalled()
component.openEmailDocument()
expect(modalSpy).toHaveBeenCalled()
})
})

View File

@@ -88,7 +88,6 @@ import { CorrespondentEditDialogComponent } from '../common/edit-dialog/correspo
import { DocumentTypeEditDialogComponent } from '../common/edit-dialog/document-type-edit-dialog/document-type-edit-dialog.component'
import { EditDialogMode } from '../common/edit-dialog/edit-dialog.component'
import { StoragePathEditDialogComponent } from '../common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component'
import { EmailDocumentDialogComponent } from '../common/email-document-dialog/email-document-dialog.component'
import { CheckComponent } from '../common/input/check/check.component'
import { DateComponent } from '../common/input/date/date.component'
import { DocumentLinkComponent } from '../common/input/document-link/document-link.component'
@@ -100,7 +99,7 @@ import { TagsComponent } from '../common/input/tags/tags.component'
import { TextComponent } from '../common/input/text/text.component'
import { UrlComponent } from '../common/input/url/url.component'
import { PageHeaderComponent } from '../common/page-header/page-header.component'
import { ShareLinksDialogComponent } from '../common/share-links-dialog/share-links-dialog.component'
import { ShareLinksDropdownComponent } from '../common/share-links-dropdown/share-links-dropdown.component'
import { DocumentHistoryComponent } from '../document-history/document-history.component'
import { DocumentNotesComponent } from '../document-notes/document-notes.component'
import { ComponentWithPermissions } from '../with-permissions/with-permissions.component'
@@ -146,6 +145,7 @@ export enum ZoomSetting {
CustomFieldsDropdownComponent,
DocumentNotesComponent,
DocumentHistoryComponent,
ShareLinksDropdownComponent,
CheckComponent,
DateComponent,
DocumentLinkComponent,
@@ -1426,26 +1426,6 @@ export class DocumentDetailComponent
})
}
public openShareLinks() {
const modal = this.modalService.open(ShareLinksDialogComponent)
modal.componentInstance.documentId = this.document.id
modal.componentInstance.hasArchiveVersion =
!!this.document?.archived_file_name
}
get emailEnabled(): boolean {
return this.settings.get(SETTINGS_KEYS.EMAIL_ENABLED)
}
public openEmailDocument() {
const modal = this.modalService.open(EmailDocumentDialogComponent, {
backdrop: 'static',
})
modal.componentInstance.documentId = this.document.id
modal.componentInstance.hasArchiveVersion =
!!this.document?.archived_file_name
}
private tryRenderTiff() {
this.http.get(this.previewUrl, { responseType: 'arraybuffer' }).subscribe({
next: (res) => {

View File

@@ -84,6 +84,4 @@ export interface MailRule extends ObjectWithPermissions {
assign_correspondent?: number // PaperlessCorrespondent.id
assign_owner_from_rule: boolean
stop_processing: boolean
}

View File

@@ -1,8 +1,15 @@
import { ObjectWithId } from './object-with-id'
export enum PaperlessTaskType {
// just file tasks, for now
File = 'file',
Auto = 'auto_task',
ScheduledTask = 'scheduled_task',
ManualTask = 'manual_task',
}
export enum PaperlessTaskName {
ConsumeFile = 'consume_file',
TrainClassifier = 'train_classifier',
SanityCheck = 'check_sanity',
}
export enum PaperlessTaskStatus {
@@ -23,6 +30,8 @@ export interface PaperlessTask extends ObjectWithId {
task_file_name: string
task_name: PaperlessTaskName
date_created: Date
date_done?: Date

View File

@@ -32,11 +32,16 @@ export interface SystemStatus {
redis_status: SystemStatusItemStatus
redis_error: string
celery_status: SystemStatusItemStatus
celery_url: string
celery_error: string
index_status: SystemStatusItemStatus
index_last_modified: string // ISO date string
index_error: string
classifier_status: SystemStatusItemStatus
classifier_last_trained: string // ISO date string
classifier_error: string
sanity_check_status: SystemStatusItemStatus
sanity_check_last_run: string // ISO date string
sanity_check_error: string
}
}

View File

@@ -355,21 +355,6 @@ it('should include custom fields in sort fields if user has permission', () => {
])
})
it('should call appropriate api endpoint for email document', () => {
subscription = service
.emailDocument(
documents[0].id,
'hello@paperless-ngx.com',
'hello',
'world',
true
)
.subscribe()
httpTestingController.expectOne(
`${environment.apiBaseUrl}${endpoint}/${documents[0].id}/email/`
)
})
afterEach(() => {
subscription?.unsubscribe()
httpTestingController.verify()

View File

@@ -258,19 +258,4 @@ export class DocumentService extends AbstractPaperlessService<Document> {
public get searchQuery(): string {
return this._searchQuery
}
emailDocument(
documentId: number,
addresses: string,
subject: string,
message: string,
useArchiveVersion: boolean
): Observable<any> {
return this.http.post(this.getResourceUrl(documentId, 'email'), {
addresses: addresses,
subject: subject,
message: message,
use_archive_version: useArchiveVersion,
})
}
}

View File

@@ -33,7 +33,6 @@ const mail_rules = [
action: MailAction.MarkRead,
assign_title_from: MailMetadataTitleOption.FromSubject,
assign_owner_from_rule: true,
stop_processing: false,
},
{
name: 'Mail Rule 2',
@@ -53,7 +52,6 @@ const mail_rules = [
action: MailAction.Delete,
assign_title_from: MailMetadataTitleOption.FromSubject,
assign_owner_from_rule: true,
stop_processing: false,
},
{
name: 'Mail Rule 3',
@@ -73,7 +71,6 @@ const mail_rules = [
action: MailAction.Flag,
assign_title_from: MailMetadataTitleOption.FromSubject,
assign_owner_from_rule: false,
stop_processing: false,
},
]

View File

@@ -5,7 +5,11 @@ import {
} from '@angular/common/http/testing'
import { TestBed } from '@angular/core/testing'
import { environment } from 'src/environments/environment'
import { PaperlessTaskStatus, PaperlessTaskType } from '../data/paperless-task'
import {
PaperlessTaskName,
PaperlessTaskStatus,
PaperlessTaskType,
} from '../data/paperless-task'
import { TasksService } from './tasks.service'
describe('TasksService', () => {
@@ -33,7 +37,7 @@ describe('TasksService', () => {
it('calls tasks api endpoint on reload', () => {
tasksService.reload()
const req = httpTestingController.expectOne(
`${environment.apiBaseUrl}tasks/`
`${environment.apiBaseUrl}tasks/?task_name=consume_file&acknowledged=false`
)
expect(req.request.method).toEqual('GET')
})
@@ -41,7 +45,9 @@ describe('TasksService', () => {
it('does not call tasks api endpoint on reload if already loading', () => {
tasksService.loading = true
tasksService.reload()
httpTestingController.expectNone(`${environment.apiBaseUrl}tasks/`)
httpTestingController.expectNone(
`${environment.apiBaseUrl}tasks/?task_name=consume_file&acknowledged=false`
)
})
it('calls acknowledge_tasks api endpoint on dismiss and reloads', () => {
@@ -55,14 +61,19 @@ describe('TasksService', () => {
})
req.flush([])
// reload is then called
httpTestingController.expectOne(`${environment.apiBaseUrl}tasks/`).flush([])
httpTestingController
.expectOne(
`${environment.apiBaseUrl}tasks/?task_name=consume_file&acknowledged=false`
)
.flush([])
})
it('sorts tasks returned from api', () => {
expect(tasksService.total).toEqual(0)
const mockTasks = [
{
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Complete,
acknowledged: false,
task_id: '1234',
@@ -70,7 +81,8 @@ describe('TasksService', () => {
date_created: new Date(),
},
{
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Failed,
acknowledged: false,
task_id: '1235',
@@ -78,7 +90,8 @@ describe('TasksService', () => {
date_created: new Date(),
},
{
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Pending,
acknowledged: false,
task_id: '1236',
@@ -86,7 +99,8 @@ describe('TasksService', () => {
date_created: new Date(),
},
{
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Started,
acknowledged: false,
task_id: '1237',
@@ -94,7 +108,8 @@ describe('TasksService', () => {
date_created: new Date(),
},
{
type: PaperlessTaskType.File,
type: PaperlessTaskType.Auto,
task_name: PaperlessTaskName.ConsumeFile,
status: PaperlessTaskStatus.Complete,
acknowledged: false,
task_id: '1238',
@@ -106,7 +121,7 @@ describe('TasksService', () => {
tasksService.reload()
const req = httpTestingController.expectOne(
`${environment.apiBaseUrl}tasks/`
`${environment.apiBaseUrl}tasks/?task_name=consume_file&acknowledged=false`
)
req.flush(mockTasks)

View File

@@ -4,8 +4,8 @@ import { Subject } from 'rxjs'
import { first, takeUntil } from 'rxjs/operators'
import {
PaperlessTask,
PaperlessTaskName,
PaperlessTaskStatus,
PaperlessTaskType,
} from 'src/app/data/paperless-task'
import { environment } from 'src/environments/environment'
@@ -54,10 +54,14 @@ export class TasksService {
this.loading = true
this.http
.get<PaperlessTask[]>(`${this.baseUrl}tasks/`)
.get<PaperlessTask[]>(
`${this.baseUrl}tasks/?task_name=consume_file&acknowledged=false`
)
.pipe(takeUntil(this.unsubscribeNotifer), first())
.subscribe((r) => {
this.fileTasks = r.filter((t) => t.type == PaperlessTaskType.File) // they're all File tasks, for now
this.fileTasks = r.filter(
(t) => t.task_name == PaperlessTaskName.ConsumeFile
)
this.loading = false
})
}

View File

@@ -112,7 +112,6 @@ import {
questionCircle,
scissors,
search,
send,
slashCircle,
sliders2Vertical,
sortAlphaDown,
@@ -317,7 +316,6 @@ const icons = {
questionCircle,
scissors,
search,
send,
slashCircle,
sliders2Vertical,
sortAlphaDown,

View File

@@ -21,10 +21,12 @@
--pngx-success-darken-10: hsl(152, 69%, 11%); // based on success #198754
--pngx-bg-alt: #fff;
--pngx-bg-darker: var(--bs-gray-100);
--pngx-bg-alt2: var(--bs-gray-200);
--pngx-bg-alt2: var(--bs-gray-200); // #e9ecef
--pngx-bg-disabled: #f7f7f7;
--pngx-focus-alpha: 0.3;
--pngx-toast-max-width: 360px;
--bs-info: var(--pngx-bg-alt2);
--bs-info-rgb: 233, 236, 239;
@media screen and (min-width: 1024px) {
--pngx-toast-max-width: 450px;
}
@@ -71,8 +73,15 @@ $form-check-radio-checked-bg-image-dark: url("data:image/svg+xml,<svg xmlns='htt
}
@mixin dark-mode {
--bs-body-color: #{$text-color-dark-bg};
--pngx-body-color-accent: #{$text-color-dark-bg-accent};
--pngx-bg-alt: #242529;
--pngx-bg-alt2: #232323;
--pngx-bg-darker: #101216;
--pngx-bg-disabled: var(--pngx-bg-alt);
--pngx-focus-alpha: 0.6;
--pngx-primary-faded: var(--pngx-primary-darken-15);
--pngx-primary-text-contrast: var(--bs-body-color);
--bs-body-color: #{$text-color-dark-bg};
--bs-secondary-color: #6c757d;
--bs-danger: #b71631;
--bs-danger-rgb: 183, 22, 49;
@@ -80,15 +89,10 @@ $form-check-radio-checked-bg-image-dark: url("data:image/svg+xml,<svg xmlns='htt
--bs-body-bg-rgb: 22, 22, 24;
--bs-light: #1c1c1f;
--bs-light-rgb: 28, 28, 31;
--bs-info: var(--pngx-bg-alt);
--bs-info-rgb: 36, 36, 39;
--bs-border-color: #47494f;
--pngx-bg-alt2: #232323;
--pngx-bg-darker: #101216;
--bs-tertiary-bg: var(--pngx-bg-darker);
--pngx-bg-alt: #242529;
--pngx-bg-disabled: var(--pngx-bg-alt);
--pngx-focus-alpha: 0.6;
--pngx-primary-faded: var(--pngx-primary-darken-15);
--pngx-primary-text-contrast: var(--bs-body-color);
--bs-dark-border-subtle: var(--pngx-bg-darker);
--bs-border-color-translucent: rgba(0, 0, 0, .175); // override bs

View File

@@ -1,7 +1,6 @@
import logging
import pickle
import re
import time
import warnings
from collections.abc import Iterator
from hashlib import sha256
@@ -142,19 +141,6 @@ class DocumentClassifier:
):
raise IncompatibleClassifierVersionError("sklearn version update")
def set_last_checked(self) -> None:
# save a timestamp of the last time we checked for retraining to a file
with Path(settings.MODEL_FILE.with_suffix(".last_checked")).open("w") as f:
f.write(str(time.time()))
def get_last_checked(self) -> float | None:
# load the timestamp of the last time we checked for retraining
try:
with Path(settings.MODEL_FILE.with_suffix(".last_checked")).open("r") as f:
return float(f.read())
except FileNotFoundError: # pragma: no cover
return None
def save(self) -> None:
target_file: Path = settings.MODEL_FILE
target_file_temp: Path = target_file.with_suffix(".pickle.part")
@@ -175,7 +161,6 @@ class DocumentClassifier:
pickle.dump(self.storage_path_classifier, f)
target_file_temp.rename(target_file)
self.set_last_checked()
def train(self) -> bool:
# Get non-inbox documents
@@ -244,7 +229,6 @@ class DocumentClassifier:
and self.last_doc_change_time >= latest_doc_change
) and self.last_auto_type_hash == hasher.digest():
logger.info("No updates since last training")
self.set_last_checked()
# Set the classifier information into the cache
# Caching for 50 minutes, so slightly less than the normal retrain time
cache.set(

View File

@@ -35,6 +35,7 @@ from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Log
from documents.models import PaperlessTask
from documents.models import ShareLink
from documents.models import StoragePath
from documents.models import Tag
@@ -770,6 +771,21 @@ class ShareLinkFilterSet(FilterSet):
}
class PaperlessTaskFilterSet(FilterSet):
acknowledged = BooleanFilter(
label="Acknowledged",
field_name="acknowledged",
)
class Meta:
model = PaperlessTask
fields = {
"type": ["exact"],
"task_name": ["exact"],
"status": ["exact"],
}
class ObjectOwnedOrGrantedPermissionsFilter(ObjectPermissionsFilter):
"""
A filter backend that limits results to those where the requesting user

View File

@@ -10,4 +10,4 @@ class Command(BaseCommand):
)
def handle(self, *args, **options):
train_classifier()
train_classifier(scheduled=False)

View File

@@ -12,6 +12,6 @@ class Command(ProgressBarMixin, BaseCommand):
def handle(self, *args, **options):
self.handle_progress_bar_mixin(**options)
messages = check_sanity(progress=self.use_progress_bar)
messages = check_sanity(progress=self.use_progress_bar, scheduled=False)
messages.log_messages()

View File

@@ -1,4 +1,4 @@
# Generated by Django 5.1.6 on 2025-02-20 04:55
# Generated by Django 5.1.6 on 2025-02-21 16:34
import multiselectfield.db.fields
from django.db import migrations
@@ -16,12 +16,51 @@ def update_workflow_sources(apps, schema_editor):
trigger.save()
def make_existing_tasks_consume_auto(apps, schema_editor):
PaperlessTask = apps.get_model("documents", "PaperlessTask")
PaperlessTask.objects.all().update(type="auto_task", task_name="consume_file")
class Migration(migrations.Migration):
dependencies = [
("documents", "1062_alter_savedviewfilterrule_rule_type"),
]
operations = [
migrations.AddField(
model_name="paperlesstask",
name="type",
field=models.CharField(
choices=[
("auto_task", "Auto Task"),
("scheduled_task", "Scheduled Task"),
("manual_task", "Manual Task"),
],
default="auto_task",
help_text="The type of task that was run",
max_length=30,
verbose_name="Task Type",
),
),
migrations.AlterField(
model_name="paperlesstask",
name="task_name",
field=models.CharField(
choices=[
("consume_file", "Consume File"),
("train_classifier", "Train Classifier"),
("check_sanity", "Check Sanity"),
],
help_text="Name of the task that was run",
max_length=255,
null=True,
verbose_name="Task Name",
),
),
migrations.RunPython(
code=make_existing_tasks_consume_auto,
reverse_code=migrations.RunPython.noop,
),
migrations.AlterField(
model_name="workflowactionwebhook",
name="url",

View File

@@ -650,6 +650,16 @@ class PaperlessTask(ModelWithOwner):
ALL_STATES = sorted(states.ALL_STATES)
TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES))
class TaskType(models.TextChoices):
AUTO = ("auto_task", _("Auto Task"))
SCHEDULED_TASK = ("scheduled_task", _("Scheduled Task"))
MANUAL_TASK = ("manual_task", _("Manual Task"))
class TaskName(models.TextChoices):
CONSUME_FILE = ("consume_file", _("Consume File"))
TRAIN_CLASSIFIER = ("train_classifier", _("Train Classifier"))
CHECK_SANITY = ("check_sanity", _("Check Sanity"))
task_id = models.CharField(
max_length=255,
unique=True,
@@ -673,8 +683,9 @@ class PaperlessTask(ModelWithOwner):
task_name = models.CharField(
null=True,
max_length=255,
choices=TaskName.choices,
verbose_name=_("Task Name"),
help_text=_("Name of the Task which was run"),
help_text=_("Name of the task that was run"),
)
status = models.CharField(
@@ -684,24 +695,28 @@ class PaperlessTask(ModelWithOwner):
verbose_name=_("Task State"),
help_text=_("Current state of the task being run"),
)
date_created = models.DateTimeField(
null=True,
default=timezone.now,
verbose_name=_("Created DateTime"),
help_text=_("Datetime field when the task result was created in UTC"),
)
date_started = models.DateTimeField(
null=True,
default=None,
verbose_name=_("Started DateTime"),
help_text=_("Datetime field when the task was started in UTC"),
)
date_done = models.DateTimeField(
null=True,
default=None,
verbose_name=_("Completed DateTime"),
help_text=_("Datetime field when the task was completed in UTC"),
)
result = models.TextField(
null=True,
default=None,
@@ -711,6 +726,14 @@ class PaperlessTask(ModelWithOwner):
),
)
type = models.CharField(
max_length=30,
choices=TaskType.choices,
default=TaskType.AUTO,
verbose_name=_("Task Type"),
help_text=_("The type of task that was run"),
)
def __str__(self) -> str:
return f"Task {self.task_id}"

View File

@@ -1,13 +1,17 @@
import hashlib
import logging
import uuid
from collections import defaultdict
from pathlib import Path
from typing import Final
from celery import states
from django.conf import settings
from django.utils import timezone
from tqdm import tqdm
from documents.models import Document
from documents.models import PaperlessTask
class SanityCheckMessages:
@@ -57,7 +61,17 @@ class SanityCheckFailedException(Exception):
pass
def check_sanity(*, progress=False) -> SanityCheckMessages:
def check_sanity(*, progress=False, scheduled=True) -> SanityCheckMessages:
paperless_task = PaperlessTask.objects.create(
task_id=uuid.uuid4(),
type=PaperlessTask.TaskType.SCHEDULED_TASK
if scheduled
else PaperlessTask.TaskType.MANUAL_TASK,
task_name=PaperlessTask.TaskName.CHECK_SANITY,
status=states.STARTED,
date_created=timezone.now(),
date_started=timezone.now(),
)
messages = SanityCheckMessages()
present_files = {
@@ -142,4 +156,11 @@ def check_sanity(*, progress=False) -> SanityCheckMessages:
for extra_file in present_files:
messages.warning(None, f"Orphaned file in media dir: {extra_file}")
paperless_task.status = states.SUCCESS if not messages.has_error else states.FAILURE
# result is concatenated messages
paperless_task.result = f"{len(messages)} issues found."
if messages.has_error:
paperless_task.result += " Check logs for details."
paperless_task.date_done = timezone.now()
paperless_task.save(update_fields=["status", "result", "date_done"])
return messages

View File

@@ -1704,6 +1704,7 @@ class TasksViewSerializer(OwnedObjectSerializer):
fields = (
"id",
"task_id",
"task_name",
"task_file_name",
"date_created",
"date_done",
@@ -1715,12 +1716,6 @@ class TasksViewSerializer(OwnedObjectSerializer):
"owner",
)
type = serializers.SerializerMethodField()
def get_type(self, obj) -> str:
# just file tasks, for now
return "file"
related_document = serializers.SerializerMethodField()
created_doc_re = re.compile(r"New document id (\d+) created")
duplicate_doc_re = re.compile(r"It is a duplicate of .* \(#(\d+)\)")
@@ -1728,20 +1723,21 @@ class TasksViewSerializer(OwnedObjectSerializer):
def get_related_document(self, obj) -> str | None:
result = None
re = None
match obj.status:
case states.SUCCESS:
re = self.created_doc_re
case states.FAILURE:
re = (
self.duplicate_doc_re
if "existing document is in the trash" not in obj.result
else None
)
if re is not None:
try:
result = re.search(obj.result).group(1)
except Exception:
pass
if obj.result:
match obj.status:
case states.SUCCESS:
re = self.created_doc_re
case states.FAILURE:
re = (
self.duplicate_doc_re
if "existing document is in the trash" not in obj.result
else None
)
if re is not None:
try:
result = re.search(obj.result).group(1)
except Exception:
pass
return result

View File

@@ -1221,10 +1221,11 @@ def before_task_publish_handler(sender=None, headers=None, body=None, **kwargs):
user_id = overrides.owner_id if overrides else None
PaperlessTask.objects.create(
type=PaperlessTask.TaskType.AUTO,
task_id=headers["id"],
status=states.PENDING,
task_file_name=task_file_name,
task_name=headers["task"],
task_name=PaperlessTask.TaskName.CONSUME_FILE,
result=None,
date_created=timezone.now(),
date_started=None,

View File

@@ -9,6 +9,7 @@ from tempfile import TemporaryDirectory
import tqdm
from celery import Task
from celery import shared_task
from celery import states
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db import models
@@ -35,6 +36,7 @@ from documents.models import Correspondent
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import PaperlessTask
from documents.models import StoragePath
from documents.models import Tag
from documents.models import Workflow
@@ -74,19 +76,34 @@ def index_reindex(*, progress_bar_disable=False):
@shared_task
def train_classifier():
def train_classifier(*, scheduled=True):
task = PaperlessTask.objects.create(
type=PaperlessTask.TaskType.SCHEDULED_TASK
if scheduled
else PaperlessTask.TaskType.MANUAL_TASK,
task_id=uuid.uuid4(),
task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
status=states.STARTED,
date_created=timezone.now(),
date_started=timezone.now(),
)
if (
not Tag.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
and not DocumentType.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
and not Correspondent.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
and not StoragePath.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
):
logger.info("No automatic matching items, not training")
result = "No automatic matching items, not training"
logger.info(result)
# Special case, items were once auto and trained, so remove the model
# and prevent its use again
if settings.MODEL_FILE.exists():
logger.info(f"Removing {settings.MODEL_FILE} so it won't be used")
settings.MODEL_FILE.unlink()
task.status = states.SUCCESS
task.result = result
task.date_done = timezone.now()
task.save()
return
classifier = load_classifier()
@@ -100,11 +117,19 @@ def train_classifier():
f"Saving updated classifier model to {settings.MODEL_FILE}...",
)
classifier.save()
task.result = "Training completed successfully"
else:
logger.debug("Training data unchanged.")
task.result = "Training data unchanged"
task.status = states.SUCCESS
task.date_done = timezone.now()
task.save(update_fields=["status", "result", "date_done"])
except Exception as e:
logger.warning("Classifier error: " + str(e))
task.status = states.FAILURE
task.result = str(e)
@shared_task(bind=True)

View File

@@ -15,7 +15,6 @@ from dateutil import parser
from django.conf import settings
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from django.core import mail
from django.core.cache import cache
from django.db import DataError
from django.test import override_settings
@@ -2652,153 +2651,6 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(doc1.tags.count(), 2)
@override_settings(
EMAIL_ENABLED=True,
EMAIL_BACKEND="django.core.mail.backends.locmem.EmailBackend",
)
def test_email_document(self):
"""
GIVEN:
- Existing document
WHEN:
- API request is made to email document action
THEN:
- Email is sent, with document (original or archive) attached
"""
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="this is a document 1",
checksum="1",
filename="test.pdf",
archive_checksum="A",
archive_filename="archive.pdf",
)
doc2 = Document.objects.create(
title="test2",
mime_type="application/pdf",
content="this is a document 2",
checksum="2",
filename="test2.pdf",
)
archive_file = Path(__file__).parent / "samples" / "simple.pdf"
source_file = Path(__file__).parent / "samples" / "simple.pdf"
shutil.copy(archive_file, doc.archive_path)
shutil.copy(source_file, doc2.source_path)
self.client.post(
f"/api/documents/{doc.pk}/email/",
{
"addresses": "hello@paperless-ngx.com",
"subject": "test",
"message": "hello",
},
)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].attachments[0][0], "archive.pdf")
self.client.post(
f"/api/documents/{doc2.pk}/email/",
{
"addresses": "hello@paperless-ngx.com",
"subject": "test",
"message": "hello",
"use_archive_version": False,
},
)
self.assertEqual(len(mail.outbox), 2)
self.assertEqual(mail.outbox[1].attachments[0][0], "test2.pdf")
@mock.patch("django.core.mail.message.EmailMessage.send", side_effect=Exception)
def test_email_document_errors(self, mocked_send):
"""
GIVEN:
- Existing document
WHEN:
- API request is made to email document action with insufficient permissions
- API request is made to email document action with invalid document id
- API request is made to email document action with missing data
- API request is made to email document action with invalid email address
- API request is made to email document action and error occurs during email send
THEN:
- Error response is returned
"""
user1 = User.objects.create_user(username="test1")
user1.user_permissions.add(*Permission.objects.all())
user1.save()
doc = Document.objects.create(
title="test",
mime_type="application/pdf",
content="this is a document 1",
checksum="1",
filename="test.pdf",
archive_checksum="A",
archive_filename="archive.pdf",
)
doc2 = Document.objects.create(
title="test2",
mime_type="application/pdf",
content="this is a document 2",
checksum="2",
owner=self.user,
)
self.client.force_authenticate(user1)
resp = self.client.post(
f"/api/documents/{doc2.pk}/email/",
{
"addresses": "hello@paperless-ngx.com",
"subject": "test",
"message": "hello",
},
)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
resp = self.client.post(
"/api/documents/999/email/",
{
"addresses": "hello@paperless-ngx.com",
"subject": "test",
"message": "hello",
},
)
self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
resp = self.client.post(
f"/api/documents/{doc.pk}/email/",
{
"addresses": "hello@paperless-ngx.com",
},
)
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
resp = self.client.post(
f"/api/documents/{doc.pk}/email/",
{
"addresses": "hello@paperless-ngx.com,hello",
"subject": "test",
"message": "hello",
},
)
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
resp = self.client.post(
f"/api/documents/{doc.pk}/email/",
{
"addresses": "hello@paperless-ngx.com",
"subject": "test",
"message": "hello",
},
)
self.assertEqual(resp.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
@mock.patch("django_softdelete.models.SoftDeleteModel.delete")
def test_warn_on_delete_with_old_uuid_field(self, mocked_delete):
"""

View File

@@ -1,18 +1,14 @@
import os
import tempfile
from pathlib import Path
from unittest import mock
from celery import states
from django.contrib.auth.models import User
from django.test import override_settings
from rest_framework import status
from rest_framework.test import APITestCase
from documents.classifier import ClassifierModelCorruptError
from documents.classifier import DocumentClassifier
from documents.classifier import load_classifier
from documents.models import Document
from documents.models import Tag
from documents.models import PaperlessTask
from paperless import version
@@ -193,7 +189,6 @@ class TestSystemStatus(APITestCase):
self.assertEqual(response.data["tasks"]["index_status"], "ERROR")
self.assertIsNotNone(response.data["tasks"]["index_error"])
@override_settings(DATA_DIR=Path("/tmp/does_not_exist/data/"))
def test_system_status_classifier_ok(self):
"""
GIVEN:
@@ -203,9 +198,11 @@ class TestSystemStatus(APITestCase):
THEN:
- The response contains an OK classifier status
"""
load_classifier()
test_classifier = DocumentClassifier()
test_classifier.save()
PaperlessTask.objects.create(
type=PaperlessTask.TaskType.SCHEDULED_TASK,
status=states.SUCCESS,
task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
)
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -215,73 +212,101 @@ class TestSystemStatus(APITestCase):
def test_system_status_classifier_warning(self):
"""
GIVEN:
- The classifier does not exist yet
- > 0 documents and tags with auto matching exist
- No classifier task is found
WHEN:
- The user requests the system status
THEN:
- The response contains an WARNING classifier status
- The response contains a WARNING classifier status
"""
with override_settings(MODEL_FILE=Path("does_not_exist")):
Document.objects.create(
title="Test Document",
)
Tag.objects.create(name="Test Tag", matching_algorithm=Tag.MATCH_AUTO)
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["tasks"]["classifier_status"], "WARNING")
self.assertIsNotNone(response.data["tasks"]["classifier_error"])
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
response.data["tasks"]["classifier_status"],
"WARNING",
)
@mock.patch(
"documents.classifier.load_classifier",
side_effect=ClassifierModelCorruptError(),
)
def test_system_status_classifier_error(self, mock_load_classifier):
def test_system_status_classifier_error(self):
"""
GIVEN:
- The classifier does exist but is corrupt
- > 0 documents and tags with auto matching exist
- An error occurred while loading the classifier
WHEN:
- The user requests the system status
THEN:
- The response contains an ERROR classifier status
"""
with (
tempfile.NamedTemporaryFile(
dir="/tmp",
delete=False,
) as does_exist,
override_settings(MODEL_FILE=Path(does_exist.name)),
):
Document.objects.create(
title="Test Document",
)
Tag.objects.create(
name="Test Tag",
matching_algorithm=Tag.MATCH_AUTO,
)
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
response.data["tasks"]["classifier_status"],
"ERROR",
)
self.assertIsNotNone(response.data["tasks"]["classifier_error"])
PaperlessTask.objects.create(
type=PaperlessTask.TaskType.SCHEDULED_TASK,
status=states.FAILURE,
task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
result="Classifier training failed",
)
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
response.data["tasks"]["classifier_status"],
"ERROR",
)
self.assertIsNotNone(response.data["tasks"]["classifier_error"])
def test_system_status_classifier_ok_no_objects(self):
def test_system_status_sanity_check_ok(self):
"""
GIVEN:
- The classifier does not exist (and should not)
- No documents nor objects with auto matching exist
- The sanity check is successful
WHEN:
- The user requests the system status
THEN:
- The response contains an OK classifier status
- The response contains an OK sanity check status
"""
with override_settings(MODEL_FILE=Path("does_not_exist")):
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["tasks"]["classifier_status"], "OK")
PaperlessTask.objects.create(
type=PaperlessTask.TaskType.SCHEDULED_TASK,
status=states.SUCCESS,
task_name=PaperlessTask.TaskName.CHECK_SANITY,
)
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["tasks"]["sanity_check_status"], "OK")
self.assertIsNone(response.data["tasks"]["sanity_check_error"])
def test_system_status_sanity_check_warning(self):
"""
GIVEN:
- No sanity check task is found
WHEN:
- The user requests the system status
THEN:
- The response contains a WARNING sanity check status
"""
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
response.data["tasks"]["sanity_check_status"],
"WARNING",
)
def test_system_status_sanity_check_error(self):
"""
GIVEN:
- The sanity check failed
WHEN:
- The user requests the system status
THEN:
- The response contains an ERROR sanity check status
"""
PaperlessTask.objects.create(
type=PaperlessTask.TaskType.SCHEDULED_TASK,
status=states.FAILURE,
task_name=PaperlessTask.TaskName.CHECK_SANITY,
result="5 issues found.",
)
self.client.force_login(self.user)
response = self.client.get(self.ENDPOINT)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
response.data["tasks"]["sanity_check_status"],
"ERROR",
)
self.assertIsNotNone(response.data["tasks"]["sanity_check_error"])

View File

@@ -130,7 +130,7 @@ class TestTasks(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
response = self.client.get(self.ENDPOINT)
response = self.client.get(self.ENDPOINT + "?acknowledged=false")
self.assertEqual(len(response.data), 0)
def test_tasks_owner_aware(self):
@@ -246,7 +246,7 @@ class TestTasks(DirectoriesMixin, APITestCase):
PaperlessTask.objects.create(
task_id=str(uuid.uuid4()),
task_file_name="test.pdf",
task_name="documents.tasks.some_task",
task_name=PaperlessTask.TaskName.CONSUME_FILE,
status=celery.states.SUCCESS,
)
@@ -272,7 +272,7 @@ class TestTasks(DirectoriesMixin, APITestCase):
PaperlessTask.objects.create(
task_id=str(uuid.uuid4()),
task_file_name="anothertest.pdf",
task_name="documents.tasks.some_task",
task_name=PaperlessTask.TaskName.CONSUME_FILE,
status=celery.states.SUCCESS,
)

View File

@@ -8,7 +8,7 @@ class TestMigrateWorkflow(TestMigrations):
dependencies = (
(
"paperless_mail",
"0030_mailrule_stop_processing",
"0029_mailrule_pdf_layout",
),
)

View File

@@ -68,7 +68,7 @@ class TestTaskSignalHandler(DirectoriesMixin, TestCase):
self.assertIsNotNone(task)
self.assertEqual(headers["id"], task.task_id)
self.assertEqual("hello-999.pdf", task.task_file_name)
self.assertEqual("documents.tasks.consume_file", task.task_name)
self.assertEqual(PaperlessTask.TaskName.CONSUME_FILE, task.task_name)
self.assertEqual(1, task.owner_id)
self.assertEqual(celery.states.PENDING, task.status)

View File

@@ -15,6 +15,7 @@ from urllib.parse import quote
from urllib.parse import urlparse
import pathvalidate
from celery import states
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
@@ -37,7 +38,6 @@ from django.http import HttpResponse
from django.http import HttpResponseBadRequest
from django.http import HttpResponseForbidden
from django.http import HttpResponseRedirect
from django.http import HttpResponseServerError
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.decorators import method_decorator
@@ -104,10 +104,10 @@ from documents.filters import DocumentsOrderingFilter
from documents.filters import DocumentTypeFilterSet
from documents.filters import ObjectOwnedOrGrantedPermissionsFilter
from documents.filters import ObjectOwnedPermissionsFilter
from documents.filters import PaperlessTaskFilterSet
from documents.filters import ShareLinkFilterSet
from documents.filters import StoragePathFilterSet
from documents.filters import TagFilterSet
from documents.mail import send_email
from documents.matching import match_correspondents
from documents.matching import match_document_types
from documents.matching import match_storage_paths
@@ -1025,57 +1025,6 @@ class DocumentViewSet(
return Response(sorted(entries, key=lambda x: x["timestamp"], reverse=True))
@action(methods=["post"], detail=True)
def email(self, request, pk=None):
try:
doc = Document.objects.select_related("owner").get(pk=pk)
if request.user is not None and not has_perms_owner_aware(
request.user,
"view_document",
doc,
):
return HttpResponseForbidden("Insufficient permissions")
except Document.DoesNotExist:
raise Http404
try:
if (
"addresses" not in request.data
or "subject" not in request.data
or "message" not in request.data
):
return HttpResponseBadRequest("Missing required fields")
use_archive_version = request.data.get("use_archive_version", True)
addresses = request.data.get("addresses").split(",")
if not all(
re.match(r"[^@]+@[^@]+\.[^@]+", address.strip())
for address in addresses
):
return HttpResponseBadRequest("Invalid email address found")
send_email(
subject=request.data.get("subject"),
body=request.data.get("message"),
to=addresses,
attachment=(
doc.archive_path
if use_archive_version and doc.has_archive_version
else doc.source_path
),
attachment_mime_type=doc.mime_type,
)
logger.debug(
f"Sent document {doc.id} via email to {addresses}",
)
return Response({"message": "Email sent"})
except Exception as e:
logger.warning(f"An error occurred emailing document: {e!s}")
return HttpResponseServerError(
"Error emailing document, check logs for more detail.",
)
@extend_schema_view(
list=extend_schema(
@@ -2277,16 +2226,15 @@ class RemoteVersionView(GenericAPIView):
class TasksViewSet(ReadOnlyModelViewSet):
permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
serializer_class = TasksViewSerializer
filter_backends = (ObjectOwnedOrGrantedPermissionsFilter,)
filter_backends = (
DjangoFilterBackend,
OrderingFilter,
ObjectOwnedOrGrantedPermissionsFilter,
)
filterset_class = PaperlessTaskFilterSet
def get_queryset(self):
queryset = (
PaperlessTask.objects.filter(
acknowledged=False,
)
.order_by("date_created")
.reverse()
)
queryset = PaperlessTask.objects.all().order_by("-date_created")
task_id = self.request.query_params.get("task_id")
if task_id is not None:
queryset = PaperlessTask.objects.filter(task_id=task_id)
@@ -2615,6 +2563,14 @@ class CustomFieldViewSet(ModelViewSet):
"last_trained": serializers.DateTimeField(),
},
),
"sanity_check": inline_serializer(
name="SanityCheck",
fields={
"status": serializers.CharField(),
"error": serializers.CharField(),
"last_run": serializers.DateTimeField(),
},
),
},
),
},
@@ -2623,6 +2579,17 @@ class CustomFieldViewSet(ModelViewSet):
class SystemStatusView(PassUserMixin):
permission_classes = (IsAuthenticated,)
def _get_next_scheduled_task_schedule(
self,
schedule: dict,
task_name: str,
last_run,
) -> datetime | None:
# example: {'Check all e-mail accounts': {'task': 'paperless_mail.tasks.process_mail_accounts', 'schedule': <crontab: */10 * * * * (m/h/dM/MY/d)>, 'options': {'expires': 540.0}}, 'Train the classifier': {'task': 'documents.tasks.train_classifier', 'schedule': <crontab: 5 */1 * * * (m/h/dM/MY/d)>, 'options': {'expires': 3540.0}}, 'Optimize the index': {'task': 'documents.tasks.index_optimize', 'schedule': <crontab: 0 0 * * * (m/h/dM/MY/d)>, 'options': {'expires': 82800.0}}, 'Perform sanity check': {'task': 'documents.tasks.sanity_check', 'schedule': <crontab: 30 0 * * sun (m/h/dM/MY/d)>, 'options': {'expires': 601200.0}}, 'Empty trash': {'task': 'documents.tasks.empty_trash', 'schedule': <crontab: 0 1 * * * (m/h/dM/MY/d)>, 'options': {'expires': 82800.0}}, 'Check and run scheduled workflows': {'task': 'documents.tasks.check_scheduled_workflows', 'schedule': <crontab: 5 */1 * * * (m/h/dM/MY/d)>, 'options': {'expires': 3540.0}}}
for _, task_data in schedule.items():
if task_data["task"] and task_data["task"].find(task_name) != -1:
return task_data["schedule"]
def get(self, request, format=None):
if not request.user.is_staff:
return HttpResponseForbidden("Insufficient permissions")
@@ -2675,13 +2642,22 @@ class SystemStatusView(PassUserMixin):
)
redis_error = "Error connecting to redis, check logs for more detail."
celery_error = None
celery_url = None
schedule = None
try:
celery_ping = celery_app.control.inspect().ping()
first_worker_ping = celery_ping[next(iter(celery_ping.keys()))]
celery_url = next(iter(celery_ping.keys()))
first_worker_ping = celery_ping[celery_url]
schedule = celery_app.conf.beat_schedule
if first_worker_ping["ok"] == "pong":
celery_active = "OK"
except Exception:
except Exception as e:
celery_active = "ERROR"
logger.exception(
f"System status detected a possible problem while connecting to celery: {e}",
)
celery_error = "Error connecting to celery, check logs for more detail."
index_error = None
try:
@@ -2698,54 +2674,72 @@ class SystemStatusView(PassUserMixin):
)
index_last_modified = None
classifier_error = None
classifier_status = None
try:
classifier = load_classifier(raise_exception=True)
if classifier is None:
# Make sure classifier should exist
docs_queryset = Document.objects.exclude(
tags__is_inbox_tag=True,
)
if (
docs_queryset.count() > 0
and (
Tag.objects.filter(matching_algorithm=Tag.MATCH_AUTO).exists()
or DocumentType.objects.filter(
matching_algorithm=Tag.MATCH_AUTO,
).exists()
or Correspondent.objects.filter(
matching_algorithm=Tag.MATCH_AUTO,
).exists()
or StoragePath.objects.filter(
matching_algorithm=Tag.MATCH_AUTO,
).exists()
)
and not settings.MODEL_FILE.exists()
):
# if classifier file doesn't exist just classify as a warning
classifier_error = "Classifier file does not exist (yet). Re-training may be pending."
classifier_status = "WARNING"
raise FileNotFoundError(classifier_error)
classifier_status = "OK"
classifier_last_trained = (
make_aware(
datetime.fromtimestamp(classifier.get_last_checked()),
)
if settings.MODEL_FILE.exists()
and classifier.get_last_checked() is not None
else None
last_trained_task = (
PaperlessTask.objects.filter(
task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
)
except Exception as e:
if classifier_status is None:
classifier_status = "ERROR"
classifier_last_trained = None
if classifier_error is None:
classifier_error = (
"Unable to load classifier, check logs for more detail."
)
logger.exception(
f"System status detected a possible problem while loading the classifier: {e}",
.order_by("-date_done")
.first()
)
classifier_status = "OK"
classifier_error = None
classifier_next_training = None
if last_trained_task is None:
classifier_status = "WARNING"
classifier_error = "No classifier training tasks found"
elif last_trained_task and last_trained_task.status == states.FAILURE:
classifier_status = "ERROR"
classifier_error = last_trained_task.result
classifier_last_trained = (
last_trained_task.date_done if last_trained_task else None
)
last_scheduled_trained_task = (
PaperlessTask.objects.filter(
task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
type=PaperlessTask.TaskType.SCHEDULED_TASK,
)
.order_by("-date_done")
.first()
)
if last_scheduled_trained_task and schedule:
classifier_next_training: datetime = self._get_next_scheduled_task_schedule(
schedule=schedule,
task_name=PaperlessTask.TaskName.TRAIN_CLASSIFIER,
last_run=last_trained_task.date_done,
)
last_sanity_check = (
PaperlessTask.objects.filter(
task_name=PaperlessTask.TaskName.CHECK_SANITY,
)
.order_by("-date_done")
.first()
)
sanity_check_status = "OK"
sanity_check_error = None
sanity_check_next_run = None
if last_sanity_check is None:
sanity_check_status = "WARNING"
sanity_check_error = "No sanity check tasks found"
elif last_sanity_check and last_sanity_check.status == states.FAILURE:
sanity_check_status = "ERROR"
sanity_check_error = last_sanity_check.result
sanity_check_last_run = (
last_sanity_check.date_done if last_sanity_check else None
)
last_scheduled_sanity_check = (
PaperlessTask.objects.filter(
task_name=PaperlessTask.TaskName.CHECK_SANITY,
type=PaperlessTask.TaskType.SCHEDULED_TASK,
)
.order_by("-date_done")
.first()
)
if last_scheduled_sanity_check and schedule:
sanity_check_next_run: datetime = self._get_next_scheduled_task_schedule(
schedule=schedule,
task_name=PaperlessTask.TaskName.CHECK_SANITY,
last_run=last_sanity_check.date_done,
)
return Response(
@@ -2774,12 +2768,19 @@ class SystemStatusView(PassUserMixin):
"redis_status": redis_status,
"redis_error": redis_error,
"celery_status": celery_active,
"celery_url": celery_url,
"celery_error": celery_error,
"index_status": index_status,
"index_last_modified": index_last_modified,
"index_error": index_error,
"classifier_status": classifier_status,
"classifier_last_trained": classifier_last_trained,
"classifier_next_training": classifier_next_training,
"classifier_error": classifier_error,
"sanity_check_status": sanity_check_status,
"sanity_check_last_run": sanity_check_last_run,
"sanity_check_next_run": sanity_check_next_run,
"sanity_check_error": sanity_check_error,
},
},
)

File diff suppressed because it is too large Load Diff

View File

@@ -571,11 +571,6 @@ class MailAccountHandler(LoggingMixin):
rule,
supports_gmail_labels=supports_gmail_labels,
)
if total_processed_files > 0 and rule.stop_processing:
self.log.debug(
f"Rule {rule}: Stopping processing rules due to stop_processing flag",
)
break
except Exception as e:
self.log.exception(
f"Rule {rule}: Error while processing rule: {e}",

View File

@@ -1,22 +0,0 @@
# Generated by Django 5.1.6 on 2025-02-24 16:07
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("paperless_mail", "0029_mailrule_pdf_layout"),
]
operations = [
migrations.AddField(
model_name="mailrule",
name="stop_processing",
field=models.BooleanField(
default=False,
help_text="If True, no further rules will be processed after this one if any document is consumed.",
verbose_name="Stop processing further rules",
),
),
]

View File

@@ -301,14 +301,6 @@ class MailRule(document_models.ModelWithOwner):
default=True,
)
stop_processing = models.BooleanField(
_("Stop processing further rules"),
default=False,
help_text=_(
"If True, no further rules will be processed after this one if any document is queued.",
),
)
def __str__(self):
return f"{self.account.name}.{self.name}"

View File

@@ -101,7 +101,6 @@ class MailRuleSerializer(OwnedObjectSerializer):
"user_can_change",
"permissions",
"set_permissions",
"stop_processing",
]
def update(self, instance, validated_data):

View File

@@ -1671,39 +1671,6 @@ class TestTasks(TestCase):
result = tasks.process_mail_accounts(account_ids=[account_b.id])
self.assertIn("No new", result)
@mock.patch("paperless_mail.tasks.MailAccountHandler.handle_mail_account")
def test_rule_with_stop_processing(self, m):
"""
GIVEN:
- Mail account with a rule with stop_processing=True
WHEN:
- Mail account is processed
THEN:
- Should only process the first rule
"""
m.side_effect = lambda account: 6
account = MailAccount.objects.create(
name="A",
imap_server="A",
username="A",
password="A",
)
MailRule.objects.create(
name="A",
account=account,
stop_processing=True,
)
MailRule.objects.create(
name="B",
account=account,
)
result = tasks.process_mail_accounts()
self.assertEqual(m.call_count, 1)
self.assertIn("Added 6", result)
class TestMailAccountTestView(APITestCase):
def setUp(self):