From 1cdd8d9ba8900dcd54bc7b6595f1ee8f25f12579 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 21 Sep 2025 16:32:21 -0700 Subject: [PATCH 1/4] Clarify repo maintenance rules --- .github/workflows/repo-maintenance.yml | 1 + CONTRIBUTING.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/repo-maintenance.yml b/.github/workflows/repo-maintenance.yml index 0cfc57938..ae473da37 100644 --- a/.github/workflows/repo-maintenance.yml +++ b/.github/workflows/repo-maintenance.yml @@ -241,6 +241,7 @@ jobs: ) { nodes { id, + createdAt, number, updatedAt, upvoteCount, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 24f563c7a..20d7bfd36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -135,7 +135,7 @@ community members. That said, in an effort to keep the repository organized and - Issues, pull requests and discussions that are closed will be locked after 30 days of inactivity. - Discussions with a marked answer will be automatically closed. - Discussions in the 'General' or 'Support' categories will be closed after 180 days of inactivity. -- Feature requests that do not meet the following thresholds will be closed: 180 days of inactivity, < 5 "up-votes" after 180 days, < 20 "up-votes" after 1 year or < 80 "up-votes" at 2 years. +- Feature requests that do not meet the following thresholds will be closed: 180 days of inactivity with less than 80 "up-votes", < 5 "up-votes" after 180 days, < 20 "up-votes" after 1 year or < 40 "up-votes" at 2 years. In all cases, threads can be re-opened by project maintainers and, of course, users can always create a new discussion for related concerns. Finally, remember that all information remains searchable and 'closed' feature requests can still serve as inspiration for new features. From 19a54b3b23d3e7a8e00f322562a0b50f6b99d056 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 22 Sep 2025 11:17:42 -0700 Subject: [PATCH 2/4] Feature: processed mail UI (#10866) --- docs/usage.md | 4 + .../manage/mail/mail.component.html | 22 +- .../manage/mail/mail.component.spec.ts | 9 + .../components/manage/mail/mail.component.ts | 9 + .../processed-mail-dialog.component.html | 107 +++++++ .../processed-mail-dialog.component.scss | 8 + .../processed-mail-dialog.component.spec.ts | 150 +++++++++ .../processed-mail-dialog.component.ts | 96 ++++++ src-ui/src/app/data/processed-mail.ts | 12 + .../src/app/services/permissions.service.ts | 1 + .../rest/processed-mail.service.spec.ts | 39 +++ .../services/rest/processed-mail.service.ts | 19 ++ src-ui/src/main.ts | 4 + src/paperless/urls.py | 2 + src/paperless_mail/filters.py | 12 + src/paperless_mail/serialisers.py | 18 ++ src/paperless_mail/tests/test_api.py | 284 ++++++++++++++++++ src/paperless_mail/views.py | 36 +++ 18 files changed, 824 insertions(+), 8 deletions(-) create mode 100644 src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html create mode 100644 src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.scss create mode 100644 src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.spec.ts create mode 100644 src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.ts create mode 100644 src-ui/src/app/data/processed-mail.ts create mode 100644 src-ui/src/app/services/rest/processed-mail.service.spec.ts create mode 100644 src-ui/src/app/services/rest/processed-mail.service.ts create mode 100644 src/paperless_mail/filters.py diff --git a/docs/usage.md b/docs/usage.md index 94ef5ae1b..32441862d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -261,6 +261,10 @@ different means. These are as follows: Paperless is set up to check your mails every 10 minutes. This can be configured via [`PAPERLESS_EMAIL_TASK_CRON`](configuration.md#PAPERLESS_EMAIL_TASK_CRON) +#### Processed Mail + +Paperless keeps track of emails it has processed in order to avoid processing the same mail multiple times. This uses the message `UID` provided by the mail server, which should be unique for each message. You can view and manage processed mails from the web UI under Mail > Processed Mails. If you need to re-process a message, you can delete the corresponding processed mail entry, which will allow Paperless-ngx to process the email again the next time the mail fetch task runs. + #### OAuth Email Setup Paperless-ngx supports OAuth2 authentication for Gmail and Outlook email accounts. To set up an email account with OAuth2, you will need to create a 'developer' app with the respective provider and obtain the client ID and client secret and set the appropriate [configuration variables](configuration.md#email_oauth). You will also need to set either [`PAPERLESS_OAUTH_CALLBACK_BASE_URL`](configuration.md#PAPERLESS_OAUTH_CALLBACK_BASE_URL) or [`PAPERLESS_URL`](configuration.md#PAPERLESS_URL) to the correct value for the OAuth2 flow to work correctly. diff --git a/src-ui/src/app/components/manage/mail/mail.component.html b/src-ui/src/app/components/manage/mail/mail.component.html index 16e8e88fb..97b2bf507 100644 --- a/src-ui/src/app/components/manage/mail/mail.component.html +++ b/src-ui/src/app/components/manage/mail/mail.component.html @@ -109,10 +109,11 @@
  • Name
    -
    Sort Order
    -
    Account
    -
    Status
    -
    Actions
    +
    Sort Order
    +
    Account
    +
    Status
    +
    Processed Mail
    +
    Actions
    • @@ -127,9 +128,9 @@
  • -
    {{rule.order}}
    -
    {{(mailAccountService.getCached(rule.account) | async)?.name}}
    -
    +
    {{rule.order}}
    +
    {{(mailAccountService.getCached(rule.account) | async)?.name}}
    +
    -
    +
    + +
    +
    + + Read more + + + +
    +
    + @if (loading) { +
    + +
    + } @else if (processedMails.length === 0) { + No processed email messages found. + } @else { +
    + + + + + + + + + + + + + @for (mail of processedMails; track mail.id) { + +
    + {{mail.status}} +
    +     + + + + + + + + + } + +
    +
    + + +
    +     		SubjectReceivedProcessedStatusError
    +
    + + +
    +     		{{ mail.subject }}{{ mail.received | customDate:'longDate' }}{{ mail.processed | customDate:'longDate' }} + @switch (mail.status) { + @case ('SUCCESS') { + + } + @case ('FAILED') { + + } + @default { + + } + } + + + 
    +                    {{ mail.error }}
+
    +     + @if (mail.error) { + {{ mail.error | slice:0:20 }} + } +
    +
    +
    + + + +
    + + +
    +
    + } +
    diff --git a/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.scss b/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.scss new file mode 100644 index 000000000..6aadd8330 --- /dev/null +++ b/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.scss @@ -0,0 +1,8 @@ +::ng-deep .popover { + max-width: 350px; + + pre { + white-space: pre-wrap; + word-break: break-word; + } +} diff --git a/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.spec.ts b/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.spec.ts new file mode 100644 index 000000000..c34c97ef2 --- /dev/null +++ b/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.spec.ts @@ -0,0 +1,150 @@ +import { DatePipe } from '@angular/common' +import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http' +import { + HttpTestingController, + provideHttpClientTesting, +} from '@angular/common/http/testing' +import { ComponentFixture, TestBed } from '@angular/core/testing' +import { FormsModule } from '@angular/forms' +import { By } from '@angular/platform-browser' +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' +import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' +import { ToastService } from 'src/app/services/toast.service' +import { environment } from 'src/environments/environment' +import { ProcessedMailDialogComponent } from './processed-mail-dialog.component' + +describe('ProcessedMailDialogComponent', () => { + let component: ProcessedMailDialogComponent + let fixture: ComponentFixture + let httpTestingController: HttpTestingController + let toastService: ToastService + + const rule: any = { id: 10, name: 'Mail Rule' } // minimal rule object for tests + const mails = [ + { + id: 1, + rule: rule.id, + folder: 'INBOX', + uid: 111, + subject: 'A', + received: new Date().toISOString(), + processed: new Date().toISOString(), + status: 'SUCCESS', + error: null, + }, + { + id: 2, + rule: rule.id, + folder: 'INBOX', + uid: 222, + subject: 'B', + received: new Date().toISOString(), + processed: new Date().toISOString(), + status: 'FAILED', + error: 'Oops', + }, + ] + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [ + ProcessedMailDialogComponent, + FormsModule, + NgxBootstrapIconsModule.pick(allIcons), + ], + providers: [ + DatePipe, + NgbActiveModal, + provideHttpClient(withInterceptorsFromDi()), + provideHttpClientTesting(), + ], + }).compileComponents() + + httpTestingController = TestBed.inject(HttpTestingController) + toastService = TestBed.inject(ToastService) + fixture = TestBed.createComponent(ProcessedMailDialogComponent) + component = fixture.componentInstance + component.rule = rule + }) + + afterEach(() => { + httpTestingController.verify() + }) + + function expectListRequest(ruleId: number) { + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}processed_mail/?page=1&page_size=50&ordering=-processed_at&rule=${ruleId}` + ) + expect(req.request.method).toEqual('GET') + return req + } + + it('should load processed mails on init', () => { + fixture.detectChanges() + const req = expectListRequest(rule.id) + req.flush({ count: 2, results: mails }) + expect(component.loading).toBeFalsy() + expect(component.processedMails).toEqual(mails) + }) + + it('should delete selected mails and reload', () => { + fixture.detectChanges() + // initial load + const initialReq = expectListRequest(rule.id) + initialReq.flush({ count: 0, results: [] }) + + // select a couple of mails and delete + component.selectedMailIds.add(5) + component.selectedMailIds.add(6) + const toastInfoSpy = jest.spyOn(toastService, 'showInfo') + component.deleteSelected() + + const delReq = httpTestingController.expectOne( + `${environment.apiBaseUrl}processed_mail/bulk_delete/` + ) + expect(delReq.request.method).toEqual('POST') + expect(delReq.request.body).toEqual({ mail_ids: [5, 6] }) + delReq.flush({}) + + // reload after delete + const reloadReq = expectListRequest(rule.id) + reloadReq.flush({ count: 0, results: [] }) + expect(toastInfoSpy).toHaveBeenCalled() + }) + + it('should toggle all, toggle selected, and clear selection', () => { + fixture.detectChanges() + // initial load with two mails + const req = expectListRequest(rule.id) + req.flush({ count: 2, results: mails }) + fixture.detectChanges() + + // toggle all via header checkbox + const inputs = fixture.debugElement.queryAll( + By.css('input.form-check-input') + ) + const header = inputs[0].nativeElement as HTMLInputElement + header.dispatchEvent(new Event('click')) + header.checked = true + header.dispatchEvent(new Event('click')) + expect(component.selectedMailIds.size).toEqual(mails.length) + + // toggle a single mail + component.toggleSelected(mails[0] as any) + expect(component.selectedMailIds.has(mails[0].id)).toBeFalsy() + component.toggleSelected(mails[0] as any) + expect(component.selectedMailIds.has(mails[0].id)).toBeTruthy() + + // clear selection + component.clearSelection() + expect(component.selectedMailIds.size).toEqual(0) + expect(component.toggleAllEnabled).toBeFalsy() + }) + + it('should close the dialog', () => { + const activeModal = TestBed.inject(NgbActiveModal) + const closeSpy = jest.spyOn(activeModal, 'close') + component.close() + expect(closeSpy).toHaveBeenCalled() + }) +}) diff --git a/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.ts b/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.ts new file mode 100644 index 000000000..ed51ad0ed --- /dev/null +++ b/src-ui/src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.ts @@ -0,0 +1,96 @@ +import { SlicePipe } from '@angular/common' +import { Component, inject, Input, OnInit } from '@angular/core' +import { FormsModule, ReactiveFormsModule } from '@angular/forms' +import { + NgbActiveModal, + NgbPagination, + NgbPopoverModule, + NgbTooltipModule, +} from '@ng-bootstrap/ng-bootstrap' +import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons' +import { ConfirmButtonComponent } from 'src/app/components/common/confirm-button/confirm-button.component' +import { MailRule } from 'src/app/data/mail-rule' +import { ProcessedMail } from 'src/app/data/processed-mail' +import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe' +import { ProcessedMailService } from 'src/app/services/rest/processed-mail.service' +import { ToastService } from 'src/app/services/toast.service' + +@Component({ + selector: 'pngx-processed-mail-dialog', + imports: [ + ConfirmButtonComponent, + CustomDatePipe, + NgbPagination, + NgbPopoverModule, + NgbTooltipModule, + NgxBootstrapIconsModule, + FormsModule, + ReactiveFormsModule, + SlicePipe, + ], + templateUrl: './processed-mail-dialog.component.html', + styleUrl: './processed-mail-dialog.component.scss', +}) +export class ProcessedMailDialogComponent implements OnInit { + private readonly activeModal = inject(NgbActiveModal) + private readonly processedMailService = inject(ProcessedMailService) + private readonly toastService = inject(ToastService) + + public processedMails: ProcessedMail[] = [] + + public loading: boolean = true + public toggleAllEnabled: boolean = false + public readonly selectedMailIds: Set = new Set() + + public page: number = 1 + + @Input() rule: MailRule + + ngOnInit(): void { + this.loadProcessedMails() + } + + public close() { + this.activeModal.close() + } + + private loadProcessedMails(): void { + this.loading = true + this.clearSelection() + this.processedMailService + .list(this.page, 50, 'processed_at', true, { rule: this.rule.id }) + .subscribe((result) => { + this.processedMails = result.results + this.loading = false + }) + } + + public deleteSelected(): void { + this.processedMailService + .bulk_delete(Array.from(this.selectedMailIds)) + .subscribe(() => { + this.toastService.showInfo($localize`Processed mail(s) deleted`) + this.loadProcessedMails() + }) + } + + public toggleAll(event: PointerEvent) { + if ((event.target as HTMLInputElement).checked) { + this.selectedMailIds.clear() + this.processedMails.forEach((mail) => this.selectedMailIds.add(mail.id)) + } else { + this.clearSelection() + } + } + + public clearSelection() { + this.toggleAllEnabled = false + this.selectedMailIds.clear() + } + + public toggleSelected(mail: ProcessedMail) { + this.selectedMailIds.has(mail.id) + ? this.selectedMailIds.delete(mail.id) + : this.selectedMailIds.add(mail.id) + } +} diff --git a/src-ui/src/app/data/processed-mail.ts b/src-ui/src/app/data/processed-mail.ts new file mode 100644 index 000000000..7eacf2415 --- /dev/null +++ b/src-ui/src/app/data/processed-mail.ts @@ -0,0 +1,12 @@ +import { ObjectWithId } from './object-with-id' + +export interface ProcessedMail extends ObjectWithId { + rule: number // MailRule.id + folder: string + uid: number + subject: string + received: Date + processed: Date + status: string + error: string +} diff --git a/src-ui/src/app/services/permissions.service.ts b/src-ui/src/app/services/permissions.service.ts index 3d88b10cc..0c36b646f 100644 --- a/src-ui/src/app/services/permissions.service.ts +++ b/src-ui/src/app/services/permissions.service.ts @@ -28,6 +28,7 @@ export enum PermissionType { ShareLink = '%s_sharelink', CustomField = '%s_customfield', Workflow = '%s_workflow', + ProcessedMail = '%s_processedmail', } @Injectable({ diff --git a/src-ui/src/app/services/rest/processed-mail.service.spec.ts b/src-ui/src/app/services/rest/processed-mail.service.spec.ts new file mode 100644 index 000000000..a424c2cbb --- /dev/null +++ b/src-ui/src/app/services/rest/processed-mail.service.spec.ts @@ -0,0 +1,39 @@ +import { HttpTestingController } from '@angular/common/http/testing' +import { TestBed } from '@angular/core/testing' +import { Subscription } from 'rxjs' +import { environment } from 'src/environments/environment' +import { commonAbstractPaperlessServiceTests } from './abstract-paperless-service.spec' +import { ProcessedMailService } from './processed-mail.service' + +let httpTestingController: HttpTestingController +let service: ProcessedMailService +let subscription: Subscription +const endpoint = 'processed_mail' + +// run common tests +commonAbstractPaperlessServiceTests(endpoint, ProcessedMailService) + +describe('Additional service tests for ProcessedMailService', () => { + beforeEach(() => { + // Dont need to setup again + + httpTestingController = TestBed.inject(HttpTestingController) + service = TestBed.inject(ProcessedMailService) + }) + + afterEach(() => { + subscription?.unsubscribe() + httpTestingController.verify() + }) + + it('should call appropriate api endpoint for bulk delete', () => { + const ids = [1, 2, 3] + subscription = service.bulk_delete(ids).subscribe() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}${endpoint}/bulk_delete/` + ) + expect(req.request.method).toEqual('POST') + expect(req.request.body).toEqual({ mail_ids: ids }) + req.flush({}) + }) +}) diff --git a/src-ui/src/app/services/rest/processed-mail.service.ts b/src-ui/src/app/services/rest/processed-mail.service.ts new file mode 100644 index 000000000..e1ea327da --- /dev/null +++ b/src-ui/src/app/services/rest/processed-mail.service.ts @@ -0,0 +1,19 @@ +import { Injectable } from '@angular/core' +import { ProcessedMail } from 'src/app/data/processed-mail' +import { AbstractPaperlessService } from './abstract-paperless-service' + +@Injectable({ + providedIn: 'root', +}) +export class ProcessedMailService extends AbstractPaperlessService { + constructor() { + super() + this.resourceName = 'processed_mail' + } + + public bulk_delete(mailIds: number[]) { + return this.http.post(`${this.getResourceUrl()}bulk_delete/`, { + mail_ids: mailIds, + }) + } +} diff --git a/src-ui/src/main.ts b/src-ui/src/main.ts index cd1f4ef59..7e57edcea 100644 --- a/src-ui/src/main.ts +++ b/src-ui/src/main.ts @@ -51,6 +51,7 @@ import { check, check2All, checkAll, + checkCircle, checkCircleFill, checkLg, chevronDoubleLeft, @@ -60,6 +61,7 @@ import { clipboardCheck, clipboardCheckFill, clipboardFill, + clockHistory, dash, dashCircle, diagram3, @@ -263,6 +265,7 @@ const icons = { check, check2All, checkAll, + checkCircle, checkCircleFill, checkLg, chevronDoubleLeft, @@ -272,6 +275,7 @@ const icons = { clipboardCheck, clipboardCheckFill, clipboardFill, + clockHistory, dash, dashCircle, diagram3, diff --git a/src/paperless/urls.py b/src/paperless/urls.py index c37331ce2..e24d1a459 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -57,6 +57,7 @@ from paperless.views import UserViewSet from paperless_mail.views import MailAccountViewSet from paperless_mail.views import MailRuleViewSet from paperless_mail.views import OauthCallbackView +from paperless_mail.views import ProcessedMailViewSet api_router = DefaultRouter() api_router.register(r"correspondents", CorrespondentViewSet) @@ -77,6 +78,7 @@ api_router.register(r"workflow_actions", WorkflowActionViewSet) api_router.register(r"workflows", WorkflowViewSet) api_router.register(r"custom_fields", CustomFieldViewSet) api_router.register(r"config", ApplicationConfigurationViewSet) +api_router.register(r"processed_mail", ProcessedMailViewSet) urlpatterns = [ diff --git a/src/paperless_mail/filters.py b/src/paperless_mail/filters.py new file mode 100644 index 000000000..57b8dec77 --- /dev/null +++ b/src/paperless_mail/filters.py @@ -0,0 +1,12 @@ +from django_filters import FilterSet + +from paperless_mail.models import ProcessedMail + + +class ProcessedMailFilterSet(FilterSet): + class Meta: + model = ProcessedMail + fields = { + "rule": ["exact"], + "status": ["exact"], + } diff --git a/src/paperless_mail/serialisers.py b/src/paperless_mail/serialisers.py index fa025fcbe..b38c8e78c 100644 --- a/src/paperless_mail/serialisers.py +++ b/src/paperless_mail/serialisers.py @@ -6,6 +6,7 @@ from documents.serialisers import OwnedObjectSerializer from documents.serialisers import TagsField from paperless_mail.models import MailAccount from paperless_mail.models import MailRule +from paperless_mail.models import ProcessedMail class ObfuscatedPasswordField(serializers.CharField): @@ -130,3 +131,20 @@ class MailRuleSerializer(OwnedObjectSerializer): if value > 36500: # ~100 years raise serializers.ValidationError("Maximum mail age is unreasonably large.") return value + + +class ProcessedMailSerializer(OwnedObjectSerializer): + class Meta: + model = ProcessedMail + fields = [ + "id", + "owner", + "rule", + "folder", + "uid", + "subject", + "received", + "processed", + "status", + "error", + ] diff --git a/src/paperless_mail/tests/test_api.py b/src/paperless_mail/tests/test_api.py index 3ba06a746..dd63c67ab 100644 --- a/src/paperless_mail/tests/test_api.py +++ b/src/paperless_mail/tests/test_api.py @@ -3,6 +3,7 @@ from unittest import mock from django.contrib.auth.models import Permission from django.contrib.auth.models import User +from django.utils import timezone from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase @@ -13,6 +14,7 @@ from documents.models import Tag from documents.tests.utils import DirectoriesMixin from paperless_mail.models import MailAccount from paperless_mail.models import MailRule +from paperless_mail.models import ProcessedMail from paperless_mail.tests.test_mail import BogusMailBox @@ -721,3 +723,285 @@ class TestAPIMailRules(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn("maximum_age", response.data) + + +class TestAPIProcessedMails(DirectoriesMixin, APITestCase): + ENDPOINT = "/api/processed_mail/" + + def setUp(self): + super().setUp() + + self.user = User.objects.create_user(username="temp_admin") + self.user.user_permissions.add(*Permission.objects.all()) + self.user.save() + self.client.force_authenticate(user=self.user) + + def test_get_processed_mails_owner_aware(self): + """ + GIVEN: + - Configured processed mails with different users + WHEN: + - API call is made to get processed mails + THEN: + - Only unowned, owned by user or granted processed mails are provided + """ + user2 = User.objects.create_user(username="temp_admin2") + + account = MailAccount.objects.create( + name="Email1", + username="username1", + password="password1", + imap_server="server.example.com", + imap_port=443, + imap_security=MailAccount.ImapSecurity.SSL, + character_set="UTF-8", + ) + + rule = MailRule.objects.create( + name="Rule1", + account=account, + folder="INBOX", + filter_from="from@example.com", + order=0, + ) + + pm1 = ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="1", + subject="Subj1", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + ) + + pm2 = ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="2", + subject="Subj2", + received=timezone.now(), + processed=timezone.now(), + status="FAILED", + error="err", + owner=self.user, + ) + + ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="3", + subject="Subj3", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + owner=user2, + ) + + pm4 = ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="4", + subject="Subj4", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + ) + pm4.owner = user2 + pm4.save() + assign_perm("view_processedmail", self.user, pm4) + + response = self.client.get(self.ENDPOINT) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 3) + returned_ids = {r["id"] for r in response.data["results"]} + self.assertSetEqual(returned_ids, {pm1.id, pm2.id, pm4.id}) + + def test_get_processed_mails_filter_by_rule(self): + """ + GIVEN: + - Processed mails belonging to two different rules + WHEN: + - API call is made with rule filter + THEN: + - Only processed mails for that rule are returned + """ + account = MailAccount.objects.create( + name="Email1", + username="username1", + password="password1", + imap_server="server.example.com", + imap_port=443, + imap_security=MailAccount.ImapSecurity.SSL, + character_set="UTF-8", + ) + + rule1 = MailRule.objects.create( + name="Rule1", + account=account, + folder="INBOX", + filter_from="from1@example.com", + order=0, + ) + rule2 = MailRule.objects.create( + name="Rule2", + account=account, + folder="INBOX", + filter_from="from2@example.com", + order=1, + ) + + pm1 = ProcessedMail.objects.create( + rule=rule1, + folder="INBOX", + uid="r1-1", + subject="R1-A", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + owner=self.user, + ) + pm2 = ProcessedMail.objects.create( + rule=rule1, + folder="INBOX", + uid="r1-2", + subject="R1-B", + received=timezone.now(), + processed=timezone.now(), + status="FAILED", + error="e", + ) + ProcessedMail.objects.create( + rule=rule2, + folder="INBOX", + uid="r2-1", + subject="R2-A", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + ) + + response = self.client.get(f"{self.ENDPOINT}?rule={rule1.pk}") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + returned_ids = {r["id"] for r in response.data["results"]} + self.assertSetEqual(returned_ids, {pm1.id, pm2.id}) + + def test_bulk_delete_processed_mails(self): + """ + GIVEN: + - Processed mails belonging to two different rules and different users + WHEN: + - API call is made to bulk delete some of the processed mails + THEN: + - Only the specified processed mails are deleted, respecting ownership and permissions + """ + user2 = User.objects.create_user(username="temp_admin2") + + account = MailAccount.objects.create( + name="Email1", + username="username1", + password="password1", + imap_server="server.example.com", + imap_port=443, + imap_security=MailAccount.ImapSecurity.SSL, + character_set="UTF-8", + ) + + rule = MailRule.objects.create( + name="Rule1", + account=account, + folder="INBOX", + filter_from="from@example.com", + order=0, + ) + + # unowned and owned by self, and one with explicit object perm + pm_unowned = ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="u1", + subject="Unowned", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + ) + pm_owned = ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="u2", + subject="Owned", + received=timezone.now(), + processed=timezone.now(), + status="FAILED", + error="e", + owner=self.user, + ) + pm_granted = ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="u3", + subject="Granted", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + owner=user2, + ) + assign_perm("delete_processedmail", self.user, pm_granted) + pm_forbidden = ProcessedMail.objects.create( + rule=rule, + folder="INBOX", + uid="u4", + subject="Forbidden", + received=timezone.now(), + processed=timezone.now(), + status="SUCCESS", + error=None, + owner=user2, + ) + + # Success for allowed items + response = self.client.post( + f"{self.ENDPOINT}bulk_delete/", + data={ + "mail_ids": [pm_unowned.id, pm_owned.id, pm_granted.id], + }, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["result"], "OK") + self.assertSetEqual( + set(response.data["deleted_mail_ids"]), + {pm_unowned.id, pm_owned.id, pm_granted.id}, + ) + self.assertFalse(ProcessedMail.objects.filter(id=pm_unowned.id).exists()) + self.assertFalse(ProcessedMail.objects.filter(id=pm_owned.id).exists()) + self.assertFalse(ProcessedMail.objects.filter(id=pm_granted.id).exists()) + self.assertTrue(ProcessedMail.objects.filter(id=pm_forbidden.id).exists()) + + # 403 and not deleted + response = self.client.post( + f"{self.ENDPOINT}bulk_delete/", + data={ + "mail_ids": [pm_forbidden.id], + }, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertTrue(ProcessedMail.objects.filter(id=pm_forbidden.id).exists()) + + # missing mail_ids + response = self.client.post( + f"{self.ENDPOINT}bulk_delete/", + data={"mail_ids": "not-a-list"}, + format="json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/src/paperless_mail/views.py b/src/paperless_mail/views.py index e48049a36..b54bcb5f7 100644 --- a/src/paperless_mail/views.py +++ b/src/paperless_mail/views.py @@ -3,8 +3,10 @@ import logging from datetime import timedelta from django.http import HttpResponseBadRequest +from django.http import HttpResponseForbidden from django.http import HttpResponseRedirect from django.utils import timezone +from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema from drf_spectacular.utils import extend_schema_view @@ -12,23 +14,29 @@ from drf_spectacular.utils import inline_serializer from httpx_oauth.oauth2 import GetAccessTokenError from rest_framework import serializers from rest_framework.decorators import action +from rest_framework.filters import OrderingFilter from rest_framework.generics import GenericAPIView from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from rest_framework.viewsets import ReadOnlyModelViewSet from documents.filters import ObjectOwnedOrGrantedPermissionsFilter from documents.permissions import PaperlessObjectPermissions +from documents.permissions import has_perms_owner_aware from documents.views import PassUserMixin from paperless.views import StandardPagination +from paperless_mail.filters import ProcessedMailFilterSet from paperless_mail.mail import MailError from paperless_mail.mail import get_mailbox from paperless_mail.mail import mailbox_login from paperless_mail.models import MailAccount from paperless_mail.models import MailRule +from paperless_mail.models import ProcessedMail from paperless_mail.oauth import PaperlessMailOAuth2Manager from paperless_mail.serialisers import MailAccountSerializer from paperless_mail.serialisers import MailRuleSerializer +from paperless_mail.serialisers import ProcessedMailSerializer from paperless_mail.tasks import process_mail_accounts @@ -126,6 +134,34 @@ class MailAccountViewSet(ModelViewSet, PassUserMixin): return Response({"result": "OK"}) +class ProcessedMailViewSet(ReadOnlyModelViewSet, PassUserMixin): + permission_classes = (IsAuthenticated, PaperlessObjectPermissions) + serializer_class = ProcessedMailSerializer + pagination_class = StandardPagination + filter_backends = ( + DjangoFilterBackend, + OrderingFilter, + ObjectOwnedOrGrantedPermissionsFilter, + ) + filterset_class = ProcessedMailFilterSet + + queryset = ProcessedMail.objects.all().order_by("-processed") + + @action(methods=["post"], detail=False) + def bulk_delete(self, request): + mail_ids = request.data.get("mail_ids", []) + if not isinstance(mail_ids, list) or not all( + isinstance(i, int) for i in mail_ids + ): + return HttpResponseBadRequest("mail_ids must be a list of integers") + mails = ProcessedMail.objects.filter(id__in=mail_ids) + for mail in mails: + if not has_perms_owner_aware(request.user, "delete_processedmail", mail): + return HttpResponseForbidden("Insufficient permissions") + mail.delete() + return Response({"result": "OK", "deleted_mail_ids": mail_ids}) + + class MailRuleViewSet(ModelViewSet, PassUserMixin): model = MailRule From c8850fa7527d010a071ff90dfd740f68e5726775 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 18:21:26 +0000 Subject: [PATCH 3/4] Auto translate strings --- src-ui/messages.xlf | 160 ++++++++++++++++++------- src/locale/en_US/LC_MESSAGES/django.po | 4 +- 2 files changed, 121 insertions(+), 43 deletions(-) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index a8cf235a6..17f1bedf5 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -755,11 +755,15 @@ src/app/components/manage/mail/mail.component.html - 122 + 123 src/app/components/manage/mail/mail.component.html - 186 + 192 + + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 16 src/app/components/manage/management-list/management-list.component.html @@ -972,6 +976,10 @@ src/app/components/common/permissions-select/permissions-select.component.html 4 + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 3 + Update checking works by pinging the public GitHub API for the latest release to determine whether a new version is available. Actual updating of the app must still be performed manually. @@ -1217,11 +1225,11 @@ src/app/components/manage/mail/mail.component.html - 148 + 154 src/app/components/manage/mail/mail.component.html - 160 + 166 src/app/components/manage/management-list/management-list.component.html @@ -1812,7 +1820,7 @@ src/app/components/manage/mail/mail.component.html - 115 + 116 src/app/components/manage/management-list/management-list.component.html @@ -2004,6 +2012,14 @@ src/app/components/admin/trash/trash.component.html 14 + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 87 + + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 89 + Empty trash @@ -2113,11 +2129,11 @@ src/app/components/manage/mail/mail.component.html - 149 + 155 src/app/components/manage/mail/mail.component.html - 163 + 169 src/app/components/manage/management-list/management-list.component.html @@ -2241,11 +2257,11 @@ src/app/components/manage/mail/mail.component.ts - 191 + 192 src/app/components/manage/mail/mail.component.ts - 292 + 293 src/app/components/manage/management-list/management-list.component.ts @@ -2432,11 +2448,11 @@ src/app/components/manage/mail/mail.component.html - 147 + 153 src/app/components/manage/mail/mail.component.html - 157 + 163 src/app/components/manage/management-list/management-list.component.html @@ -2568,11 +2584,11 @@ src/app/components/manage/mail/mail.component.ts - 193 + 194 src/app/components/manage/mail/mail.component.ts - 294 + 295 src/app/components/manage/management-list/management-list.component.ts @@ -3129,6 +3145,10 @@ src/app/components/common/clearable-badge/clearable-badge.component.html 2 + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 85 + Are you sure? @@ -3896,7 +3916,7 @@ src/app/components/manage/mail/mail.component.html - 136 + 137 src/app/components/manage/workflows/workflows.component.html @@ -4106,6 +4126,10 @@ src/app/components/common/toast/toast.component.html 30 + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 36 + Only process attachments @@ -5109,6 +5133,10 @@ src/app/components/common/email-document-dialog/email-document-dialog.component.html 11 + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 32 + Message @@ -5478,6 +5506,10 @@ src/app/components/common/permissions-select/permissions-select.component.html 9 + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 7 + Select all pages @@ -5745,11 +5777,11 @@ src/app/components/manage/mail/mail.component.html - 150 + 156 src/app/components/manage/mail/mail.component.html - 168 + 174 src/app/components/manage/workflows/workflows.component.html @@ -6127,6 +6159,10 @@ src/app/components/manage/mail/mail.component.html 114 + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 35 + src/app/components/manage/workflows/workflows.component.html 19 @@ -8517,185 +8553,227 @@ Disabled src/app/components/manage/mail/mail.component.html - 136 + 137 src/app/components/manage/workflows/workflows.component.html 41 + + View Processed Mail + + src/app/components/manage/mail/mail.component.html + 143 + + No mail rules defined. src/app/components/manage/mail/mail.component.html - 177 + 183 Error retrieving mail accounts src/app/components/manage/mail/mail.component.ts - 104 + 105 Error retrieving mail rules src/app/components/manage/mail/mail.component.ts - 126 + 127 OAuth2 authentication success src/app/components/manage/mail/mail.component.ts - 134 + 135 OAuth2 authentication failed, see logs for details src/app/components/manage/mail/mail.component.ts - 145 + 146 Saved account "". src/app/components/manage/mail/mail.component.ts - 169 + 170 Error saving account. src/app/components/manage/mail/mail.component.ts - 181 + 182 Confirm delete mail account src/app/components/manage/mail/mail.component.ts - 189 + 190 This operation will permanently delete this mail account. src/app/components/manage/mail/mail.component.ts - 190 + 191 Deleted mail account "" src/app/components/manage/mail/mail.component.ts - 200 + 201 Error deleting mail account "". src/app/components/manage/mail/mail.component.ts - 211 + 212 Processing mail account "" src/app/components/manage/mail/mail.component.ts - 223 + 224 Error processing mail account "" src/app/components/manage/mail/mail.component.ts - 228 + 229 Saved rule "". src/app/components/manage/mail/mail.component.ts - 246 + 247 Error saving rule. src/app/components/manage/mail/mail.component.ts - 257 + 258 Rule "" enabled. src/app/components/manage/mail/mail.component.ts - 273 + 274 Rule "" disabled. src/app/components/manage/mail/mail.component.ts - 274 + 275 Error toggling rule "". src/app/components/manage/mail/mail.component.ts - 279 + 280 Confirm delete mail rule src/app/components/manage/mail/mail.component.ts - 290 + 291 This operation will permanently delete this mail rule. src/app/components/manage/mail/mail.component.ts - 291 + 292 Deleted mail rule "" src/app/components/manage/mail/mail.component.ts - 301 + 302 Error deleting mail rule "". src/app/components/manage/mail/mail.component.ts - 312 + 313 Permissions updated src/app/components/manage/mail/mail.component.ts - 336 + 337 Error updating permissions src/app/components/manage/mail/mail.component.ts - 341 + 342 src/app/components/manage/management-list/management-list.component.ts 339 + + Processed Mail for + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 2 + + + + No processed email messages found. + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 20 + + + + Received + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 33 + + + + Processed + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.html + 34 + + + + Processed mail(s) deleted + + src/app/components/manage/mail/processed-mail-dialog/processed-mail-dialog.component.ts + 72 + + Filter by: diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index 1b3bc6d06..ebefeb420 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2025-09-17 22:44+0000\n" +"POT-Creation-Date: 2025-09-22 18:20+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1827,7 +1827,7 @@ msgstr "" msgid "Chinese Traditional" msgstr "" -#: paperless/urls.py:368 +#: paperless/urls.py:370 msgid "Paperless-ngx administration" msgstr "" From 8d1f23e9d61066b95e2e4a11d806e0651ddcf8a8 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Mon, 22 Sep 2025 12:53:32 -0700 Subject: [PATCH 4/4] Chore: Enable SonarQube scanning (#10904) --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- .github/workflows/ci.yml | 91 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + sonar-project.properties | 24 +++++++++++ 3 files changed, 116 insertions(+) create mode 100644 sonar-project.properties diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e28e537d7..edb6a5641 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,18 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} flags: backend-python-${{ matrix.python-version }} files: coverage.xml + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: backend-coverage-${{ matrix.python-version }} + path: | + .coverage + coverage.xml + junit.xml + retention-days: 1 + include-hidden-files: true + if-no-files-found: error - name: Stop containers if: always() run: | @@ -233,6 +245,17 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} flags: frontend-node-${{ matrix.node-version }} directory: src-ui/coverage/ + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + if: always() + with: + name: frontend-coverage-${{ matrix.shard-index }} + path: | + src-ui/coverage/lcov.info + src-ui/coverage/coverage-final.json + src-ui/junit.xml + retention-days: 1 + if-no-files-found: error tests-frontend-e2e: name: "Frontend E2E Tests (Node ${{ matrix.node-version }} - ${{ matrix.shard-index }}/${{ matrix.shard-count }})" runs-on: ubuntu-24.04 @@ -313,6 +336,74 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: cd src-ui && pnpm run build --configuration=production + sonarqube-analysis: + name: "SonarQube Analysis" + runs-on: ubuntu-24.04 + needs: + - tests-backend + - tests-frontend + if: github.repository_owner == 'paperless-ngx' + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Download all backend coverage + uses: actions/download-artifact@v5.0.0 + with: + pattern: backend-coverage-* + path: ./coverage/ + - name: Download all frontend coverage + uses: actions/download-artifact@v5.0.0 + with: + pattern: frontend-coverage-* + path: ./coverage/ + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + - name: Install coverage tools + run: | + pip install coverage + npm install -g nyc + # Merge backend coverage from all Python versions + - name: Merge backend coverage + run: | + coverage combine coverage/backend-coverage-*/.coverage + coverage xml -o merged-backend-coverage.xml + # Merge frontend coverage from all shards + - name: Merge frontend coverage + run: | + # Find all coverage-final.json files from the shards, exit with error if none found + shopt -s nullglob + files=(coverage/frontend-coverage-*/coverage/coverage-final.json) + if [ ${#files[@]} -eq 0 ]; then + echo "No frontend coverage JSON found under coverage/" >&2 + exit 1 + fi + # Create .nyc_output directory and copy each shard's coverage JSON into it with a unique name + mkdir -p .nyc_output + for coverage_json in "${files[@]}"; do + shard=$(basename "$(dirname "$(dirname "$coverage_json")")") + cp "$coverage_json" ".nyc_output/${shard}.json" + done + npx nyc merge .nyc_output .nyc_output/out.json + npx nyc report --reporter=lcovonly --report-dir coverage + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4.6.2 + with: + name: merged-coverage + path: | + merged-backend-coverage.xml + .nyc_output/* + coverage/lcov.info + retention-days: 7 + if-no-files-found: error + include-hidden-files: true + - name: SonarQube Analysis + uses: SonarSource/sonarqube-scan-action@v5 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} build-docker-image: name: Build Docker image for ${{ github.ref_name }} runs-on: ubuntu-24.04 diff --git a/pyproject.toml b/pyproject.toml index a49e94f38..f3b270c77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -255,6 +255,7 @@ PAPERLESS_DISABLE_DBHANDLER = "true" PAPERLESS_CACHE_BACKEND = "django.core.cache.backends.locmem.LocMemCache" [tool.coverage.run] +relative_files = true source = [ "src/", ] diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..d9d341e87 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,24 @@ +sonar.projectKey=paperless-ngx_paperless-ngx +sonar.organization=paperless-ngx +sonar.projectName=Paperless-ngx +sonar.projectVersion=1.0 + +# Source and test directories +sonar.sources=src/,src-ui/ +sonar.test.inclusions=**/test_*.py,**/tests.py,**/*.spec.ts,**/*.test.ts + +# Language specific settings +sonar.python.version=3.10,3.11,3.12,3.13 + +# Coverage reports +sonar.python.coverage.reportPaths=merged-backend-coverage.xml +sonar.javascript.lcov.reportPaths=coverage/lcov.info + +# Test execution reports +sonar.junit.reportPaths=**/junit.xml,**/test-results.xml + +# Encoding +sonar.sourceEncoding=UTF-8 + +# Exclusions +sonar.exclusions=**/migrations/**,**/node_modules/**,**/static/**,**/venv/**,**/.venv/**,**/dist/**