diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml index d33e4c38d..e7a32bec0 100644 --- a/docker/hub/docker-compose.postgres.yml +++ b/docker/hub/docker-compose.postgres.yml @@ -15,7 +15,7 @@ services: POSTGRES_PASSWORD: paperless webserver: - image: jonaswinkler/paperless-ng:0.9.8 + image: jonaswinkler/paperless-ng:0.9.9 restart: always depends_on: - db diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml index c130dfef6..98b7d70a2 100644 --- a/docker/hub/docker-compose.sqlite.yml +++ b/docker/hub/docker-compose.sqlite.yml @@ -5,7 +5,7 @@ services: restart: always webserver: - image: jonaswinkler/paperless-ng:0.9.8 + image: jonaswinkler/paperless-ng:0.9.9 restart: always depends_on: - broker diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index 48a86384c..8f6b91b4c 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -5,85 +5,6 @@ Advanced topics Paperless offers a couple features that automate certain tasks and make your life easier. -Guesswork -######### - - -Any document you put into the consumption directory will be consumed, but if -you name the file right, it'll automatically set some values in the database -for you. This is is the logic the consumer follows: - -1. Try to find the correspondent, title, and tags in the file name following - the pattern: ``Date - Correspondent - Title - tag,tag,tag.pdf``. Note that - the format of the date is **rigidly defined** as ``YYYYMMDDHHMMSSZ`` or - ``YYYYMMDDZ``. The ``Z`` refers "Zulu time" AKA "UTC". - The tags are optional, so the format ``Date - Correspondent - Title.pdf`` - works as well. -2. If that doesn't work, we skip the date and try this pattern: - ``Correspondent - Title - tag,tag,tag.pdf``. -3. If that doesn't work, we try to find the correspondent and title in the file - name following the pattern: ``Correspondent - Title.pdf``. -4. If that doesn't work, just assume that the name of the file is the title. - -So given the above, the following examples would work as you'd expect: - -* ``20150314000700Z - Some Company Name - Invoice 2016-01-01 - money,invoices.pdf`` -* ``20150314Z - Some Company Name - Invoice 2016-01-01 - money,invoices.pdf`` -* ``Some Company Name - Invoice 2016-01-01 - money,invoices.pdf`` -* ``Another Company - Letter of Reference.jpg`` -* ``Dad's Recipe for Pancakes.png`` - -These however wouldn't work: - -* ``2015-03-14 00:07:00 UTC - Some Company Name, Invoice 2016-01-01, money, invoices.pdf`` -* ``2015-03-14 - Some Company Name, Invoice 2016-01-01, money, invoices.pdf`` -* ``Some Company Name, Invoice 2016-01-01, money, invoices.pdf`` -* ``Another Company- Letter of Reference.jpg`` - -Do I have to be so strict about naming? -======================================= - -Rather than using the strict document naming rules, one can also set the option -``PAPERLESS_FILENAME_DATE_ORDER`` in ``paperless.conf`` to any date order -that is accepted by dateparser_. Doing so will cause ``paperless`` to default -to any date format that is found in the title, instead of a date pulled from -the document's text, without requiring the strict formatting of the document -filename as described above. - -.. _dateparser: https://github.com/scrapinghub/dateparser/blob/v0.7.0/docs/usage.rst#settings - -.. _advanced-transforming_filenames: - -Transforming filenames for parsing -================================== - -Some devices can't produce filenames that can be parsed by the default -parser. By configuring the option ``PAPERLESS_FILENAME_PARSE_TRANSFORMS`` in -``paperless.conf`` one can add transformations that are applied to the filename -before it's parsed. - -The option contains a list of dictionaries of regular expressions (key: -``pattern``) and replacements (key: ``repl``) in JSON format, which are -applied in order by passing them to ``re.subn``. Transformation stops -after the first match, so at most one transformation is applied. The general -syntax is - -.. code:: python - - [{"pattern":"pattern1", "repl":"repl1"}, {"pattern":"pattern2", "repl":"repl2"}, ..., {"pattern":"patternN", "repl":"replN"}] - -The example below is for a Brother ADS-2400N, a scanner that allows -different names to different hardware buttons (useful for handling -multiple entities in one instance), but insists on adding ``_`` -to the filename. - -.. code:: python - - # Brother profile configuration, support "Name_Date_Count" (the default - # setting) and "Name_Count" (use "Name" as tag and "Count" as title). - PAPERLESS_FILENAME_PARSE_TRANSFORMS=[{"pattern":"^([a-z]+)_(\\d{8})_(\\d{6})_([0-9]+)\\.", "repl":"\\2\\3Z - \\4 - \\1."}, {"pattern":"^([a-z]+)_([0-9]+)\\.", "repl":" - \\2 - \\1."}] - - .. _advanced-matching: Matching tags, correspondents and document types diff --git a/docs/api.rst b/docs/api.rst index d352758fa..cff72a970 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -221,21 +221,16 @@ Each fragment contains a list of strings, and some of them are marked as a highl [ [ - {"text": "This is a sample text with a "}, - {"text": "highlighted", "term": 0}, - {"text": " word."} + {"text": "This is a sample text with a ", "highlight": false}, + {"text": "highlighted", "highlight": true}, + {"text": " word.", "highlight": false} ], [ - {"text": "Another", "term": 1}, - {"text": " fragment with a highlight."} + {"text": "Another", "highlight": true}, + {"text": " fragment with a highlight.", "highlight": false} ] ] - - -When ``term`` is present within a string, the word within ``text`` should be highlighted. -The term index groups multiple matches together and words with the same index -should get identical highlighting. A client may use this example to produce the following output: ... This is a sample text with a **highlighted** word. ... **Another** fragment with a highlight. ... diff --git a/docs/changelog.rst b/docs/changelog.rst index a993eb530..e63c19d7d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,40 @@ Changelog ********* +paperless-ng 0.9.9 +################## + +Christmas release! + +* Bulk editing + + * Paperless now supports bulk editing. + * The following operations are available: Add and remove correspondents, tags, document types from selected documents, as well as mass-deleting documents. + * We've got a more fancy UI in the works that makes these features more accessible, but that's not quite ready yet. + +* Searching + + * Paperless now supports searching for similar documents ("More like this") both from the document detail page as well as from individual search results. + * A search score indicates how well a document matches the search query, or how similar a document is to a given reference document. + +* Other additions and changes + + * Clarification in the UI that the fields "Match" and "Is insensitive" are not relevant for the Auto matching algorithm. + * New select interface for tags, types and correspondents allows filtering. This also improves tag selection. Thanks again to `Michael Shamoon`_! + * Page navigation controls for the document viewer, thanks to `Michael Shamoon`_. + * Layout changes to the small cards document list. + * The dashboard now displays the username (or full name if specified in the admin) on the dashboard. + +* Fixes + + * An error that caused the document importer to crash was fixed. + * An issue with changes not being possible when ``PAPERLESS_COOKIE_PREFIX`` is used was fixed. + * The date selection filters now allow manual entry of dates. + +* Feature Removal + + * Most of the guesswork features have been removed. Paperless no longer tries to extract correspondents and tags from file names. + paperless-ng 0.9.8 ################## diff --git a/docs/configuration.rst b/docs/configuration.rst index d3f47215b..efc1a9db1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -400,11 +400,6 @@ PAPERLESS_FILENAME_DATE_ORDER= Defaults to none, which disables this feature. -PAPERLESS_FILENAME_PARSE_TRANSFORMS - Transforms filenames before they are processed by paperless. See - :ref:`advanced-transforming_filenames` for details. - - Defaults to none, which disables this feature. Binaries ######## diff --git a/docs/setup.rst b/docs/setup.rst index e20b2e54a..4d29ce640 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -120,6 +120,8 @@ The `bare metal route`_ is more complicated to setup but makes it easier should you want to contribute some code back. You need to configure and run the above mentioned components yourself. +.. _setup-docker_route: + Docker Route ============ diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index dc5bf7f5d..4c06ec4cd 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -39,7 +39,7 @@ Operation not permitted You might see errors such as: -.. code:: +.. code:: shell-session chown: changing ownership of '../export': Operation not permitted @@ -49,3 +49,29 @@ to these folders. This happens when pointing these directories to NFS shares, for example. Ensure that `chown` is possible on these directories. + +Classifier error: No training data available +############################################ + +This indicates that the Auto matching algorithm found no documents to learn from. +This may have two reasons: + +* You don't use the Auto matching algorithm: The error can be safely ignored in this case. +* You are using the Auto matching algorithm: The classifier explicitly excludes documents + with Inbox tags. Verify that there are documents in your archive without inbox tags. + The algorithm will only learn from documents not in your inbox. + +Permission denied errors in the consumption directory +##################################################### + +You might encounter errors such as: + +.. code:: shell-session + + The following error occured while consuming document.pdf: [Errno 13] Permission denied: '/usr/src/paperless/src/../consume/document.pdf' + +This happens when paperless does not have permission to delete files inside the consumption directory. +Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group id you use on the host operating system, if these are +different from ``1000``. See :ref:`setup-docker_route`. + +Also ensure that you are able to read and write to the consumption directory on the host. diff --git a/src-ui/package-lock.json b/src-ui/package-lock.json index 5eca0b3c0..10215a32d 100644 --- a/src-ui/package-lock.json +++ b/src-ui/package-lock.json @@ -2056,6 +2056,14 @@ "tslib": "^2.0.0" } }, + "@ng-select/ng-select": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/@ng-select/ng-select/-/ng-select-5.0.9.tgz", + "integrity": "sha512-YZeSAiS8/Nx/eHZJPmOOYL8YmcvSq+dr1P8WIrsKmRA7mueorBpPc5xlUj+nLQbpLtsiQvdWDQspf/ykOvD/lA==", + "requires": { + "tslib": "^2.0.0" + } + }, "@ngtools/webpack": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-10.2.0.tgz", diff --git a/src-ui/package.json b/src-ui/package.json index 6293f2672..14d828483 100644 --- a/src-ui/package.json +++ b/src-ui/package.json @@ -21,6 +21,7 @@ "@angular/platform-browser-dynamic": "~10.1.5", "@angular/router": "~10.1.5", "@ng-bootstrap/ng-bootstrap": "^8.0.0", + "@ng-select/ng-select": "^5.0.9", "bootstrap": "^4.5.0", "ng-bootstrap": "^1.6.3", "ng2-pdf-viewer": "^6.3.2", diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index 3c00cd0b7..37b3a027d 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -54,6 +54,8 @@ import { FileSizePipe } from './pipes/file-size.pipe'; import { FilterPipe } from './pipes/filter.pipe'; import { DocumentTitlePipe } from './pipes/document-title.pipe'; import { MetadataCollapseComponent } from './components/document-detail/metadata-collapse/metadata-collapse.component'; +import { NgSelectModule } from '@ng-select/ng-select'; +import { SelectDialogComponent } from './components/common/select-dialog/select-dialog.component'; @NgModule({ declarations: [ @@ -99,7 +101,8 @@ import { MetadataCollapseComponent } from './components/document-detail/metadata FileSizePipe, FilterPipe, DocumentTitlePipe, - MetadataCollapseComponent + MetadataCollapseComponent, + SelectDialogComponent ], imports: [ BrowserModule, @@ -110,7 +113,8 @@ import { MetadataCollapseComponent } from './components/document-detail/metadata ReactiveFormsModule, NgxFileDropModule, InfiniteScrollModule, - PdfViewerModule + PdfViewerModule, + NgSelectModule ], providers: [ DatePipe, diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html index 53b613244..f4dffa7d1 100644 --- a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html @@ -10,5 +10,8 @@
- -
\ No newline at end of file + + diff --git a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts index e207f4598..4791d0e77 100644 --- a/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts +++ b/src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts @@ -28,6 +28,21 @@ export class ConfirmDialogComponent implements OnInit { @Input() btnCaption = "Confirm" + confirmButtonEnabled = true + seconds = 0 + + delayConfirm(seconds: number) { + this.confirmButtonEnabled = false + this.seconds = seconds + setTimeout(() => { + if (this.seconds <= 1) { + this.confirmButtonEnabled = true + } else { + this.delayConfirm(seconds - 1) + } + }, 1000) + } + ngOnInit(): void { } diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html index 717aa7964..780dc5686 100644 --- a/src-ui/src/app/components/common/input/select/select.component.html +++ b/src-ui/src/app/components/common/input/select/select.component.html @@ -1,11 +1,16 @@ -
+
- + + {{i.name}} + +
{{hint}} -
\ No newline at end of file +
diff --git a/src-ui/src/app/components/common/input/select/select.component.scss b/src-ui/src/app/components/common/input/select/select.component.scss index e69de29bb..8faec3bc0 100644 --- a/src-ui/src/app/components/common/input/select/select.component.scss +++ b/src-ui/src/app/components/common/input/select/select.component.scss @@ -0,0 +1 @@ +// styles for ng-select child are in styles.scss diff --git a/src-ui/src/app/components/common/input/tags/tags.component.html b/src-ui/src/app/components/common/input/tags/tags.component.html index 8029dd860..8a5dbc4f2 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.html +++ b/src-ui/src/app/components/common/input/tags/tags.component.html @@ -1,30 +1,41 @@ -
- +
+ -
-
- -
+
+ -
- -
- -
-
+ + + + + + + + + +
+
+ + + +
+ +
+ +
-
-
{{hint}} -
\ No newline at end of file +
diff --git a/src-ui/src/app/components/common/input/tags/tags.component.scss b/src-ui/src/app/components/common/input/tags/tags.component.scss index f2635b7f2..2eaaa4f6d 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.scss +++ b/src-ui/src/app/components/common/input/tags/tags.component.scss @@ -1,10 +1,12 @@ -.tags-form-control { - height: auto; +.selected-icon { + min-width: 1em; + min-height: 1em; } +.tag-wrap { + font-size: 1rem; +} -.scrollable-menu { - height: auto; - max-height: 300px; - overflow-x: hidden; -} \ No newline at end of file +.tag-wrap-delete { + cursor: pointer; +} diff --git a/src-ui/src/app/components/common/input/tags/tags.component.ts b/src-ui/src/app/components/common/input/tags/tags.component.ts index cca99cc55..5501ac5a6 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.ts +++ b/src-ui/src/app/components/common/input/tags/tags.component.ts @@ -21,7 +21,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor { onChange = (newValue: number[]) => {}; - + onTouched = () => {}; writeValue(newValue: number[]): void { @@ -66,29 +66,28 @@ export class TagsComponent implements OnInit, ControlValueAccessor { removeTag(id) { let index = this.displayValue.indexOf(id) if (index > -1) { - this.displayValue.splice(index, 1) + let oldValue = this.displayValue + oldValue.splice(index, 1) + this.displayValue = [...oldValue] this.onChange(this.displayValue) } } - addTag(id) { - let index = this.displayValue.indexOf(id) - if (index == -1) { - this.displayValue.push(id) - this.onChange(this.displayValue) - } - } - - createTag() { var modal = this.modalService.open(TagEditDialogComponent, {backdrop: 'static'}) modal.componentInstance.dialogMode = 'create' modal.componentInstance.success.subscribe(newTag => { this.tagService.listAll().subscribe(tags => { this.tags = tags.results - this.addTag(newTag.id) + this.displayValue = [...this.displayValue, newTag.id] + this.onChange(this.displayValue) }) }) } + ngSelectChange() { + this.value = this.displayValue + this.onChange(this.displayValue) + } + } diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.html b/src-ui/src/app/components/common/select-dialog/select-dialog.component.html new file mode 100644 index 000000000..8bde38d61 --- /dev/null +++ b/src-ui/src/app/components/common/select-dialog/select-dialog.component.html @@ -0,0 +1,15 @@ +
+ + +
+
+ + + +
+
+ + +
\ No newline at end of file diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.scss b/src-ui/src/app/components/common/select-dialog/select-dialog.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.spec.ts b/src-ui/src/app/components/common/select-dialog/select-dialog.component.spec.ts new file mode 100644 index 000000000..3810bcbea --- /dev/null +++ b/src-ui/src/app/components/common/select-dialog/select-dialog.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { SelectDialogComponent } from './select-dialog.component'; + +describe('SelectDialogComponent', () => { + let component: SelectDialogComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ SelectDialogComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(SelectDialogComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.ts b/src-ui/src/app/components/common/select-dialog/select-dialog.component.ts new file mode 100644 index 000000000..76b23491c --- /dev/null +++ b/src-ui/src/app/components/common/select-dialog/select-dialog.component.ts @@ -0,0 +1,34 @@ +import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; +import { ObjectWithId } from 'src/app/data/object-with-id'; + +@Component({ + selector: 'app-select-dialog', + templateUrl: './select-dialog.component.html', + styleUrls: ['./select-dialog.component.scss'] +}) + +export class SelectDialogComponent implements OnInit { + constructor(public activeModal: NgbActiveModal) { } + + @Output() + public selectClicked = new EventEmitter() + + @Input() + title = "Select" + + @Input() + message = "Please select an object" + + @Input() + objects: ObjectWithId[] = [] + + selected: number + + ngOnInit(): void { + } + + cancelClicked() { + this.activeModal.close() + } +} diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html index 627e7ff22..541255a68 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.html +++ b/src-ui/src/app/components/dashboard/dashboard.component.html @@ -1,4 +1,4 @@ - + diff --git a/src-ui/src/app/components/dashboard/dashboard.component.ts b/src-ui/src/app/components/dashboard/dashboard.component.ts index a14ec5e90..db9b5d425 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.ts +++ b/src-ui/src/app/components/dashboard/dashboard.component.ts @@ -1,4 +1,5 @@ import { Component, OnInit } from '@angular/core'; +import { Meta } from '@angular/platform-browser'; import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'; import { SavedViewService } from 'src/app/services/rest/saved-view.service'; @@ -11,8 +12,29 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service'; export class DashboardComponent implements OnInit { constructor( - private savedViewService: SavedViewService) { } + private savedViewService: SavedViewService, + private meta: Meta + ) { } + get displayName() { + let tagFullName = this.meta.getTag('name=full_name') + let tagUsername = this.meta.getTag('name=username') + if (tagFullName && tagFullName.content) { + return tagFullName.content + } else if (tagUsername && tagUsername.content) { + return tagUsername.content + } else { + return null + } + } + + get subtitle() { + if (this.displayName) { + return `Hello ${this.displayName}, welcome to Paperless-ng!` + } else { + return `Welcome to Paperless-ng!` + } + } savedViews: PaperlessSavedView[] = [] diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts index 5bfecc640..f200d8db9 100644 --- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts +++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts @@ -23,7 +23,7 @@ export class SavedViewWidgetComponent implements OnInit { documents: PaperlessDocument[] = [] ngOnInit(): void { - this.documentService.list(1,10,this.savedView.sort_field, this.savedView.sort_reverse, this.savedView.filter_rules).subscribe(result => { + this.documentService.listFiltered(1,10,this.savedView.sort_field, this.savedView.sort_reverse, this.savedView.filter_rules).subscribe(result => { this.documents = result.results }) } diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index f4a64c2cc..ae3fb0c0a 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -1,4 +1,14 @@ +
+
+
Page
+
+ +
+
of {{previewNumPages}}
+
+
+
+ +
+ + + +
+ + + + + + +
+ +
+ +
-

{{list.collectionSize || 0}} document(s) (filtered)

+

Selected {{list.selected.size}} of {{list.collectionSize || 0}} document(s) (filtered)

- +
+ @@ -87,7 +112,13 @@ - + + @@ -116,6 +147,6 @@
ASN Correspondent TitleAdded
+
+ + +
+ 		{{d.archive_serial_number}}
-
- +
+
diff --git a/src-ui/src/app/components/document-list/document-list.component.scss b/src-ui/src/app/components/document-list/document-list.component.scss index e69de29bb..d7c08abec 100644 --- a/src-ui/src/app/components/document-list/document-list.component.scss +++ b/src-ui/src/app/components/document-list/document-list.component.scss @@ -0,0 +1,27 @@ +@import "/src/theme"; + +.table-row-selected { + background-color: $primaryFaded; +} + +$paperless-card-breakpoints: ( + 0: 2, // xs + 768px: 3, //md + 992px: 4, //lg + 1200px: 5, //xl + 1400px: 6, // xxl + 1600px: 7, + 1800px: 8, + 2000px: 9 +); + +.row-cols-paperless-cards { + @each $width, $n_cols in $paperless-card-breakpoints { + @media(min-width: $width) { + > * { + flex: 0 0 auto; + width: 100% / $n_cols; + } + } + } +} diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts index 25d92e9db..f72a92aa9 100644 --- a/src-ui/src/app/components/document-list/document-list.component.ts +++ b/src-ui/src/app/components/document-list/document-list.component.ts @@ -1,14 +1,22 @@ import { Component, OnInit, ViewChild } from '@angular/core'; import { ActivatedRoute, Router } from '@angular/router'; import { NgbModal } from '@ng-bootstrap/ng-bootstrap'; -import { FILTER_CORRESPONDENT } from 'src/app/data/filter-rule-type'; +import { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; +import { PaperlessDocument } from 'src/app/data/paperless-document'; import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'; import { DocumentListViewService } from 'src/app/services/document-list-view.service'; -import { DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service'; +import { CorrespondentService } from 'src/app/services/rest/correspondent.service'; +import { DocumentTypeService } from 'src/app/services/rest/document-type.service'; +import { DocumentService, DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service'; +import { TagService } from 'src/app/services/rest/tag.service'; import { SavedViewService } from 'src/app/services/rest/saved-view.service'; import { Toast, ToastService } from 'src/app/services/toast.service'; import { FilterEditorComponent } from '../filter-editor/filter-editor.component'; +import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component'; +import { SelectDialogComponent } from '../common/select-dialog/select-dialog.component'; import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-view-config-dialog.component'; +import { OpenDocumentsService } from 'src/app/services/open-documents.service'; @Component({ selector: 'app-document-list', @@ -23,7 +31,12 @@ export class DocumentListComponent implements OnInit { public route: ActivatedRoute, private router: Router, private toastService: ToastService, - public modalService: NgbModal) { } + public modalService: NgbModal, + private correspondentService: CorrespondentService, + private documentTypeService: DocumentTypeService, + private tagService: TagService, + private documentService: DocumentService, + private openDocumentService: OpenDocumentsService) { } @ViewChild("filterEditor") private filterEditor: FilterEditorComponent @@ -113,4 +126,122 @@ export class DocumentListComponent implements OnInit { this.filterEditor.toggleDocumentType(documentTypeID) } + trackByDocumentId(index, item: PaperlessDocument) { + return item.id + } + + private executeBulkOperation(method: string, args): Observable { + return this.documentService.bulkEdit(Array.from(this.list.selected), method, args).pipe( + tap(() => { + this.list.reload() + this.list.selected.forEach(id => { + this.openDocumentService.refreshDocument(id) + }) + this.list.selectNone() + }) + ) + } + + bulkSetCorrespondent() { + let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'}) + modal.componentInstance.title = "Select correspondent" + modal.componentInstance.message = `Select the correspondent you wish to assign to ${this.list.selected.size} selected document(s):` + this.correspondentService.listAll().subscribe(response => { + modal.componentInstance.objects = response.results + }) + modal.componentInstance.selectClicked.subscribe(selectedId => { + this.executeBulkOperation('set_correspondent', {"correspondent": selectedId}).subscribe( + response => { + modal.close() + } + ) + }) + } + + bulkRemoveCorrespondent() { + let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'}) + modal.componentInstance.title = "Remove correspondent" + modal.componentInstance.message = `This operation will remove the correspondent from all ${this.list.selected.size} selected document(s).` + modal.componentInstance.confirmClicked.subscribe(() => { + this.executeBulkOperation('set_correspondent', {"correspondent": null}).subscribe(r => { + modal.close() + }) + }) + } + + bulkSetDocumentType() { + let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'}) + modal.componentInstance.title = "Select document type" + modal.componentInstance.message = `Select the document type you wish to assign to ${this.list.selected.size} selected document(s):` + this.documentTypeService.listAll().subscribe(response => { + modal.componentInstance.objects = response.results + }) + modal.componentInstance.selectClicked.subscribe(selectedId => { + this.executeBulkOperation('set_document_type', {"document_type": selectedId}).subscribe( + response => { + modal.close() + } + ) + }) + } + + bulkRemoveDocumentType() { + let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'}) + modal.componentInstance.title = "Remove document type" + modal.componentInstance.message = `This operation will remove the document type from all ${this.list.selected.size} selected document(s).` + modal.componentInstance.confirmClicked.subscribe(() => { + this.executeBulkOperation('set_document_type', {"document_type": null}).subscribe(r => { + modal.close() + }) + }) + } + + bulkAddTag() { + let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'}) + modal.componentInstance.title = "Select tag" + modal.componentInstance.message = `Select the tag you wish to assign to ${this.list.selected.size} selected document(s):` + this.tagService.listAll().subscribe(response => { + modal.componentInstance.objects = response.results + }) + modal.componentInstance.selectClicked.subscribe(selectedId => { + this.executeBulkOperation('add_tag', {"tag": selectedId}).subscribe( + response => { + modal.close() + } + ) + }) + } + + bulkRemoveTag() { + let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'}) + modal.componentInstance.title = "Select tag" + modal.componentInstance.message = `Select the tag you wish to remove from ${this.list.selected.size} selected document(s):` + this.tagService.listAll().subscribe(response => { + modal.componentInstance.objects = response.results + }) + modal.componentInstance.selectClicked.subscribe(selectedId => { + this.executeBulkOperation('remove_tag', {"tag": selectedId}).subscribe( + response => { + modal.close() + } + ) + }) + } + + bulkDelete() { + let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'}) + modal.componentInstance.delayConfirm(5) + modal.componentInstance.title = "Delete confirm" + modal.componentInstance.messageBold = `This operation will permanently delete all ${this.list.selected.size} selected document(s).` + modal.componentInstance.message = `This operation cannot be undone.` + modal.componentInstance.btnClass = "btn-danger" + modal.componentInstance.btnCaption = "Delete document(s)" + modal.componentInstance.confirmClicked.subscribe(() => { + this.executeBulkOperation("delete", {}).subscribe( + response => { + modal.close() + } + ) + }) + } } diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html index 6f6a42fe2..aca6e836c 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html +++ b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html @@ -4,38 +4,39 @@
- -
diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts index 91402d084..e570862cd 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts +++ b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts @@ -1,24 +1,37 @@ -import { Component, EventEmitter, Input, Output, ElementRef, ViewChild, SimpleChange } from '@angular/core'; -import { NgbDate, NgbDateStruct, NgbDatepicker } from '@ng-bootstrap/ng-bootstrap'; +import { formatDate } from '@angular/common'; +import { Component, EventEmitter, Input, Output, OnInit, OnDestroy } from '@angular/core'; +import { Subject, Subscription } from 'rxjs'; +import { debounceTime, distinctUntilChanged } from 'rxjs/operators'; export interface DateSelection { - before?: NgbDateStruct - after?: NgbDateStruct + before?: string + after?: string } +const FILTER_LAST_7_DAYS = 0 +const FILTER_LAST_MONTH = 1 +const FILTER_LAST_3_MONTHS = 2 +const FILTER_LAST_YEAR = 3 @Component({ selector: 'app-filter-dropdown-date', templateUrl: './filter-dropdown-date.component.html', styleUrls: ['./filter-dropdown-date.component.scss'] }) -export class FilterDropdownDateComponent { +export class FilterDropdownDateComponent implements OnInit, OnDestroy { + + quickFilters = [ + {id: FILTER_LAST_7_DAYS, name: "Last 7 days"}, + {id: FILTER_LAST_MONTH, name: "Last month"}, + {id: FILTER_LAST_3_MONTHS, name: "Last 3 months"}, + {id: FILTER_LAST_YEAR, name: "Last year"} + ] @Input() - dateBefore: NgbDateStruct + dateBefore: string @Input() - dateAfter: NgbDateStruct + dateAfter: string @Input() title: string @@ -26,87 +39,65 @@ export class FilterDropdownDateComponent { @Output() datesSet = new EventEmitter() - @ViewChild('dpAfter') dpAfter: NgbDatepicker - @ViewChild('dpBefore') dpBefore: NgbDatepicker + private datesSetDebounce$ = new Subject() - _dateBefore: NgbDateStruct - _dateAfter: NgbDateStruct - - get _maxDate(): NgbDate { - let date = new Date() - return NgbDate.from({year: date.getFullYear(), month: date.getMonth() + 1, day: date.getDate()}) + private sub: Subscription + + ngOnInit() { + this.sub = this.datesSetDebounce$.pipe( + debounceTime(400) + ).subscribe(() => { + this.onChange() + }) } - isStringRange(range: any) { - return typeof range == 'string' - } - - ngOnChanges(changes: SimpleChange) { - // this is a hacky workaround perhaps because of https://github.com/angular/angular/issues/11097 - let dateString: string = '' - let dateAfterChange: SimpleChange - let dateBeforeChange: SimpleChange - if (changes) { - dateAfterChange = changes['dateAfter'] - dateBeforeChange = changes['dateBefore'] + ngOnDestroy() { + if (this.sub) { + this.sub.unsubscribe() } + } - if (this.dpBefore && this.dpAfter) { - let dpAfterElRef: ElementRef = this.dpAfter['_elRef'] - let dpBeforeElRef: ElementRef = this.dpBefore['_elRef'] + setDateQuickFilter(qf: number) { + this.dateBefore = null + let date = new Date() + switch (qf) { + case FILTER_LAST_7_DAYS: + date.setDate(date.getDate() - 7) + break; - if (dateAfterChange && dateAfterChange.currentValue) { - let dateAfterDate = dateAfterChange.currentValue as NgbDateStruct - dateString = `${dateAfterDate.year}-${dateAfterDate.month.toString().padStart(2,'0')}-${dateAfterDate.day.toString().padStart(2,'0')}` - dpAfterElRef.nativeElement.value = dateString - } else if (dateBeforeChange && dateBeforeChange.currentValue) { - let dateBeforeDate = dateBeforeChange.currentValue as NgbDateStruct - dateString = `${dateBeforeDate.year}-${dateBeforeDate.month.toString().padStart(2,'0')}-${dateBeforeDate.day.toString().padStart(2,'0')}` - dpBeforeElRef.nativeElement.value = dateString - } else { - dpAfterElRef.nativeElement.value = dateString - dpBeforeElRef.nativeElement.value = dateString + case FILTER_LAST_MONTH: + date.setMonth(date.getMonth() - 1) + break; + + case FILTER_LAST_3_MONTHS: + date.setMonth(date.getMonth() - 3) + break + + case FILTER_LAST_YEAR: + date.setFullYear(date.getFullYear() - 1) + break + } - } + this.dateAfter = formatDate(date, 'yyyy-MM-dd', "en-us", "UTC") + this.onChange() } - setDateQuickFilter(range: any) { - let date = new Date() - let newDate: NgbDateStruct = { year: date.getFullYear(), month: date.getMonth() + 1, day: date.getDate() } - switch (typeof range) { - case 'number': - date.setDate(date.getDate() - range) - newDate.year = date.getFullYear() - newDate.month = date.getMonth() + 1 - newDate.day = date.getDate() - break - - case 'string': - newDate.day = 1 - if (range == 'year') newDate.month = 1 - break - - default: - break - } - this._dateAfter = newDate - this._dateBefore = null - this.datesSet.emit({after: newDate, before: null}) + onChange() { + this.datesSet.emit({after: this.dateAfter, before: this.dateBefore}) } - onBeforeSelected(date: NgbDateStruct) { - this._dateBefore = date - this.datesSet.emit({after: this._dateAfter, before: date}) + onChangeDebounce() { + this.datesSetDebounce$.next({after: this.dateAfter, before: this.dateBefore}) } - onAfterSelected(date: NgbDateStruct) { - this._dateAfter = date - this.datesSet.emit({after: date, before: this._dateBefore}) + clearBefore() { + this.dateBefore = null; + this.onChange() } - clear() { - this._dateBefore = null - this._dateAfter = null - this.datesSet.emit({after: null, before: null}) + clearAfter() { + this.dateAfter = null; + this.onChange() } + } diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/filter-editor/filter-editor.component.ts index f762c6138..913c738a5 100644 --- a/src-ui/src/app/components/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/filter-editor/filter-editor.component.ts @@ -179,54 +179,53 @@ export class FilterEditorComponent implements OnInit, OnDestroy { this.applyFilters() } - get dateCreatedBefore(): NgbDateStruct { + get dateCreatedBefore(): string { let createdBeforeRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_CREATED_BEFORE) - return createdBeforeRule ? this.dateParser.parse(createdBeforeRule.value) : null + return createdBeforeRule ? createdBeforeRule.value : null } - get dateCreatedAfter(): NgbDateStruct { + get dateCreatedAfter(): string { let createdAfterRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_CREATED_AFTER) - return createdAfterRule ? this.dateParser.parse(createdAfterRule.value) : null + return createdAfterRule ? createdAfterRule.value : null } - get dateAddedBefore(): NgbDateStruct { + get dateAddedBefore(): string { let addedBeforeRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_ADDED_BEFORE) - return addedBeforeRule ? this.dateParser.parse(addedBeforeRule.value) : null + return addedBeforeRule ? addedBeforeRule.value : null } - get dateAddedAfter(): NgbDateStruct { + get dateAddedAfter(): string { let addedAfterRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_ADDED_AFTER) - return addedAfterRule ? this.dateParser.parse(addedAfterRule.value) : null + return addedAfterRule ? addedAfterRule.value : null } - setDateCreatedBefore(date?: NgbDateStruct) { + setDateCreatedBefore(date?: string) { if (date) this.setDateFilter(date, FILTER_CREATED_BEFORE) else this.clearDateFilter(FILTER_CREATED_BEFORE) } - setDateCreatedAfter(date?: NgbDateStruct) { + setDateCreatedAfter(date?: string) { if (date) this.setDateFilter(date, FILTER_CREATED_AFTER) else this.clearDateFilter(FILTER_CREATED_AFTER) } - setDateAddedBefore(date?: NgbDateStruct) { + setDateAddedBefore(date?: string) { if (date) this.setDateFilter(date, FILTER_ADDED_BEFORE) else this.clearDateFilter(FILTER_ADDED_BEFORE) } - setDateAddedAfter(date?: NgbDateStruct) { + setDateAddedAfter(date?: string) { if (date) this.setDateFilter(date, FILTER_ADDED_AFTER) else this.clearDateFilter(FILTER_ADDED_AFTER) } - setDateFilter(date: NgbDateStruct, dateRuleTypeID: number) { + setDateFilter(date: string, dateRuleTypeID: number) { let existingRule = this.filterRules.find(rule => rule.rule_type == dateRuleTypeID) - let newValue = this.dateParser.format(date) if (existingRule) { - existingRule.value = newValue + existingRule.value = date } else { - this.filterRules.push({rule_type: dateRuleTypeID, value: newValue}) + this.filterRules.push({rule_type: dateRuleTypeID, value: date}) } } diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html b/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html index 307c78c3c..e09ea38bf 100644 --- a/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html @@ -8,10 +8,9 @@
- - - + +
diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html index 013c5a947..3338c40c3 100644 --- a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html @@ -8,9 +8,9 @@
- - + +
diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html index 8048b0c80..138d3e7cd 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html @@ -7,11 +7,21 @@
- + + +
+ + + + {{item.name}} + + +
+ - - + +
diff --git a/src-ui/src/app/components/search/result-highlight/result-highlight.component.html b/src-ui/src/app/components/search/result-highlight/result-highlight.component.html index 1842f5cea..5dc5baa94 100644 --- a/src-ui/src/app/components/search/result-highlight/result-highlight.component.html +++ b/src-ui/src/app/components/search/result-highlight/result-highlight.component.html @@ -1,3 +1,3 @@ ... - {{token.text}} ... + {{token.text}} ... \ No newline at end of file diff --git a/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss b/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss index 645fb0426..e04dd13b2 100644 --- a/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss +++ b/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss @@ -1,4 +1,4 @@ .match { color: black; - background-color: orange; + background-color: rgb(255, 211, 66); } \ No newline at end of file diff --git a/src-ui/src/app/components/search/search.component.html b/src-ui/src/app/components/search/search.component.html index 55fcee900..de6f0133f 100644 --- a/src-ui/src/app/components/search/search.component.html +++ b/src-ui/src/app/components/search/search.component.html @@ -3,7 +3,12 @@
Invalid search query: {{errorMessage}}
-

+

+ Showing documents similar to + {{more_like_doc?.original_file_name}} +

+ +

Search string: {{query}} - Did you mean "{{correctedQuery}}"? @@ -15,7 +20,9 @@

{{resultCount}} result(s)

+ [details]="result.highlights" + [searchScore]="result.score / maxScore" + [moreLikeThis]="true">
diff --git a/src-ui/src/app/components/search/search.component.ts b/src-ui/src/app/components/search/search.component.ts index de8b4652f..4570ac3fa 100644 --- a/src-ui/src/app/components/search/search.component.ts +++ b/src-ui/src/app/components/search/search.component.ts @@ -1,6 +1,9 @@ import { Component, OnInit } from '@angular/core'; import { ActivatedRoute, Router } from '@angular/router'; +import { PaperlessDocument } from 'src/app/data/paperless-document'; +import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'; import { SearchHit } from 'src/app/data/search-result'; +import { DocumentService } from 'src/app/services/rest/document.service'; import { SearchService } from 'src/app/services/rest/search.service'; @Component({ @@ -14,6 +17,10 @@ export class SearchComponent implements OnInit { query: string = "" + more_like: number + + more_like_doc: PaperlessDocument + searching = false currentPage = 1 @@ -26,11 +33,24 @@ export class SearchComponent implements OnInit { errorMessage: string - constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router) { } + get maxScore() { + return this.results?.length > 0 ? this.results[0].score : 100 + } + + constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router, private documentService: DocumentService) { } ngOnInit(): void { this.route.queryParamMap.subscribe(paramMap => { + window.scrollTo(0, 0) this.query = paramMap.get('query') + this.more_like = paramMap.has('more_like') ? +paramMap.get('more_like') : null + if (this.more_like) { + this.documentService.get(this.more_like).subscribe(r => { + this.more_like_doc = r + }) + } else { + this.more_like_doc = null + } this.searching = true this.currentPage = 1 this.loadPage() @@ -39,13 +59,14 @@ export class SearchComponent implements OnInit { } searchCorrectedQuery() { - this.router.navigate(["search"], {queryParams: {query: this.correctedQuery}}) + this.router.navigate(["search"], {queryParams: {query: this.correctedQuery, more_like: this.more_like}}) } loadPage(append: boolean = false) { this.errorMessage = null this.correctedQuery = null - this.searchService.search(this.query, this.currentPage).subscribe(result => { + + this.searchService.search(this.query, this.currentPage, this.more_like).subscribe(result => { if (append) { this.results.push(...result.results) } else { diff --git a/src-ui/src/app/data/paperless-tag.ts b/src-ui/src/app/data/paperless-tag.ts index 551c6e03a..979a200a8 100644 --- a/src-ui/src/app/data/paperless-tag.ts +++ b/src-ui/src/app/data/paperless-tag.ts @@ -6,14 +6,14 @@ export const TAG_COLOURS = [ {id: 1, value: "#a6cee3", name: "Light Blue", textColor: "#000000"}, {id: 2, value: "#1f78b4", name: "Blue", textColor: "#ffffff"}, {id: 3, value: "#b2df8a", name: "Light Green", textColor: "#000000"}, - {id: 4, value: "#33a02c", name: "Green", textColor: "#000000"}, + {id: 4, value: "#33a02c", name: "Green", textColor: "#ffffff"}, {id: 5, value: "#fb9a99", name: "Light Red", textColor: "#000000"}, {id: 6, value: "#e31a1c", name: "Red ", textColor: "#ffffff"}, {id: 7, value: "#fdbf6f", name: "Light Orange", textColor: "#000000"}, {id: 8, value: "#ff7f00", name: "Orange", textColor: "#000000"}, {id: 9, value: "#cab2d6", name: "Light Violet", textColor: "#000000"}, {id: 10, value: "#6a3d9a", name: "Violet", textColor: "#ffffff"}, - {id: 11, value: "#b15928", name: "Brown", textColor: "#000000"}, + {id: 11, value: "#b15928", name: "Brown", textColor: "#ffffff"}, {id: 12, value: "#000000", name: "Black", textColor: "#ffffff"}, {id: 13, value: "#cccccc", name: "Light Grey", textColor: "#000000"} ] diff --git a/src-ui/src/app/interceptors/csrf.interceptor.ts b/src-ui/src/app/interceptors/csrf.interceptor.ts index 32f3e99dc..2c654aa36 100644 --- a/src-ui/src/app/interceptors/csrf.interceptor.ts +++ b/src-ui/src/app/interceptors/csrf.interceptor.ts @@ -7,16 +7,21 @@ import { } from '@angular/common/http'; import { Observable } from 'rxjs'; import { CookieService } from 'ngx-cookie-service'; +import { Meta } from '@angular/platform-browser'; @Injectable() export class CsrfInterceptor implements HttpInterceptor { - constructor(private cookieService: CookieService) { + constructor(private cookieService: CookieService, private meta: Meta) { } intercept(request: HttpRequest, next: HttpHandler): Observable> { - let csrfToken = this.cookieService.get('csrftoken') + let prefix = "" + if (this.meta.getTag('name=cookie_prefix')) { + prefix = this.meta.getTag('name=cookie_prefix').content + } + let csrfToken = this.cookieService.get(`${prefix?prefix:''}csrftoken`) if (csrfToken) { request = request.clone({ setHeaders: { diff --git a/src-ui/src/app/services/document-list-view.service.ts b/src-ui/src/app/services/document-list-view.service.ts index 57d0a3f0e..b148d4087 100644 --- a/src-ui/src/app/services/document-list-view.service.ts +++ b/src-ui/src/app/services/document-list-view.service.ts @@ -40,10 +40,14 @@ export class DocumentListViewService { } set savedView(value: PaperlessSavedView) { - if (value) { + if (value && !this._savedViewConfig || value && value.id != this._savedViewConfig.id) { + //saved view inactive and should be active now, or saved view active, but a different view is requested //this is here so that we don't modify value, which might be the actual instance of the saved view. + this.selectNone() this._savedViewConfig = Object.assign({}, value) - } else { + } else if (this._savedViewConfig && !value) { + //saved view active, but document list requested + this.selectNone() this._savedViewConfig = null } } @@ -90,7 +94,7 @@ export class DocumentListViewService { reload(onFinish?) { this.isReloading = true - this.documentService.list( + this.documentService.listFiltered( this.currentPage, this.currentPageSize, this.view.sort_field, @@ -118,6 +122,7 @@ export class DocumentListViewService { //want changes in the filter editor to propagate into here right away. this.view.filter_rules = filterRules this.reload() + this.reduceSelectionToFilter() this.saveDocumentListView() } @@ -191,6 +196,49 @@ export class DocumentListViewService { } } + selected = new Set() + + selectNone() { + this.selected.clear() + } + + private reduceSelectionToFilter() { + if (this.selected.size > 0) { + this.documentService.listAllFilteredIds(this.filterRules).subscribe(ids => { + let subset = new Set() + for (let id of ids) { + if (this.selected.has(id)) { + subset.add(id) + } + } + this.selected = subset + }) + } + } + + selectAll() { + this.documentService.listAllFilteredIds(this.filterRules).subscribe(ids => ids.forEach(id => this.selected.add(id))) + } + + selectPage() { + this.selected.clear() + this.documents.forEach(doc => { + this.selected.add(doc.id) + }) + } + + isSelected(d: PaperlessDocument) { + return this.selected.has(d.id) + } + + setSelected(d: PaperlessDocument, value: boolean) { + if (value) { + this.selected.add(d.id) + } else if (!value) { + this.selected.delete(d.id) + } + } + constructor(private documentService: DocumentService) { let documentListViewConfigJson = sessionStorage.getItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG) if (documentListViewConfigJson) { diff --git a/src-ui/src/app/services/open-documents.service.ts b/src-ui/src/app/services/open-documents.service.ts index e37f5db8c..c91031f83 100644 --- a/src-ui/src/app/services/open-documents.service.ts +++ b/src-ui/src/app/services/open-documents.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@angular/core'; import { PaperlessDocument } from '../data/paperless-document'; import { OPEN_DOCUMENT_SERVICE } from '../data/storage-keys'; +import { DocumentService } from './rest/document.service'; @Injectable({ providedIn: 'root' @@ -9,7 +10,7 @@ export class OpenDocumentsService { private MAX_OPEN_DOCUMENTS = 5 - constructor() { + constructor(private documentService: DocumentService) { if (sessionStorage.getItem(OPEN_DOCUMENT_SERVICE.DOCUMENTS)) { try { this.openDocuments = JSON.parse(sessionStorage.getItem(OPEN_DOCUMENT_SERVICE.DOCUMENTS)) @@ -22,6 +23,15 @@ export class OpenDocumentsService { private openDocuments: PaperlessDocument[] = [] + refreshDocument(id: number) { + let index = this.openDocuments.findIndex(doc => doc.id == id) + if (index > -1) { + this.documentService.get(id).subscribe(doc => { + this.openDocuments[index] = doc + }) + } + } + getOpenDocuments(): PaperlessDocument[] { return this.openDocuments } diff --git a/src-ui/src/app/services/rest/abstract-paperless-service.ts b/src-ui/src/app/services/rest/abstract-paperless-service.ts index 93e1a0c85..f57956754 100644 --- a/src-ui/src/app/services/rest/abstract-paperless-service.ts +++ b/src-ui/src/app/services/rest/abstract-paperless-service.ts @@ -52,9 +52,9 @@ export abstract class AbstractPaperlessService { private _listAll: Observable> - listAll(ordering?: string, extraParams?): Observable> { + listAll(sortField?: string, sortReverse?: boolean, extraParams?): Observable> { if (!this._listAll) { - this._listAll = this.list(1, 100000, ordering, extraParams).pipe( + this._listAll = this.list(1, 100000, sortField, sortReverse, extraParams).pipe( publishReplay(1), refCount() ) diff --git a/src-ui/src/app/services/rest/document.service.ts b/src-ui/src/app/services/rest/document.service.ts index f50620d23..c132fa856 100644 --- a/src-ui/src/app/services/rest/document.service.ts +++ b/src-ui/src/app/services/rest/document.service.ts @@ -61,8 +61,8 @@ export class DocumentService extends AbstractPaperlessService return doc } - list(page?: number, pageSize?: number, sortField?: string, sortReverse?: boolean, filterRules?: FilterRule[]): Observable> { - return super.list(page, pageSize, sortField, sortReverse, this.filterRulesToQueryParams(filterRules)).pipe( + listFiltered(page?: number, pageSize?: number, sortField?: string, sortReverse?: boolean, filterRules?: FilterRule[], extraParams = {}): Observable> { + return this.list(page, pageSize, sortField, sortReverse, Object.assign(extraParams, this.filterRulesToQueryParams(filterRules))).pipe( map(results => { results.results.forEach(doc => this.addObservablesToDocument(doc)) return results @@ -70,6 +70,12 @@ export class DocumentService extends AbstractPaperlessService ) } + listAllFilteredIds(filterRules?: FilterRule[]): Observable { + return this.listFiltered(1, 100000, null, null, filterRules, {"fields": "id"}).pipe( + map(response => response.results.map(doc => doc.id)) + ) + } + getPreviewUrl(id: number, original: boolean = false): string { let url = this.getResourceUrl(id, 'preview') if (original) { @@ -98,4 +104,12 @@ export class DocumentService extends AbstractPaperlessService return this.http.get(this.getResourceUrl(id, 'metadata')) } + bulkEdit(ids: number[], method: string, args: any) { + return this.http.post(this.getResourceUrl(null, 'bulk_edit'), { + 'documents': ids, + 'method': method, + 'parameters': args + }) + } + } diff --git a/src-ui/src/app/services/rest/search.service.ts b/src-ui/src/app/services/rest/search.service.ts index b19a55769..3799f3dc7 100644 --- a/src-ui/src/app/services/rest/search.service.ts +++ b/src-ui/src/app/services/rest/search.service.ts @@ -15,11 +15,17 @@ export class SearchService { constructor(private http: HttpClient, private documentService: DocumentService) { } - search(query: string, page?: number): Observable { - let httpParams = new HttpParams().set('query', query) + search(query: string, page?: number, more_like?: number): Observable { + let httpParams = new HttpParams() + if (query) { + httpParams = httpParams.set('query', query) + } if (page) { httpParams = httpParams.set('page', page.toString()) } + if (more_like) { + httpParams = httpParams.set('more_like', more_like.toString()) + } return this.http.get(`${environment.apiBaseUrl}search/`, {params: httpParams}).pipe( map(result => { result.results.forEach(hit => this.documentService.addObservablesToDocument(hit.document)) diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts index f12c6a7cb..ab6b07c73 100644 --- a/src-ui/src/environments/environment.prod.ts +++ b/src-ui/src/environments/environment.prod.ts @@ -2,5 +2,5 @@ export const environment = { production: true, apiBaseUrl: "/api/", appTitle: "Paperless-ng", - version: "0.9.8" + version: "0.9.9" }; diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index b0b66b7f9..6e09db630 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -1,7 +1,6 @@ @import "theme"; - @import "node_modules/bootstrap/scss/bootstrap"; - +@import "~@ng-select/ng-select/themes/default.theme.css"; .toolbaricon { width: 1.2em; @@ -20,7 +19,7 @@ } body { - font-size: .875rem; + font-size: 0.875rem; } .form-control-dark { @@ -65,4 +64,39 @@ body { display: block; background-size: 1rem; float: right; -} \ No newline at end of file +} + +.paperless-input-select { + .ng-select { + position: relative; + flex: 1 1 auto; + margin-bottom: 0; + min-height: calc(1.5em + 0.75rem + 5px); + line-height: 1.5; + + .ng-select-container { + height: 100%; + border-top-right-radius: 0; + border-bottom-right-radius: 0; + + .ng-value-container .ng-input { + top: 10px; + } + } + + .ng-dropdown-panel .ng-dropdown-panel-items .ng-option.ng-option-selected, + .ng-dropdown-panel .ng-dropdown-panel-items .ng-option.ng-option-selected.ng-option-marked { + background: none; + } + } +} + +.paperless-input-tags { + .ng-select.ng-select-multiple .ng-select-container .ng-value-container .ng-value { + background-color: transparent; + } + + .ng-select.ng-select-multiple .ng-select-container .ng-value-container { + padding-top: 1px; + } +} diff --git a/src-ui/src/theme.scss b/src-ui/src/theme.scss index 88f3ae30f..df2aea003 100644 --- a/src-ui/src/theme.scss +++ b/src-ui/src/theme.scss @@ -1,5 +1,6 @@ $paperless-green: #17541f; $primary: #17541f; +$primaryFaded: #d1ddd2; $theme-colors: ( "primary": $primary diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py new file mode 100644 index 000000000..aa5b8ea3f --- /dev/null +++ b/src/documents/bulk_edit.py @@ -0,0 +1,72 @@ +from django.db.models import Q +from django_q.tasks import async_task + +from documents.models import Document, Correspondent, DocumentType + + +def set_correspondent(doc_ids, correspondent): + if correspondent: + correspondent = Correspondent.objects.get(id=correspondent) + + qs = Document.objects.filter( + Q(id__in=doc_ids) & ~Q(correspondent=correspondent)) + affected_docs = [doc.id for doc in qs] + qs.update(correspondent=correspondent) + + async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs) + + return "OK" + + +def set_document_type(doc_ids, document_type): + if document_type: + document_type = DocumentType.objects.get(id=document_type) + + qs = Document.objects.filter( + Q(id__in=doc_ids) & ~Q(document_type=document_type)) + affected_docs = [doc.id for doc in qs] + qs.update(document_type=document_type) + + async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs) + + return "OK" + + +def add_tag(doc_ids, tag): + + qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(tags__id=tag)) + affected_docs = [doc.id for doc in qs] + + DocumentTagRelationship = Document.tags.through + + DocumentTagRelationship.objects.bulk_create([ + DocumentTagRelationship( + document_id=doc, tag_id=tag) for doc in affected_docs + ]) + + async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs) + + return "OK" + + +def remove_tag(doc_ids, tag): + + qs = Document.objects.filter(Q(id__in=doc_ids) & Q(tags__id=tag)) + affected_docs = [doc.id for doc in qs] + + DocumentTagRelationship = Document.tags.through + + DocumentTagRelationship.objects.filter( + Q(document_id__in=affected_docs) & + Q(tag_id=tag) + ).delete() + + async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs) + + return "OK" + + +def delete(doc_ids): + Document.objects.filter(id__in=doc_ids).delete() + + return "OK" diff --git a/src/documents/checks.py b/src/documents/checks.py index b6da5bfc9..ba55b1397 100644 --- a/src/documents/checks.py +++ b/src/documents/checks.py @@ -51,6 +51,6 @@ def parser_check(app_configs, **kwargs): if len(parsers) == 0: return [Error("No parsers found. This is a bug. The consumer won't be " - "able to onsume any documents without parsers.")] + "able to consume any documents without parsers.")] else: return [] diff --git a/src/documents/consumer.py b/src/documents/consumer.py index e4da51f1d..ab4912a36 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -247,7 +247,6 @@ class Consumer(LoggingMixin): with open(self.path, "rb") as f: document = Document.objects.create( - correspondent=file_info.correspondent, title=(self.override_title or file_info.title)[:127], content=text, mime_type=mime_type, @@ -257,12 +256,6 @@ class Consumer(LoggingMixin): storage_type=storage_type ) - relevant_tags = set(file_info.tags) - if relevant_tags: - tag_names = ", ".join([t.name for t in relevant_tags]) - self.log("debug", "Tagging with {}".format(tag_names)) - document.tags.add(*relevant_tags) - self.apply_overrides(document) document.save() diff --git a/src/documents/index.py b/src/documents/index.py index 53bf34542..308ee932e 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -3,7 +3,7 @@ import os from contextlib import contextmanager from django.conf import settings -from whoosh import highlight +from whoosh import highlight, classify, query from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME from whoosh.highlight import Formatter, get_text from whoosh.index import create_in, exists_in, open_dir @@ -20,32 +20,37 @@ class JsonFormatter(Formatter): self.seen = {} def format_token(self, text, token, replace=False): - seen = self.seen ttext = self._text(get_text(text, token, replace)) - if ttext in seen: - termnum = seen[ttext] - else: - termnum = len(seen) - seen[ttext] = termnum - - return {'text': ttext, 'term': termnum} + return {'text': ttext, 'highlight': 'true'} def format_fragment(self, fragment, replace=False): output = [] index = fragment.startchar text = fragment.text - + amend_token = None for t in fragment.matches: if t.startchar is None: continue if t.startchar < index: continue if t.startchar > index: - output.append({'text': text[index:t.startchar]}) - output.append(self.format_token(text, t, replace)) + text_inbetween = text[index:t.startchar] + if amend_token and t.startchar - index < 10: + amend_token['text'] += text_inbetween + else: + output.append({'text': text_inbetween, + 'highlight': False}) + amend_token = None + token = self.format_token(text, t, replace) + if amend_token: + amend_token['text'] += token['text'] + else: + output.append(token) + amend_token = token index = t.endchar if index < fragment.endchar: - output.append({'text': text[index:fragment.endchar]}) + output.append({'text': text[index:fragment.endchar], + 'highlight': False}) return output def format(self, fragments, replace=False): @@ -120,22 +125,42 @@ def remove_document_from_index(document): @contextmanager -def query_page(ix, querystring, page): +def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content): searcher = ix.searcher() try: - qp = MultifieldParser( - ["content", "title", "correspondent", "tag", "type"], - ix.schema) - qp.add_plugin(DateParserPlugin()) + if querystring: + qp = MultifieldParser( + ["content", "title", "correspondent", "tag", "type"], + ix.schema) + qp.add_plugin(DateParserPlugin()) + str_q = qp.parse(querystring) + corrected = searcher.correct_query(str_q, querystring) + else: + str_q = None + corrected = None + + if more_like_doc_id: + docnum = searcher.document_number(id=more_like_doc_id) + kts = searcher.key_terms_from_text( + 'content', more_like_doc_content, numterms=20, + model=classify.Bo1Model, normalize=False) + more_like_q = query.Or( + [query.Term('content', word, boost=weight) + for word, weight in kts]) + result_page = searcher.search_page( + more_like_q, page, filter=str_q, mask={docnum}) + elif str_q: + result_page = searcher.search_page(str_q, page) + else: + raise ValueError( + "Either querystring or more_like_doc_id is required." + ) - q = qp.parse(querystring) - result_page = searcher.search_page(q, page) result_page.results.fragmenter = highlight.ContextFragmenter( surround=50) result_page.results.formatter = JsonFormatter() - corrected = searcher.correct_query(q, querystring) - if corrected.query != q: + if corrected and corrected.query != str_q: corrected_query = corrected.string else: corrected_query = None diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 70d05d98b..6df14a82c 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -1,18 +1,31 @@ import json +import logging import os import shutil +from contextlib import contextmanager +import tqdm from django.conf import settings from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError +from django.db.models.signals import post_save, m2m_changed from filelock import FileLock from documents.models import Document from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME, \ EXPORTER_ARCHIVE_NAME -from ...file_handling import create_source_path_directory, \ - generate_unique_filename +from ...file_handling import create_source_path_directory from ...mixins import Renderable +from ...signals.handlers import update_filename_and_move_files + + +@contextmanager +def disable_signal(sig, receiver, sender): + try: + sig.disconnect(receiver=receiver, sender=sender) + yield + finally: + sig.connect(receiver=receiver, sender=sender) class Command(Renderable, BaseCommand): @@ -32,6 +45,8 @@ class Command(Renderable, BaseCommand): def handle(self, *args, **options): + logging.getLogger().handlers[0].level = logging.ERROR + self.source = options["source"] if not os.path.exists(self.source): @@ -47,11 +62,19 @@ class Command(Renderable, BaseCommand): self.manifest = json.load(f) self._check_manifest() + with disable_signal(post_save, + receiver=update_filename_and_move_files, + sender=Document): + with disable_signal(m2m_changed, + receiver=update_filename_and_move_files, + sender=Document.tags.through): + # Fill up the database with whatever is in the manifest + call_command("loaddata", manifest_path) - # Fill up the database with whatever is in the manifest - call_command("loaddata", manifest_path) + self._import_files_from_manifest() - self._import_files_from_manifest() + print("Updating search index...") + call_command('document_index', 'reindex') @staticmethod def _check_manifest_exists(path): @@ -95,10 +118,13 @@ class Command(Renderable, BaseCommand): os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True) os.makedirs(settings.ARCHIVE_DIR, exist_ok=True) - for record in self.manifest: + print("Copy files into paperless...") - if not record["model"] == "documents.document": - continue + manifest_documents = list(filter( + lambda r: r["model"] == "documents.document", + self.manifest)) + + for record in tqdm.tqdm(manifest_documents): document = Document.objects.get(pk=record["pk"]) @@ -117,15 +143,11 @@ class Command(Renderable, BaseCommand): document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED with FileLock(settings.MEDIA_LOCK): - document.filename = generate_unique_filename( - document, settings.ORIGINALS_DIR) - if os.path.isfile(document.source_path): raise FileExistsError(document.source_path) create_source_path_directory(document.source_path) - print(f"Moving {document_path} to {document.source_path}") shutil.copy(document_path, document.source_path) shutil.copy(thumbnail_path, document.thumbnail_path) if archive_path: diff --git a/src/documents/management/commands/document_index.py b/src/documents/management/commands/document_index.py index 7dfdbaa42..08e20e1d2 100644 --- a/src/documents/management/commands/document_index.py +++ b/src/documents/management/commands/document_index.py @@ -1,4 +1,5 @@ from django.core.management import BaseCommand +from django.db import transaction from documents.mixins import Renderable from documents.tasks import index_reindex, index_optimize @@ -18,8 +19,8 @@ class Command(Renderable, BaseCommand): def handle(self, *args, **options): self.verbosity = options["verbosity"] - - if options['command'] == 'reindex': - index_reindex() - elif options['command'] == 'optimize': - index_optimize() + with transaction.atomic(): + if options['command'] == 'reindex': + index_reindex() + elif options['command'] == 'optimize': + index_optimize() diff --git a/src/documents/migrations/1003_mime_types.py b/src/documents/migrations/1003_mime_types.py index 78ecced2b..c196f29f4 100644 --- a/src/documents/migrations/1003_mime_types.py +++ b/src/documents/migrations/1003_mime_types.py @@ -11,6 +11,7 @@ from paperless.db import GnuPG STORAGE_TYPE_UNENCRYPTED = "unencrypted" STORAGE_TYPE_GPG = "gpg" + def source_path(self): if self.filename: fname = str(self.filename) diff --git a/src/documents/models.py b/src/documents/models.py index 3a6d155ed..168dd8c7b 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -357,54 +357,12 @@ class SavedViewFilterRule(models.Model): # TODO: why is this in the models file? class FileInfo: - # This epic regex *almost* worked for our needs, so I'm keeping it here for - # posterity, in the hopes that we might find a way to make it work one day. - ALMOST_REGEX = re.compile( - r"^((?P\d\d\d\d\d\d\d\d\d\d\d\d\d\dZ){separator})?" - r"((?P{non_separated_word}+){separator})??" - r"(?P{non_separated_word}+)" - r"({separator}(?P<tags>[a-z,0-9-]+))?" - r"\.(?P<extension>[a-zA-Z.-]+)$".format( - separator=r"\s+-\s+", - non_separated_word=r"([\w,. ]|([^\s]-))" - ) - ) REGEXES = OrderedDict([ - ("created-correspondent-title-tags", re.compile( - r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " - r"(?P<correspondent>.*) - " - r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)$", - flags=re.IGNORECASE - )), - ("created-title-tags", re.compile( - r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " - r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)$", - flags=re.IGNORECASE - )), - ("created-correspondent-title", re.compile( - r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " - r"(?P<correspondent>.*) - " - r"(?P<title>.*)$", - flags=re.IGNORECASE - )), ("created-title", re.compile( r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " r"(?P<title>.*)$", flags=re.IGNORECASE )), - ("correspondent-title-tags", re.compile( - r"(?P<correspondent>.*) - " - r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)$", - flags=re.IGNORECASE - )), - ("correspondent-title", re.compile( - r"(?P<correspondent>.*) - " - r"(?P<title>.*)?$", - flags=re.IGNORECASE - )), ("title", re.compile( r"(?P<title>.*)$", flags=re.IGNORECASE @@ -427,23 +385,10 @@ class FileInfo: except ValueError: return None - @classmethod - def _get_correspondent(cls, name): - if not name: - return None - return Correspondent.objects.get_or_create(name=name)[0] - @classmethod def _get_title(cls, title): return title - @classmethod - def _get_tags(cls, tags): - r = [] - for t in tags.split(","): - r.append(Tag.objects.get_or_create(name=t)[0]) - return tuple(r) - @classmethod def _mangle_property(cls, properties, name): if name in properties: @@ -453,15 +398,6 @@ class FileInfo: @classmethod def from_filename(cls, filename): - """ - We use a crude naming convention to make handling the correspondent, - title, and tags easier: - "<date> - <correspondent> - <title> - <tags>" - "<correspondent> - <title> - <tags>" - "<correspondent> - <title>" - "<title>" - """ - # Mutate filename in-place before parsing its components # by applying at most one of the configured transformations. for (pattern, repl) in settings.FILENAME_PARSE_TRANSFORMS: @@ -492,7 +428,5 @@ class FileInfo: if m: properties = m.groupdict() cls._mangle_property(properties, "created") - cls._mangle_property(properties, "correspondent") cls._mangle_property(properties, "title") - cls._mangle_property(properties, "tags") return cls(**properties) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index ee0a42384..f9fde9ac8 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -3,11 +3,34 @@ from django.utils.text import slugify from rest_framework import serializers from rest_framework.fields import SerializerMethodField +from . import bulk_edit from .models import Correspondent, Tag, Document, Log, DocumentType, \ SavedView, SavedViewFilterRule from .parsers import is_mime_type_supported +# https://www.django-rest-framework.org/api-guide/serializers/#example +class DynamicFieldsModelSerializer(serializers.ModelSerializer): + """ + A ModelSerializer that takes an additional `fields` argument that + controls which fields should be displayed. + """ + + def __init__(self, *args, **kwargs): + # Don't pass the 'fields' arg up to the superclass + fields = kwargs.pop('fields', None) + + # Instantiate the superclass normally + super(DynamicFieldsModelSerializer, self).__init__(*args, **kwargs) + + if fields is not None: + # Drop any fields that are not specified in the `fields` argument. + allowed = set(fields) + existing = set(self.fields) + for field_name in existing - allowed: + self.fields.pop(field_name) + + class CorrespondentSerializer(serializers.ModelSerializer): document_count = serializers.IntegerField(read_only=True) @@ -91,7 +114,7 @@ class DocumentTypeField(serializers.PrimaryKeyRelatedField): return DocumentType.objects.all() -class DocumentSerializer(serializers.ModelSerializer): +class DocumentSerializer(DynamicFieldsModelSerializer): correspondent = CorrespondentField(allow_null=True) tags = TagsField(many=True) @@ -180,6 +203,54 @@ class SavedViewSerializer(serializers.ModelSerializer): return saved_view +class BulkEditSerializer(serializers.Serializer): + + documents = serializers.ListField( + child=serializers.IntegerField(), + label="Documents", + write_only=True + ) + + method = serializers.ChoiceField( + choices=[ + "set_correspondent", + "set_document_type", + "add_tag", + "remove_tag", + "delete" + ], + label="Method", + write_only=True, + ) + + parameters = serializers.DictField(allow_empty=True) + + def validate_documents(self, documents): + count = Document.objects.filter(id__in=documents).count() + if not count == len(documents): + raise serializers.ValidationError( + "Some documents don't exist or were specified twice.") + return documents + + def validate_method(self, method): + if method == "set_correspondent": + return bulk_edit.set_correspondent + elif method == "set_document_type": + return bulk_edit.set_document_type + elif method == "add_tag": + return bulk_edit.add_tag + elif method == "remove_tag": + return bulk_edit.remove_tag + elif method == "delete": + return bulk_edit.delete + else: + raise serializers.ValidationError("Unsupported method.") + + def validate(self, attrs): + + return attrs + + class PostDocumentSerializer(serializers.Serializer): document = serializers.FileField( diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 8c9b00dd6..fafe6e10f 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -2,6 +2,7 @@ import logging import tqdm from django.conf import settings +from django.db.models.signals import post_save from whoosh.writing import AsyncWriter from documents import index, sanity_checker @@ -87,3 +88,9 @@ def sanity_check(): raise SanityFailedError(messages) else: return "No issues detected." + + +def bulk_rename_files(document_ids): + qs = Document.objects.filter(id__in=document_ids) + for doc in qs: + post_save.send(Document, instance=doc, created=False) diff --git a/src/documents/templates/index.html b/src/documents/templates/index.html index 728f3a0e7..47a352cd5 100644 --- a/src/documents/templates/index.html +++ b/src/documents/templates/index.html @@ -5,9 +5,12 @@ <html lang="en"> <head> <meta charset="utf-8"> - <title>PaperlessUi + Paperless-ng + + + diff --git a/src/documents/tests/test_admin.py b/src/documents/tests/test_admin.py new file mode 100644 index 000000000..b280c43ea --- /dev/null +++ b/src/documents/tests/test_admin.py @@ -0,0 +1,57 @@ +from unittest import mock + +from django.contrib.admin.sites import AdminSite +from django.test import TestCase +from django.utils import timezone + +from documents.admin import DocumentAdmin +from documents.models import Document, Tag + + +class TestDocumentAdmin(TestCase): + + def setUp(self) -> None: + self.doc_admin = DocumentAdmin(model=Document, admin_site=AdminSite()) + + @mock.patch("documents.admin.index.add_or_update_document") + def test_save_model(self, m): + doc = Document.objects.create(title="test") + doc.title = "new title" + self.doc_admin.save_model(None, doc, None, None) + self.assertEqual(Document.objects.get(id=doc.id).title, "new title") + m.assert_called_once() + + def test_tags(self): + doc = Document.objects.create(title="test") + doc.tags.create(name="t1") + doc.tags.create(name="t2") + + self.assertEqual(self.doc_admin.tags_(doc), "t1, t2, ") + + def test_tags_empty(self): + doc = Document.objects.create(title="test") + + self.assertEqual(self.doc_admin.tags_(doc), "") + + @mock.patch("documents.admin.index.remove_document") + def test_delete_model(self, m): + doc = Document.objects.create(title="test") + self.doc_admin.delete_model(None, doc) + self.assertRaises(Document.DoesNotExist, Document.objects.get, id=doc.id) + m.assert_called_once() + + @mock.patch("documents.admin.index.remove_document") + def test_delete_queryset(self, m): + for i in range(42): + Document.objects.create(title="Many documents with the same title", checksum=f"{i:02}") + + self.assertEqual(Document.objects.count(), 42) + + self.doc_admin.delete_queryset(None, Document.objects.all()) + + self.assertEqual(m.call_count, 42) + self.assertEqual(Document.objects.count(), 0) + + def test_created(self): + doc = Document.objects.create(title="test", created=timezone.datetime(2020, 4, 12)) + self.assertEqual(self.doc_admin.created_(doc), "2020-04-12") diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 49dddee87..bce2a433d 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -1,3 +1,4 @@ +import json import os import shutil import tempfile @@ -7,7 +8,7 @@ from django.contrib.auth.models import User from rest_framework.test import APITestCase from whoosh.writing import AsyncWriter -from documents import index +from documents import index, bulk_edit from documents.models import Document, Correspondent, DocumentType, Tag, SavedView from documents.tests.utils import DirectoriesMixin @@ -63,6 +64,58 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(len(Document.objects.all()), 0) + def test_document_fields(self): + c = Correspondent.objects.create(name="c", pk=41) + dt = DocumentType.objects.create(name="dt", pk=63) + tag = Tag.objects.create(name="t", pk=85) + doc = Document.objects.create(title="WOW", content="the content", correspondent=c, document_type=dt, checksum="123", mime_type="application/pdf") + + response = self.client.get("/api/documents/", format='json') + self.assertEqual(response.status_code, 200) + results_full = response.data['results'] + self.assertTrue("content" in results_full[0]) + self.assertTrue("id" in results_full[0]) + + response = self.client.get("/api/documents/?fields=id", format='json') + self.assertEqual(response.status_code, 200) + results = response.data['results'] + self.assertFalse("content" in results[0]) + self.assertTrue("id" in results[0]) + self.assertEqual(len(results[0]), 1) + + response = self.client.get("/api/documents/?fields=content", format='json') + self.assertEqual(response.status_code, 200) + results = response.data['results'] + self.assertTrue("content" in results[0]) + self.assertFalse("id" in results[0]) + self.assertEqual(len(results[0]), 1) + + response = self.client.get("/api/documents/?fields=id,content", format='json') + self.assertEqual(response.status_code, 200) + results = response.data['results'] + self.assertTrue("content" in results[0]) + self.assertTrue("id" in results[0]) + self.assertEqual(len(results[0]), 2) + + response = self.client.get("/api/documents/?fields=id,conteasdnt", format='json') + self.assertEqual(response.status_code, 200) + results = response.data['results'] + self.assertFalse("content" in results[0]) + self.assertTrue("id" in results[0]) + self.assertEqual(len(results[0]), 1) + + response = self.client.get("/api/documents/?fields=", format='json') + self.assertEqual(response.status_code, 200) + results = response.data['results'] + self.assertEqual(results_full, results) + + response = self.client.get("/api/documents/?fields=dgfhs", format='json') + self.assertEqual(response.status_code, 200) + results = response.data['results'] + self.assertEqual(len(results[0]), 0) + + + def test_document_actions(self): _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir) @@ -351,6 +404,25 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): self.assertEqual(correction, None) + def test_search_more_like(self): + d1=Document.objects.create(title="invoice", content="the thing i bought at a shop and paid with bank account", checksum="A", pk=1) + d2=Document.objects.create(title="bank statement 1", content="things i paid for in august", pk=2, checksum="B") + d3=Document.objects.create(title="bank statement 3", content="things i paid for in september", pk=3, checksum="C") + with AsyncWriter(index.open_index()) as writer: + index.update_document(writer, d1) + index.update_document(writer, d2) + index.update_document(writer, d3) + + response = self.client.get(f"/api/search/?more_like={d2.id}") + + self.assertEqual(response.status_code, 200) + + results = response.data['results'] + + self.assertEqual(len(results), 2) + self.assertEqual(results[0]['id'], d3.id) + self.assertEqual(results[1]['id'], d1.id) + def test_statistics(self): doc1 = Document.objects.create(title="none1", checksum="A") @@ -595,3 +667,237 @@ class TestDocumentApi(DirectoriesMixin, APITestCase): v1 = SavedView.objects.get(id=v1.id) self.assertEqual(v1.filter_rules.count(), 0) + + +class TestBulkEdit(DirectoriesMixin, APITestCase): + + def setUp(self): + super(TestBulkEdit, self).setUp() + + user = User.objects.create_superuser(username="temp_admin") + self.client.force_login(user=user) + + patcher = mock.patch('documents.bulk_edit.async_task') + self.async_task = patcher.start() + self.addCleanup(patcher.stop) + self.c1 = Correspondent.objects.create(name="c1") + self.c2 = Correspondent.objects.create(name="c2") + self.dt1 = DocumentType.objects.create(name="dt1") + self.dt2 = DocumentType.objects.create(name="dt2") + self.t1 = Tag.objects.create(name="t1") + self.t2 = Tag.objects.create(name="t2") + self.doc1 = Document.objects.create(checksum="A", title="A") + self.doc2 = Document.objects.create(checksum="B", title="B", correspondent=self.c1, document_type=self.dt1) + self.doc3 = Document.objects.create(checksum="C", title="C", correspondent=self.c2, document_type=self.dt2) + self.doc4 = Document.objects.create(checksum="D", title="D") + self.doc5 = Document.objects.create(checksum="E", title="E") + self.doc2.tags.add(self.t1) + self.doc3.tags.add(self.t2) + self.doc4.tags.add(self.t1, self.t2) + + def test_set_correspondent(self): + self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 1) + bulk_edit.set_correspondent([self.doc1.id, self.doc2.id, self.doc3.id], self.c2.id) + self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 3) + self.async_task.assert_called_once() + args, kwargs = self.async_task.call_args + self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc2.id]) + + def test_unset_correspondent(self): + self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 1) + bulk_edit.set_correspondent([self.doc1.id, self.doc2.id, self.doc3.id], None) + self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 0) + self.async_task.assert_called_once() + args, kwargs = self.async_task.call_args + self.assertCountEqual(kwargs['document_ids'], [self.doc2.id, self.doc3.id]) + + def test_set_document_type(self): + self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 1) + bulk_edit.set_document_type([self.doc1.id, self.doc2.id, self.doc3.id], self.dt2.id) + self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 3) + self.async_task.assert_called_once() + args, kwargs = self.async_task.call_args + self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc2.id]) + + def test_unset_document_type(self): + self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 1) + bulk_edit.set_document_type([self.doc1.id, self.doc2.id, self.doc3.id], None) + self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 0) + self.async_task.assert_called_once() + args, kwargs = self.async_task.call_args + self.assertCountEqual(kwargs['document_ids'], [self.doc2.id, self.doc3.id]) + + def test_add_tag(self): + self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2) + bulk_edit.add_tag([self.doc1.id, self.doc2.id, self.doc3.id, self.doc4.id], self.t1.id) + self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 4) + self.async_task.assert_called_once() + args, kwargs = self.async_task.call_args + self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc3.id]) + + def test_remove_tag(self): + self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2) + bulk_edit.remove_tag([self.doc1.id, self.doc3.id, self.doc4.id], self.t1.id) + self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 1) + self.async_task.assert_called_once() + args, kwargs = self.async_task.call_args + self.assertCountEqual(kwargs['document_ids'], [self.doc4.id]) + + def test_delete(self): + self.assertEqual(Document.objects.count(), 5) + bulk_edit.delete([self.doc1.id, self.doc2.id]) + self.assertEqual(Document.objects.count(), 3) + self.assertCountEqual([doc.id for doc in Document.objects.all()], [self.doc3.id, self.doc4.id, self.doc5.id]) + + @mock.patch("documents.serialisers.bulk_edit.set_correspondent") + def test_api_set_correspondent(self, m): + m.return_value = "OK" + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc1.id], + "method": "set_correspondent", + "parameters": {"correspondent": self.c1.id} + }), content_type='application/json') + self.assertEqual(response.status_code, 200) + m.assert_called_once() + args, kwargs = m.call_args + self.assertEqual(args[0], [self.doc1.id]) + self.assertEqual(kwargs['correspondent'], self.c1.id) + + @mock.patch("documents.serialisers.bulk_edit.set_correspondent") + def test_api_unset_correspondent(self, m): + m.return_value = "OK" + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc1.id], + "method": "set_correspondent", + "parameters": {"correspondent": None} + }), content_type='application/json') + self.assertEqual(response.status_code, 200) + m.assert_called_once() + args, kwargs = m.call_args + self.assertEqual(args[0], [self.doc1.id]) + self.assertIsNone(kwargs['correspondent']) + + @mock.patch("documents.serialisers.bulk_edit.set_document_type") + def test_api_set_type(self, m): + m.return_value = "OK" + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc1.id], + "method": "set_document_type", + "parameters": {"document_type": self.dt1.id} + }), content_type='application/json') + self.assertEqual(response.status_code, 200) + m.assert_called_once() + args, kwargs = m.call_args + self.assertEqual(args[0], [self.doc1.id]) + self.assertEqual(kwargs['document_type'], self.dt1.id) + + @mock.patch("documents.serialisers.bulk_edit.set_document_type") + def test_api_unset_type(self, m): + m.return_value = "OK" + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc1.id], + "method": "set_document_type", + "parameters": {"document_type": None} + }), content_type='application/json') + self.assertEqual(response.status_code, 200) + m.assert_called_once() + args, kwargs = m.call_args + self.assertEqual(args[0], [self.doc1.id]) + self.assertIsNone(kwargs['document_type']) + + @mock.patch("documents.serialisers.bulk_edit.add_tag") + def test_api_add_tag(self, m): + m.return_value = "OK" + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc1.id], + "method": "add_tag", + "parameters": {"tag": self.t1.id} + }), content_type='application/json') + self.assertEqual(response.status_code, 200) + m.assert_called_once() + args, kwargs = m.call_args + self.assertEqual(args[0], [self.doc1.id]) + self.assertEqual(kwargs['tag'], self.t1.id) + + @mock.patch("documents.serialisers.bulk_edit.remove_tag") + def test_api_remove_tag(self, m): + m.return_value = "OK" + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc1.id], + "method": "remove_tag", + "parameters": {"tag": self.t1.id} + }), content_type='application/json') + self.assertEqual(response.status_code, 200) + m.assert_called_once() + args, kwargs = m.call_args + self.assertEqual(args[0], [self.doc1.id]) + self.assertEqual(kwargs['tag'], self.t1.id) + + @mock.patch("documents.serialisers.bulk_edit.delete") + def test_api_delete(self, m): + m.return_value = "OK" + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc1.id], + "method": "delete", + "parameters": {} + }), content_type='application/json') + self.assertEqual(response.status_code, 200) + m.assert_called_once() + args, kwargs = m.call_args + self.assertEqual(args[0], [self.doc1.id]) + self.assertEqual(len(kwargs), 0) + + def test_api_invalid_doc(self): + self.assertEqual(Document.objects.count(), 5) + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [-235], + "method": "delete", + "parameters": {} + }), content_type='application/json') + self.assertEqual(response.status_code, 400) + self.assertEqual(Document.objects.count(), 5) + + def test_api_invalid_method(self): + self.assertEqual(Document.objects.count(), 5) + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc2.id], + "method": "exterminate", + "parameters": {} + }), content_type='application/json') + self.assertEqual(response.status_code, 400) + self.assertEqual(Document.objects.count(), 5) + + def test_api_invalid_correspondent(self): + self.assertEqual(self.doc2.correspondent, self.c1) + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc2.id], + "method": "set_correspondent", + "parameters": {'correspondent': 345657} + }), content_type='application/json') + self.assertEqual(response.status_code, 400) + + doc2 = Document.objects.get(id=self.doc2.id) + self.assertEqual(doc2.correspondent, self.c1) + + def test_api_invalid_document_type(self): + self.assertEqual(self.doc2.document_type, self.dt1) + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc2.id], + "method": "set_document_type", + "parameters": {'document_type': 345657} + }), content_type='application/json') + self.assertEqual(response.status_code, 400) + + doc2 = Document.objects.get(id=self.doc2.id) + self.assertEqual(doc2.document_type, self.dt1) + + def test_api_invalid_tag(self): + self.assertEqual(list(self.doc2.tags.all()), [self.t1]) + response = self.client.post("/api/documents/bulk_edit/", json.dumps({ + "documents": [self.doc2.id], + "method": "add_tag", + "parameters": {'document_type': 345657} + }), content_type='application/json') + self.assertEqual(response.status_code, 400) + + self.assertEqual(list(self.doc2.tags.all()), [self.t1]) diff --git a/src/documents/tests/test_checks.py b/src/documents/tests/test_checks.py index 1027c11a0..ee4fbe8d1 100644 --- a/src/documents/tests/test_checks.py +++ b/src/documents/tests/test_checks.py @@ -1,9 +1,12 @@ import unittest +from unittest import mock +from django.core.checks import Error from django.test import TestCase from .factories import DocumentFactory -from ..checks import changed_password_check +from .. import document_consumer_declaration +from ..checks import changed_password_check, parser_check from ..models import Document @@ -15,3 +18,13 @@ class ChecksTestCase(TestCase): def test_changed_password_check_no_encryption(self): DocumentFactory.create(storage_type=Document.STORAGE_TYPE_UNENCRYPTED) self.assertEqual(changed_password_check(None), []) + + def test_parser_check(self): + + self.assertEqual(parser_check(None), []) + + with mock.patch('documents.checks.document_consumer_declaration.send') as m: + m.return_value = [] + + self.assertEqual(parser_check(None), [Error("No parsers found. This is a bug. The consumer won't be " + "able to consume any documents without parsers.")]) diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index b4b19be4c..f53981850 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -29,81 +29,6 @@ class TestAttributes(TestCase): self.assertEqual(tuple([t.name for t in file_info.tags]), tags, filename) - def test_guess_attributes_from_name0(self): - self._test_guess_attributes_from_name( - "Sender - Title.pdf", "Sender", "Title", ()) - - def test_guess_attributes_from_name1(self): - self._test_guess_attributes_from_name( - "Spaced Sender - Title.pdf", "Spaced Sender", "Title", ()) - - def test_guess_attributes_from_name2(self): - self._test_guess_attributes_from_name( - "Sender - Spaced Title.pdf", "Sender", "Spaced Title", ()) - - def test_guess_attributes_from_name3(self): - self._test_guess_attributes_from_name( - "Dashed-Sender - Title.pdf", "Dashed-Sender", "Title", ()) - - def test_guess_attributes_from_name4(self): - self._test_guess_attributes_from_name( - "Sender - Dashed-Title.pdf", "Sender", "Dashed-Title", ()) - - def test_guess_attributes_from_name5(self): - self._test_guess_attributes_from_name( - "Sender - Title - tag1,tag2,tag3.pdf", - "Sender", - "Title", - self.TAGS - ) - - def test_guess_attributes_from_name6(self): - self._test_guess_attributes_from_name( - "Spaced Sender - Title - tag1,tag2,tag3.pdf", - "Spaced Sender", - "Title", - self.TAGS - ) - - def test_guess_attributes_from_name7(self): - self._test_guess_attributes_from_name( - "Sender - Spaced Title - tag1,tag2,tag3.pdf", - "Sender", - "Spaced Title", - self.TAGS - ) - - def test_guess_attributes_from_name8(self): - self._test_guess_attributes_from_name( - "Dashed-Sender - Title - tag1,tag2,tag3.pdf", - "Dashed-Sender", - "Title", - self.TAGS - ) - - def test_guess_attributes_from_name9(self): - self._test_guess_attributes_from_name( - "Sender - Dashed-Title - tag1,tag2,tag3.pdf", - "Sender", - "Dashed-Title", - self.TAGS - ) - - def test_guess_attributes_from_name10(self): - self._test_guess_attributes_from_name( - "Σενδερ - Τιτλε - tag1,tag2,tag3.pdf", - "Σενδερ", - "Τιτλε", - self.TAGS - ) - - def test_guess_attributes_from_name_when_correspondent_empty(self): - self._test_guess_attributes_from_name( - ' - weird empty correspondent but should not break.pdf', - None, - 'weird empty correspondent but should not break', - () - ) def test_guess_attributes_from_name_when_title_starts_with_dash(self): self._test_guess_attributes_from_name( @@ -121,28 +46,6 @@ class TestAttributes(TestCase): () ) - def test_guess_attributes_from_name_when_title_is_empty(self): - self._test_guess_attributes_from_name( - 'weird correspondent but should not break - .pdf', - 'weird correspondent but should not break', - '', - () - ) - - def test_case_insensitive_tag_creation(self): - """ - Tags should be detected and created as lower case. - :return: - """ - - filename = "Title - Correspondent - tAg1,TAG2.pdf" - self.assertEqual(len(FileInfo.from_filename(filename).tags), 2) - - path = "Title - Correspondent - tag1,tag2.pdf" - self.assertEqual(len(FileInfo.from_filename(filename).tags), 2) - - self.assertEqual(Tag.objects.all().count(), 2) - class TestFieldPermutations(TestCase): @@ -199,69 +102,7 @@ class TestFieldPermutations(TestCase): filename = template.format(**spec) self._test_guessed_attributes(filename, **spec) - def test_title_and_correspondent(self): - template = '{correspondent} - {title}.pdf' - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - spec = dict(correspondent=correspondent, title=title) - filename = template.format(**spec) - self._test_guessed_attributes(filename, **spec) - - def test_title_and_correspondent_and_tags(self): - template = '{correspondent} - {title} - {tags}.pdf' - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - for tags in self.valid_tags: - spec = dict(correspondent=correspondent, title=title, - tags=tags) - filename = template.format(**spec) - self._test_guessed_attributes(filename, **spec) - - def test_created_and_correspondent_and_title_and_tags(self): - - template = ( - "{created} - " - "{correspondent} - " - "{title} - " - "{tags}.pdf" - ) - - for created in self.valid_dates: - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - for tags in self.valid_tags: - spec = { - "created": created, - "correspondent": correspondent, - "title": title, - "tags": tags, - } - self._test_guessed_attributes( - template.format(**spec), **spec) - - def test_created_and_correspondent_and_title(self): - - template = "{created} - {correspondent} - {title}.pdf" - - for created in self.valid_dates: - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - - # Skip cases where title looks like a tag as we can't - # accommodate such cases. - if title.lower() == title: - continue - - spec = { - "created": created, - "correspondent": correspondent, - "title": title - } - self._test_guessed_attributes( - template.format(**spec), **spec) - def test_created_and_title(self): - template = "{created} - {title}.pdf" for created in self.valid_dates: @@ -273,21 +114,6 @@ class TestFieldPermutations(TestCase): self._test_guessed_attributes( template.format(**spec), **spec) - def test_created_and_title_and_tags(self): - - template = "{created} - {title} - {tags}.pdf" - - for created in self.valid_dates: - for title in self.valid_titles: - for tags in self.valid_tags: - spec = { - "created": created, - "title": title, - "tags": tags - } - self._test_guessed_attributes( - template.format(**spec), **spec) - def test_invalid_date_format(self): info = FileInfo.from_filename("06112017Z - title.pdf") self.assertEqual(info.title, "title") @@ -336,32 +162,6 @@ class TestFieldPermutations(TestCase): info = FileInfo.from_filename(filename) self.assertEqual(info.title, "anotherall") - # Complex transformation without date in replacement string - with self.settings( - FILENAME_PARSE_TRANSFORMS=[(exact_patt, repl1)]): - info = FileInfo.from_filename(filename) - self.assertEqual(info.title, "0001") - self.assertEqual(len(info.tags), 2) - self.assertEqual(info.tags[0].name, "tag1") - self.assertEqual(info.tags[1].name, "tag2") - self.assertIsNone(info.created) - - # Complex transformation with date in replacement string - with self.settings( - FILENAME_PARSE_TRANSFORMS=[ - (none_patt, "none.gif"), - (exact_patt, repl2), # <-- matches - (exact_patt, repl1), - (all_patt, "all.gif")]): - info = FileInfo.from_filename(filename) - self.assertEqual(info.title, "0001") - self.assertEqual(len(info.tags), 2) - self.assertEqual(info.tags[0].name, "tag1") - self.assertEqual(info.tags[1].name, "tag2") - self.assertEqual(info.created.year, 2019) - self.assertEqual(info.created.month, 9) - self.assertEqual(info.created.day, 8) - class DummyParser(DocumentParser): @@ -476,15 +276,13 @@ class TestConsumer(DirectoriesMixin, TestCase): def testOverrideFilename(self): filename = self.get_test_file() - override_filename = "My Bank - Statement for November.pdf" + override_filename = "Statement for November.pdf" document = self.consumer.try_consume_file(filename, override_filename=override_filename) - self.assertEqual(document.correspondent.name, "My Bank") self.assertEqual(document.title, "Statement for November") def testOverrideTitle(self): - document = self.consumer.try_consume_file(self.get_test_file(), override_title="Override Title") self.assertEqual(document.title, "Override Title") @@ -594,11 +392,10 @@ class TestConsumer(DirectoriesMixin, TestCase): def testFilenameHandling(self): filename = self.get_test_file() - document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs") + document = self.consumer.try_consume_file(filename, override_title="new docs") self.assertEqual(document.title, "new docs") - self.assertEqual(document.correspondent.name, "Bank") - self.assertEqual(document.filename, "Bank/new docs.pdf") + self.assertEqual(document.filename, "none/new docs.pdf") @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}") @mock.patch("documents.signals.handlers.generate_unique_filename") @@ -617,10 +414,9 @@ class TestConsumer(DirectoriesMixin, TestCase): Tag.objects.create(name="test", is_inbox_tag=True) - document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs") + document = self.consumer.try_consume_file(filename, override_title="new docs") self.assertEqual(document.title, "new docs") - self.assertEqual(document.correspondent.name, "Bank") self.assertIsNotNone(os.path.isfile(document.title)) self.assertTrue(os.path.isfile(document.source_path)) @@ -642,3 +438,31 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertEqual(document.document_type, dtype) self.assertIn(t1, document.tags.all()) self.assertNotIn(t2, document.tags.all()) + + @override_settings(CONSUMER_DELETE_DUPLICATES=True) + def test_delete_duplicate(self): + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + doc = self.consumer.try_consume_file(dst) + + self.assertFalse(os.path.isfile(dst)) + self.assertIsNotNone(doc) + + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst) + self.assertFalse(os.path.isfile(dst)) + + @override_settings(CONSUMER_DELETE_DUPLICATES=False) + def test_no_delete_duplicate(self): + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + doc = self.consumer.try_consume_file(dst) + + self.assertFalse(os.path.isfile(dst)) + self.assertIsNotNone(doc) + + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst) + self.assertTrue(os.path.isfile(dst)) diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 2e60065f1..b24f52aa2 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -14,7 +14,7 @@ from django.utils import timezone from .utils import DirectoriesMixin from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \ generate_unique_filename -from ..models import Document, Correspondent, Tag +from ..models import Document, Correspondent, Tag, DocumentType class TestFileHandling(DirectoriesMixin, TestCase): @@ -190,6 +190,17 @@ class TestFileHandling(DirectoriesMixin, TestCase): self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), True) self.assertTrue(os.path.isfile(important_file)) + @override_settings(PAPERLESS_FILENAME_FORMAT="{document_type} - {title}") + def test_document_type(self): + dt = DocumentType.objects.create(name="my_doc_type") + d = Document.objects.create(title="the_doc", mime_type="application/pdf") + + self.assertEqual(generate_filename(d), "none - the_doc.pdf") + + d.document_type = dt + + self.assertEqual(generate_filename(d), "my_doc_type - the_doc.pdf") + @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}") def test_tags_with_underscore(self): document = Document() diff --git a/src/documents/tests/test_management.py b/src/documents/tests/test_management.py new file mode 100644 index 000000000..58aaf9342 --- /dev/null +++ b/src/documents/tests/test_management.py @@ -0,0 +1,135 @@ +import hashlib +import tempfile +import filecmp +import os +import shutil +from pathlib import Path +from unittest import mock + +from django.test import TestCase, override_settings + + +from django.core.management import call_command + +from documents.file_handling import generate_filename +from documents.management.commands.document_archiver import handle_document +from documents.models import Document +from documents.tests.utils import DirectoriesMixin + + +sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf") + + +class TestArchiver(DirectoriesMixin, TestCase): + + def make_models(self): + return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf") + + def test_archiver(self): + + doc = self.make_models() + shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) + + call_command('document_archiver') + + def test_handle_document(self): + + doc = self.make_models() + shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) + + handle_document(doc.pk) + + doc = Document.objects.get(id=doc.id) + + self.assertIsNotNone(doc.checksum) + self.assertTrue(os.path.isfile(doc.archive_path)) + self.assertTrue(os.path.isfile(doc.source_path)) + self.assertTrue(filecmp.cmp(sample_file, doc.source_path)) + + +class TestDecryptDocuments(TestCase): + + @override_settings( + ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"), + THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"), + PASSPHRASE="test", + PAPERLESS_FILENAME_FORMAT=None + ) + @mock.patch("documents.management.commands.decrypt_documents.input") + def test_decrypt(self, m): + + media_dir = tempfile.mkdtemp() + originals_dir = os.path.join(media_dir, "documents", "originals") + thumb_dir = os.path.join(media_dir, "documents", "thumbnails") + os.makedirs(originals_dir, exist_ok=True) + os.makedirs(thumb_dir, exist_ok=True) + + override_settings( + ORIGINALS_DIR=originals_dir, + THUMBNAIL_DIR=thumb_dir, + PASSPHRASE="test" + ).enable() + + doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) + + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg")) + + call_command('decrypt_documents') + + doc.refresh_from_db() + + self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED) + self.assertEqual(doc.filename, "0000002.pdf") + self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf"))) + self.assertTrue(os.path.isfile(doc.source_path)) + self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png"))) + self.assertTrue(os.path.isfile(doc.thumbnail_path)) + + with doc.source_file as f: + checksum = hashlib.md5(f.read()).hexdigest() + self.assertEqual(checksum, doc.checksum) + + +class TestMakeIndex(TestCase): + + @mock.patch("documents.management.commands.document_index.index_reindex") + def test_reindex(self, m): + call_command("document_index", "reindex") + m.assert_called_once() + + @mock.patch("documents.management.commands.document_index.index_optimize") + def test_optimize(self, m): + call_command("document_index", "optimize") + m.assert_called_once() + + +class TestRenamer(DirectoriesMixin, TestCase): + + def test_rename(self): + doc = Document.objects.create(title="test", mime_type="application/pdf") + doc.filename = generate_filename(doc) + doc.save() + + Path(doc.source_path).touch() + + old_source_path = doc.source_path + + with override_settings(PAPERLESS_FILENAME_FORMAT="{title}"): + call_command("document_renamer") + + doc2 = Document.objects.get(id=doc.id) + + self.assertEqual(doc2.filename, "test.pdf") + self.assertFalse(os.path.isfile(old_source_path)) + self.assertFalse(os.path.isfile(doc.source_path)) + self.assertTrue(os.path.isfile(doc2.source_path)) + + +class TestCreateClassifier(TestCase): + + @mock.patch("documents.management.commands.document_create_classifier.train_classifier") + def test_create_classifier(self, m): + call_command("document_create_classifier") + + m.assert_called_once() diff --git a/src/documents/tests/test_management_archiver.py b/src/documents/tests/test_management_archiver.py deleted file mode 100644 index 0828f05ff..000000000 --- a/src/documents/tests/test_management_archiver.py +++ /dev/null @@ -1,40 +0,0 @@ -import filecmp -import os -import shutil - -from django.core.management import call_command -from django.test import TestCase - -from documents.management.commands.document_archiver import handle_document -from documents.models import Document -from documents.tests.utils import DirectoriesMixin - - -sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf") - - -class TestArchiver(DirectoriesMixin, TestCase): - - def make_models(self): - return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf") - - def test_archiver(self): - - doc = self.make_models() - shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) - - call_command('document_archiver') - - def test_handle_document(self): - - doc = self.make_models() - shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) - - handle_document(doc.pk) - - doc = Document.objects.get(id=doc.id) - - self.assertIsNotNone(doc.checksum) - self.assertTrue(os.path.isfile(doc.archive_path)) - self.assertTrue(os.path.isfile(doc.source_path)) - self.assertTrue(filecmp.cmp(sample_file, doc.source_path)) diff --git a/src/documents/tests/test_management_decrypt.py b/src/documents/tests/test_management_decrypt.py deleted file mode 100644 index 1d64b1105..000000000 --- a/src/documents/tests/test_management_decrypt.py +++ /dev/null @@ -1,57 +0,0 @@ -import hashlib -import json -import os -import shutil -import tempfile -from unittest import mock - -from django.core.management import call_command -from django.test import TestCase, override_settings - -from documents.management.commands import document_exporter -from documents.models import Document, Tag, DocumentType, Correspondent - - -class TestDecryptDocuments(TestCase): - - @override_settings( - ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"), - THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"), - PASSPHRASE="test", - PAPERLESS_FILENAME_FORMAT=None - ) - @mock.patch("documents.management.commands.decrypt_documents.input") - def test_decrypt(self, m): - - media_dir = tempfile.mkdtemp() - originals_dir = os.path.join(media_dir, "documents", "originals") - thumb_dir = os.path.join(media_dir, "documents", "thumbnails") - os.makedirs(originals_dir, exist_ok=True) - os.makedirs(thumb_dir, exist_ok=True) - - override_settings( - ORIGINALS_DIR=originals_dir, - THUMBNAIL_DIR=thumb_dir, - PASSPHRASE="test" - ).enable() - - doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) - - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg")) - - call_command('decrypt_documents') - - doc.refresh_from_db() - - self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED) - self.assertEqual(doc.filename, "0000002.pdf") - self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf"))) - self.assertTrue(os.path.isfile(doc.source_path)) - self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png"))) - self.assertTrue(os.path.isfile(doc.thumbnail_path)) - - with doc.source_file as f: - checksum = hashlib.md5(f.read()).hexdigest() - self.assertEqual(checksum, doc.checksum) - diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 22d6fc7f6..d6ab7eadd 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -24,11 +24,17 @@ class TestExportImport(DirectoriesMixin, TestCase): file = os.path.join(self.dirs.originals_dir, "0000001.pdf") - Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf") - Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) - Tag.objects.create(name="t") - DocumentType.objects.create(name="dt") - Correspondent.objects.create(name="c") + d1 = Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf") + d2 = Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) + t1 = Tag.objects.create(name="t") + dt1 = DocumentType.objects.create(name="dt") + c1 = Correspondent.objects.create(name="c") + + d1.tags.add(t1) + d1.correspondents = c1 + d1.document_type = dt1 + d1.save() + d2.save() target = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, target) @@ -59,11 +65,25 @@ class TestExportImport(DirectoriesMixin, TestCase): self.assertEqual(checksum, element['fields']['archive_checksum']) with paperless_environment() as dirs: + self.assertEqual(Document.objects.count(), 2) + Document.objects.all().delete() + Correspondent.objects.all().delete() + DocumentType.objects.all().delete() + Tag.objects.all().delete() + self.assertEqual(Document.objects.count(), 0) + call_command('document_importer', target) + self.assertEqual(Document.objects.count(), 2) messages = check_sanity() # everything is alright after the test self.assertEqual(len(messages), 0, str([str(m) for m in messages])) + @override_settings( + PAPERLESS_FILENAME_FORMAT="{title}" + ) + def test_exporter_with_filename_format(self): + self.test_exporter() + def test_export_missing_files(self): target = tempfile.mkdtemp() diff --git a/src/documents/tests/test_migrations.py b/src/documents/tests/test_migrations.py new file mode 100644 index 000000000..33ba41444 --- /dev/null +++ b/src/documents/tests/test_migrations.py @@ -0,0 +1,129 @@ +import os +import shutil +from pathlib import Path + +from django.apps import apps +from django.conf import settings +from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.test import TestCase, TransactionTestCase, override_settings + +from documents.models import Document +from documents.parsers import get_default_file_extension +from documents.tests.utils import DirectoriesMixin + + +class TestMigrations(TransactionTestCase): + + @property + def app(self): + return apps.get_containing_app_config(type(self).__module__).name + + migrate_from = None + migrate_to = None + + def setUp(self): + super(TestMigrations, self).setUp() + + assert self.migrate_from and self.migrate_to, \ + "TestCase '{}' must define migrate_from and migrate_to properties".format(type(self).__name__) + self.migrate_from = [(self.app, self.migrate_from)] + self.migrate_to = [(self.app, self.migrate_to)] + executor = MigrationExecutor(connection) + old_apps = executor.loader.project_state(self.migrate_from).apps + + # Reverse to the original migration + executor.migrate(self.migrate_from) + + self.setUpBeforeMigration(old_apps) + + # Run the migration to test + executor = MigrationExecutor(connection) + executor.loader.build_graph() # reload. + executor.migrate(self.migrate_to) + + self.apps = executor.loader.project_state(self.migrate_to).apps + + def setUpBeforeMigration(self, apps): + pass + + +STORAGE_TYPE_UNENCRYPTED = "unencrypted" +STORAGE_TYPE_GPG = "gpg" + + +def source_path_before(self): + if self.filename: + fname = str(self.filename) + else: + fname = "{:07}.{}".format(self.pk, self.file_type) + if self.storage_type == STORAGE_TYPE_GPG: + fname += ".gpg" + + return os.path.join( + settings.ORIGINALS_DIR, + fname + ) + + +def file_type_after(self): + return get_default_file_extension(self.mime_type) + + +def source_path_after(doc): + if doc.filename: + fname = str(doc.filename) + else: + fname = "{:07}{}".format(doc.pk, file_type_after(doc)) + if doc.storage_type == STORAGE_TYPE_GPG: + fname += ".gpg" # pragma: no cover + + return os.path.join( + settings.ORIGINALS_DIR, + fname + ) + + +@override_settings(PASSPHRASE="test") +class TestMigrateMimeType(DirectoriesMixin, TestMigrations): + + migrate_from = '1002_auto_20201111_1105' + migrate_to = '1003_mime_types' + + def setUpBeforeMigration(self, apps): + Document = apps.get_model("documents", "Document") + doc = Document.objects.create(title="test", file_type="pdf", filename="file1.pdf") + self.doc_id = doc.id + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_before(doc)) + + doc2 = Document.objects.create(checksum="B", file_type="pdf", storage_type=STORAGE_TYPE_GPG) + self.doc2_id = doc2.id + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), source_path_before(doc2)) + + def testMimeTypesMigrated(self): + Document = self.apps.get_model('documents', 'Document') + + doc = Document.objects.get(id=self.doc_id) + self.assertEqual(doc.mime_type, "application/pdf") + + doc2 = Document.objects.get(id=self.doc2_id) + self.assertEqual(doc2.mime_type, "application/pdf") + + +@override_settings(PASSPHRASE="test") +class TestMigrateMimeTypeBackwards(DirectoriesMixin, TestMigrations): + + migrate_from = '1003_mime_types' + migrate_to = '1002_auto_20201111_1105' + + def setUpBeforeMigration(self, apps): + Document = apps.get_model("documents", "Document") + doc = Document.objects.create(title="test", mime_type="application/pdf", filename="file1.pdf") + self.doc_id = doc.id + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_after(doc)) + + def testMimeTypesReverted(self): + Document = self.apps.get_model('documents', 'Document') + + doc = Document.objects.get(id=self.doc_id) + self.assertEqual(doc.file_type, "pdf") diff --git a/src/documents/views.py b/src/documents/views.py index bf31c749b..8f6ec7f13 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -47,13 +47,21 @@ from .serialisers import ( TagSerializer, DocumentTypeSerializer, PostDocumentSerializer, - SavedViewSerializer + SavedViewSerializer, + BulkEditSerializer ) class IndexView(TemplateView): template_name = "index.html" + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['cookie_prefix'] = settings.COOKIE_PREFIX + context['username'] = self.request.user.username + context['full_name'] = self.request.user.get_full_name() + return context + class CorrespondentViewSet(ModelViewSet): model = Correspondent @@ -103,6 +111,10 @@ class DocumentTypeViewSet(ModelViewSet): ordering_fields = ("name", "matching_algorithm", "match", "document_count") +class BulkEditForm(object): + pass + + class DocumentViewSet(RetrieveModelMixin, UpdateModelMixin, DestroyModelMixin, @@ -126,6 +138,17 @@ class DocumentViewSet(RetrieveModelMixin, "added", "archive_serial_number") + def get_serializer(self, *args, **kwargs): + fields_param = self.request.query_params.get('fields', None) + if fields_param: + fields = fields_param.split(",") + else: + fields = None + serializer_class = self.get_serializer_class() + kwargs.setdefault('context', self.get_serializer_context()) + kwargs.setdefault('fields', fields) + return serializer_class(*args, **kwargs) + def update(self, request, *args, **kwargs): response = super(DocumentViewSet, self).update( request, *args, **kwargs) @@ -267,6 +290,39 @@ class SavedViewViewSet(ModelViewSet): serializer.save(user=self.request.user) +class BulkEditView(APIView): + + permission_classes = (IsAuthenticated,) + serializer_class = BulkEditSerializer + parser_classes = (parsers.JSONParser,) + + def get_serializer_context(self): + return { + 'request': self.request, + 'format': self.format_kwarg, + 'view': self + } + + def get_serializer(self, *args, **kwargs): + kwargs['context'] = self.get_serializer_context() + return self.serializer_class(*args, **kwargs) + + def post(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + method = serializer.validated_data.get("method") + parameters = serializer.validated_data.get("parameters") + documents = serializer.validated_data.get("documents") + + try: + # TODO: parameter validation + result = method(documents, **parameters) + return Response({"result": result}) + except Exception as e: + return HttpResponseBadRequest(str(e)) + + class PostDocumentView(APIView): permission_classes = (IsAuthenticated,) @@ -335,14 +391,27 @@ class SearchView(APIView): } def get(self, request, format=None): - if 'query' not in request.query_params: + + if 'query' in request.query_params: + query = request.query_params['query'] + else: + query = None + + if 'more_like' in request.query_params: + more_like_id = request.query_params['more_like'] + more_like_content = Document.objects.get(id=more_like_id).content + else: + more_like_id = None + more_like_content = None + + if not query and not more_like_id: return Response({ 'count': 0, 'page': 0, 'page_count': 0, + 'corrected_query': None, 'results': []}) - query = request.query_params['query'] try: page = int(request.query_params.get('page', 1)) except (ValueError, TypeError): @@ -352,8 +421,7 @@ class SearchView(APIView): page = 1 try: - with index.query_page(self.ix, query, page) as (result_page, - corrected_query): + with index.query_page(self.ix, page, query, more_like_id, more_like_content) as (result_page, corrected_query): # NOQA: E501 return Response( {'count': len(result_page), 'page': result_page.pagenum, diff --git a/src/paperless/checks.py b/src/paperless/checks.py index 819582ffc..1329ad679 100644 --- a/src/paperless/checks.py +++ b/src/paperless/checks.py @@ -13,18 +13,17 @@ writeable_hint = ( ) -def path_check(env_var): +def path_check(var, directory): messages = [] - directory = os.getenv(env_var) if directory: if not os.path.exists(directory): messages.append(Error( - exists_message.format(env_var), + exists_message.format(var), exists_hint.format(directory) )) elif not os.access(directory, os.W_OK | os.X_OK): messages.append(Error( - writeable_message.format(env_var), + writeable_message.format(var), writeable_hint.format(directory) )) return messages @@ -36,12 +35,9 @@ def paths_check(app_configs, **kwargs): Check the various paths for existence, readability and writeability """ - check_messages = path_check("PAPERLESS_DATA_DIR") + \ - path_check("PAPERLESS_MEDIA_ROOT") + \ - path_check("PAPERLESS_CONSUMPTION_DIR") + \ - path_check("PAPERLESS_STATICDIR") - - return check_messages + return path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \ + path_check("PAPERLESS_MEDIA_ROOT", settings.MEDIA_ROOT) + \ + path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR) @register() diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 1a6b80a0c..c6f7c9357 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -160,13 +160,6 @@ if AUTO_LOGIN_USERNAME: MIDDLEWARE.insert(_index+1, 'paperless.auth.AutoLoginMiddleware') -if DEBUG: - X_FRAME_OPTIONS = '' - # this should really be 'allow-from uri' but its not supported in any mayor - # browser. -else: - X_FRAME_OPTIONS = 'SAMEORIGIN' - # We allow CORS from localhost:8080 CORS_ALLOWED_ORIGINS = tuple(os.getenv("PAPERLESS_CORS_ALLOWED_HOSTS", "http://localhost:8000").split(",")) diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py new file mode 100644 index 000000000..e1525cab8 --- /dev/null +++ b/src/paperless/tests/test_checks.py @@ -0,0 +1,54 @@ +import os +import shutil + +from django.test import TestCase, override_settings + +from documents.tests.utils import DirectoriesMixin +from paperless import binaries_check, paths_check +from paperless.checks import debug_mode_check + + +class TestChecks(DirectoriesMixin, TestCase): + + def test_binaries(self): + self.assertEqual(binaries_check(None), []) + + @override_settings(CONVERT_BINARY="uuuhh", OPTIPNG_BINARY="forgot") + def test_binaries_fail(self): + self.assertEqual(len(binaries_check(None)), 2) + + def test_paths_check(self): + self.assertEqual(paths_check(None), []) + + @override_settings(MEDIA_ROOT="uuh", + DATA_DIR="whatever", + CONSUMPTION_DIR="idontcare") + def test_paths_check_dont_exist(self): + msgs = paths_check(None) + self.assertEqual(len(msgs), 3, str(msgs)) + + for msg in msgs: + self.assertTrue(msg.msg.endswith("is set but doesn't exist.")) + + def test_paths_check_no_access(self): + os.chmod(self.dirs.data_dir, 0o000) + os.chmod(self.dirs.media_dir, 0o000) + os.chmod(self.dirs.consumption_dir, 0o000) + + self.addCleanup(os.chmod, self.dirs.data_dir, 0o777) + self.addCleanup(os.chmod, self.dirs.media_dir, 0o777) + self.addCleanup(os.chmod, self.dirs.consumption_dir, 0o777) + + msgs = paths_check(None) + self.assertEqual(len(msgs), 3) + + for msg in msgs: + self.assertTrue(msg.msg.endswith("is not writeable")) + + @override_settings(DEBUG=False) + def test_debug_disabled(self): + self.assertEqual(debug_mode_check(None), []) + + @override_settings(DEBUG=True) + def test_debug_enabled(self): + self.assertEqual(len(debug_mode_check(None)), 1) diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 079971bb3..831eb02b4 100755 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -18,7 +18,8 @@ from documents.views import ( SearchAutoCompleteView, StatisticsView, PostDocumentView, - SavedViewViewSet + SavedViewViewSet, + BulkEditView ) from paperless.views import FaviconView @@ -52,6 +53,10 @@ urlpatterns = [ re_path(r"^documents/post_document/", PostDocumentView.as_view(), name="post_document"), + + re_path(r"^documents/bulk_edit/", BulkEditView.as_view(), + name="bulk_edit"), + path('token/', views.obtain_auth_token) ] + api_router.urls)), diff --git a/src/paperless/version.py b/src/paperless/version.py index 10283c145..b1dfc590c 100644 --- a/src/paperless/version.py +++ b/src/paperless/version.py @@ -1 +1 @@ -__version__ = (0, 9, 8) +__version__ = (0, 9, 9) diff --git a/src/paperless_tesseract/checks.py b/src/paperless_tesseract/checks.py index 41ea3c9b5..d58b7ac6d 100644 --- a/src/paperless_tesseract/checks.py +++ b/src/paperless_tesseract/checks.py @@ -1,7 +1,7 @@ import subprocess from django.conf import settings -from django.core.checks import Error, register +from django.core.checks import Error, Warning, register def get_tesseract_langs(): diff --git a/src/paperless_tesseract/languages.py b/src/paperless_tesseract/languages.py deleted file mode 100644 index 5ea560654..000000000 --- a/src/paperless_tesseract/languages.py +++ /dev/null @@ -1,194 +0,0 @@ -# Thanks to the Library of Congress and some creative use of sed and awk: -# http://www.loc.gov/standards/iso639-2/php/English_list.php - -ISO639 = { - - "aa": "aar", - "ab": "abk", - "ae": "ave", - "af": "afr", - "ak": "aka", - "am": "amh", - "an": "arg", - "ar": "ara", - "as": "asm", - "av": "ava", - "ay": "aym", - "az": "aze", - "ba": "bak", - "be": "bel", - "bg": "bul", - "bh": "bih", - "bi": "bis", - "bm": "bam", - "bn": "ben", - "bo": "bod", - "br": "bre", - "bs": "bos", - "ca": "cat", - "ce": "che", - "ch": "cha", - "co": "cos", - "cr": "cre", - "cs": "ces", - "cu": "chu", - "cv": "chv", - "cy": "cym", - "da": "dan", - "de": "deu", - "dv": "div", - "dz": "dzo", - "ee": "ewe", - "el": "ell", - "en": "eng", - "eo": "epo", - "es": "spa", - "et": "est", - "eu": "eus", - "fa": "fas", - "ff": "ful", - "fi": "fin", - "fj": "fij", - "fo": "fao", - "fr": "fra", - "fy": "fry", - "ga": "gle", - "gd": "gla", - "gl": "glg", - "gn": "grn", - "gu": "guj", - "gv": "glv", - "ha": "hau", - "he": "heb", - "hi": "hin", - "ho": "hmo", - "hr": "hrv", - "ht": "hat", - "hu": "hun", - "hy": "hye", - "hz": "her", - "ia": "ina", - "id": "ind", - "ie": "ile", - "ig": "ibo", - "ii": "iii", - "ik": "ipk", - "io": "ido", - "is": "isl", - "it": "ita", - "iu": "iku", - "ja": "jpn", - "jv": "jav", - "ka": "kat", - "kg": "kon", - "ki": "kik", - "kj": "kua", - "kk": "kaz", - "kl": "kal", - "km": "khm", - "kn": "kan", - "ko": "kor", - "kr": "kau", - "ks": "kas", - "ku": "kur", - "kv": "kom", - "kw": "cor", - "ky": "kir", - "la": "lat", - "lb": "ltz", - "lg": "lug", - "li": "lim", - "ln": "lin", - "lo": "lao", - "lt": "lit", - "lu": "lub", - "lv": "lav", - "mg": "mlg", - "mh": "mah", - "mi": "mri", - "mk": "mkd", - "ml": "mal", - "mn": "mon", - "mr": "mar", - "ms": "msa", - "mt": "mlt", - "my": "mya", - "na": "nau", - "nb": "nob", - "nd": "nde", - "ne": "nep", - "ng": "ndo", - "nl": "nld", - "no": "nor", - "nr": "nbl", - "nv": "nav", - "ny": "nya", - "oc": "oci", - "oj": "oji", - "om": "orm", - "or": "ori", - "os": "oss", - "pa": "pan", - "pi": "pli", - "pl": "pol", - "ps": "pus", - "pt": "por", - "qu": "que", - "rm": "roh", - "rn": "run", - "ro": "ron", - "ru": "rus", - "rw": "kin", - "sa": "san", - "sc": "srd", - "sd": "snd", - "se": "sme", - "sg": "sag", - "si": "sin", - "sk": "slk", - "sl": "slv", - "sm": "smo", - "sn": "sna", - "so": "som", - "sq": "sqi", - "sr": "srp", - "ss": "ssw", - "st": "sot", - "su": "sun", - "sv": "swe", - "sw": "swa", - "ta": "tam", - "te": "tel", - "tg": "tgk", - "th": "tha", - "ti": "tir", - "tk": "tuk", - "tl": "tgl", - "tn": "tsn", - "to": "ton", - "tr": "tur", - "ts": "tso", - "tt": "tat", - "tw": "twi", - "ty": "tah", - "ug": "uig", - "uk": "ukr", - "ur": "urd", - "uz": "uzb", - "ve": "ven", - "vi": "vie", - "vo": "vol", - "wa": "wln", - "wo": "wol", - "xh": "xho", - "yi": "yid", - "yo": "yor", - "za": "zha", - - # Tessdata contains two values for Chinese, "chi_sim" and "chi_tra". I - # have no idea which one is better, so I just picked the bigger file. - "zh": "chi_tra", - - "zu": "zul" - -} diff --git a/src/paperless_tesseract/tests/test_checks.py b/src/paperless_tesseract/tests/test_checks.py new file mode 100644 index 000000000..c4f15764e --- /dev/null +++ b/src/paperless_tesseract/tests/test_checks.py @@ -0,0 +1,26 @@ +from unittest import mock + +from django.core.checks import ERROR +from django.test import TestCase, override_settings + +from paperless_tesseract import check_default_language_available + + +class TestChecks(TestCase): + + def test_default_language(self): + msgs = check_default_language_available(None) + + @override_settings(OCR_LANGUAGE="") + def test_no_language(self): + msgs = check_default_language_available(None) + self.assertEqual(len(msgs), 1) + self.assertTrue(msgs[0].msg.startswith("No OCR language has been specified with PAPERLESS_OCR_LANGUAGE")) + + @override_settings(OCR_LANGUAGE="ita") + @mock.patch("paperless_tesseract.checks.get_tesseract_langs") + def test_invalid_language(self, m): + m.return_value = ["deu", "eng"] + msgs = check_default_language_available(None) + self.assertEqual(len(msgs), 1) + self.assertEqual(msgs[0].level, ERROR) diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py index 7e488ca37..030c2c2c2 100644 --- a/src/paperless_text/parsers.py +++ b/src/paperless_text/parsers.py @@ -35,15 +35,3 @@ class TextDocumentParser(DocumentParser): def parse(self, document_path, mime_type): with open(document_path, 'r') as f: self.text = f.read() - - -def run_command(*args): - environment = os.environ.copy() - if settings.CONVERT_MEMORY_LIMIT: - environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT - if settings.CONVERT_TMPDIR: - environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR - - if not subprocess.Popen(' '.join(args), env=environment, - shell=True).wait() == 0: - raise ParseError("Convert failed at {}".format(args))