From 164418880a93301b61e577363d4211b8142dcf0e Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Thu, 17 Dec 2020 21:36:21 +0100 Subject: [PATCH 01/37] more like this searching --- .../document-detail.component.html | 6 +++ .../document-detail.component.ts | 4 ++ .../document-card-large.component.html | 11 +++++- .../document-card-large.component.ts | 13 +++++++ .../result-highlight.component.scss | 2 +- .../components/search/search.component.html | 10 ++++- .../app/components/search/search.component.ts | 26 +++++++++++-- .../src/app/services/rest/search.service.ts | 10 ++++- src/documents/index.py | 37 ++++++++++++++----- src/documents/views.py | 21 +++++++---- 10 files changed, 113 insertions(+), 27 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index f4a64c2cc..f7e1ff855 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -24,6 +24,12 @@ + -
- -
- + + + + +
+ + + +
+ + +
-
- {{hint}} - \ No newline at end of file + diff --git a/src-ui/src/app/components/common/input/tags/tags.component.scss b/src-ui/src/app/components/common/input/tags/tags.component.scss index f2635b7f2..41fc6acc4 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.scss +++ b/src-ui/src/app/components/common/input/tags/tags.component.scss @@ -1,10 +1,4 @@ -.tags-form-control { - height: auto; +.selected-icon { + min-width: 1em; + min-height: 1em; } - - -.scrollable-menu { - height: auto; - max-height: 300px; - overflow-x: hidden; -} \ No newline at end of file diff --git a/src-ui/src/app/components/common/input/tags/tags.component.ts b/src-ui/src/app/components/common/input/tags/tags.component.ts index cca99cc55..5501ac5a6 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.ts +++ b/src-ui/src/app/components/common/input/tags/tags.component.ts @@ -21,7 +21,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor { onChange = (newValue: number[]) => {}; - + onTouched = () => {}; writeValue(newValue: number[]): void { @@ -66,29 +66,28 @@ export class TagsComponent implements OnInit, ControlValueAccessor { removeTag(id) { let index = this.displayValue.indexOf(id) if (index > -1) { - this.displayValue.splice(index, 1) + let oldValue = this.displayValue + oldValue.splice(index, 1) + this.displayValue = [...oldValue] this.onChange(this.displayValue) } } - addTag(id) { - let index = this.displayValue.indexOf(id) - if (index == -1) { - this.displayValue.push(id) - this.onChange(this.displayValue) - } - } - - createTag() { var modal = this.modalService.open(TagEditDialogComponent, {backdrop: 'static'}) modal.componentInstance.dialogMode = 'create' modal.componentInstance.success.subscribe(newTag => { this.tagService.listAll().subscribe(tags => { this.tags = tags.results - this.addTag(newTag.id) + this.displayValue = [...this.displayValue, newTag.id] + this.onChange(this.displayValue) }) }) } + ngSelectChange() { + this.value = this.displayValue + this.onChange(this.displayValue) + } + } diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index f4a64c2cc..a3bc7e1e6 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -52,9 +52,9 @@ + (createNew)="createCorrespondent()"> + (createNew)="createDocumentType()"> diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 2eeb40d41..0dc662e31 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -1,7 +1,5 @@ @import "theme"; - @import "node_modules/bootstrap/scss/bootstrap"; - @import "~@ng-select/ng-select/themes/default.theme.css"; .toolbaricon { @@ -21,7 +19,7 @@ } body { - font-size: .875rem; + font-size: 0.875rem; } .form-control-dark { @@ -85,5 +83,16 @@ body { top: 8px; } } + + .ng-dropdown-panel .ng-dropdown-panel-items .ng-option.ng-option-selected, + .ng-dropdown-panel .ng-dropdown-panel-items .ng-option.ng-option-selected.ng-option-marked { + background: none; + } + } +} + +.paperless-input-tags { + .ng-select.ng-select-multiple .ng-select-container .ng-value-container .ng-value { + background-color: transparent; } } From 55c4c690ef8eec3ba494cd6c3a74dc9b5cd810ec Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Fri, 18 Dec 2020 01:13:30 -0800 Subject: [PATCH 08/37] Fix wrapping with multiple tags, embiggen tags, pretty icons --- .../common/input/tags/tags.component.html | 21 ++++++++++++------- .../common/input/tags/tags.component.scss | 8 +++++++ src-ui/src/styles.scss | 2 +- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src-ui/src/app/components/common/input/tags/tags.component.html b/src-ui/src/app/components/common/input/tags/tags.component.html index 89e391813..8a5dbc4f2 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.html +++ b/src-ui/src/app/components/common/input/tags/tags.component.html @@ -1,7 +1,7 @@
-
+
- + + + + + + -
- - - +
+
+ + + +
+
- diff --git a/src-ui/src/app/components/common/input/tags/tags.component.scss b/src-ui/src/app/components/common/input/tags/tags.component.scss index 41fc6acc4..2eaaa4f6d 100644 --- a/src-ui/src/app/components/common/input/tags/tags.component.scss +++ b/src-ui/src/app/components/common/input/tags/tags.component.scss @@ -2,3 +2,11 @@ min-width: 1em; min-height: 1em; } + +.tag-wrap { + font-size: 1rem; +} + +.tag-wrap-delete { + cursor: pointer; +} diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index 0dc662e31..ffb296271 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -71,7 +71,7 @@ body { position: relative; flex: 1 1 auto; margin-bottom: 0; - height: calc(1.5em + 0.75rem + 5px); + min-height: calc(1.5em + 0.75rem + 5px); line-height: 1.5; .ng-select-container { From c05de3d57f6148d4abee8f8775d6668f6984ed2d Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Fri, 18 Dec 2020 01:18:11 -0800 Subject: [PATCH 09/37] Tiny padding fixes --- src-ui/src/styles.scss | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index ffb296271..6e09db630 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -80,7 +80,7 @@ body { border-bottom-right-radius: 0; .ng-value-container .ng-input { - top: 8px; + top: 10px; } } @@ -95,4 +95,8 @@ body { .ng-select.ng-select-multiple .ng-select-container .ng-value-container .ng-value { background-color: transparent; } + + .ng-select.ng-select-multiple .ng-select-container .ng-value-container { + padding-top: 1px; + } } From 273c474e3fe00f68898c1dc74256a2f4de7174e9 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Fri, 18 Dec 2020 14:09:12 +0100 Subject: [PATCH 10/37] layout changes --- .../document-card-large/document-card-large.component.html | 7 +++++-- .../document-card-large/document-card-large.component.scss | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html index 58c0f6241..5bf0c9af2 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html @@ -23,7 +23,7 @@

-
+
@@ -51,9 +51,12 @@ Download -
+ + Score: + + Created: {{document.created | date}}
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss index 438d2c768..a20a56672 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss @@ -14,5 +14,5 @@ .search-score-bar { width: 100px; height: 5px; - margin: 10px; + margin-top: 2px; } \ No newline at end of file From 789abb3bbb3c14b85952e98485cf98c289deb9c0 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Fri, 18 Dec 2020 16:42:33 +0100 Subject: [PATCH 11/37] changed up the highlight fragment formatter --- docs/api.rst | 15 ++++------ .../result-highlight.component.html | 2 +- src/documents/index.py | 29 +++++++++++-------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index d352758fa..cff72a970 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -221,21 +221,16 @@ Each fragment contains a list of strings, and some of them are marked as a highl [ [ - {"text": "This is a sample text with a "}, - {"text": "highlighted", "term": 0}, - {"text": " word."} + {"text": "This is a sample text with a ", "highlight": false}, + {"text": "highlighted", "highlight": true}, + {"text": " word.", "highlight": false} ], [ - {"text": "Another", "term": 1}, - {"text": " fragment with a highlight."} + {"text": "Another", "highlight": true}, + {"text": " fragment with a highlight.", "highlight": false} ] ] - - -When ``term`` is present within a string, the word within ``text`` should be highlighted. -The term index groups multiple matches together and words with the same index -should get identical highlighting. A client may use this example to produce the following output: ... This is a sample text with a **highlighted** word. ... **Another** fragment with a highlight. ... diff --git a/src-ui/src/app/components/search/result-highlight/result-highlight.component.html b/src-ui/src/app/components/search/result-highlight/result-highlight.component.html index 1842f5cea..5dc5baa94 100644 --- a/src-ui/src/app/components/search/result-highlight/result-highlight.component.html +++ b/src-ui/src/app/components/search/result-highlight/result-highlight.component.html @@ -1,3 +1,3 @@ ... - {{token.text}} ... + {{token.text}} ... \ No newline at end of file diff --git a/src/documents/index.py b/src/documents/index.py index fdf7d7041..308ee932e 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -20,32 +20,37 @@ class JsonFormatter(Formatter): self.seen = {} def format_token(self, text, token, replace=False): - seen = self.seen ttext = self._text(get_text(text, token, replace)) - if ttext in seen: - termnum = seen[ttext] - else: - termnum = len(seen) - seen[ttext] = termnum - - return {'text': ttext, 'term': termnum} + return {'text': ttext, 'highlight': 'true'} def format_fragment(self, fragment, replace=False): output = [] index = fragment.startchar text = fragment.text - + amend_token = None for t in fragment.matches: if t.startchar is None: continue if t.startchar < index: continue if t.startchar > index: - output.append({'text': text[index:t.startchar]}) - output.append(self.format_token(text, t, replace)) + text_inbetween = text[index:t.startchar] + if amend_token and t.startchar - index < 10: + amend_token['text'] += text_inbetween + else: + output.append({'text': text_inbetween, + 'highlight': False}) + amend_token = None + token = self.format_token(text, t, replace) + if amend_token: + amend_token['text'] += token['text'] + else: + output.append(token) + amend_token = token index = t.endchar if index < fragment.endchar: - output.append({'text': text[index:fragment.endchar]}) + output.append({'text': text[index:fragment.endchar], + 'highlight': False}) return output def format(self, fragments, replace=False): From dfb88ebf8328ef9ed6b71c9da4b9ddac49768ea5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Fri, 18 Dec 2020 20:17:17 +0100 Subject: [PATCH 12/37] removed the date hack. fixes #144 also refer to #148 --- .../filter-dropdown-date.component.html | 51 +++---- .../filter-dropdown-date.component.ts | 139 ++++++++---------- .../filter-editor/filter-editor.component.ts | 31 ++-- 3 files changed, 106 insertions(+), 115 deletions(-) diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html index 6f6a42fe2..aca6e836c 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html +++ b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html @@ -4,38 +4,39 @@
- -
diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts index 91402d084..e570862cd 100644 --- a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts +++ b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts @@ -1,24 +1,37 @@ -import { Component, EventEmitter, Input, Output, ElementRef, ViewChild, SimpleChange } from '@angular/core'; -import { NgbDate, NgbDateStruct, NgbDatepicker } from '@ng-bootstrap/ng-bootstrap'; +import { formatDate } from '@angular/common'; +import { Component, EventEmitter, Input, Output, OnInit, OnDestroy } from '@angular/core'; +import { Subject, Subscription } from 'rxjs'; +import { debounceTime, distinctUntilChanged } from 'rxjs/operators'; export interface DateSelection { - before?: NgbDateStruct - after?: NgbDateStruct + before?: string + after?: string } +const FILTER_LAST_7_DAYS = 0 +const FILTER_LAST_MONTH = 1 +const FILTER_LAST_3_MONTHS = 2 +const FILTER_LAST_YEAR = 3 @Component({ selector: 'app-filter-dropdown-date', templateUrl: './filter-dropdown-date.component.html', styleUrls: ['./filter-dropdown-date.component.scss'] }) -export class FilterDropdownDateComponent { +export class FilterDropdownDateComponent implements OnInit, OnDestroy { + + quickFilters = [ + {id: FILTER_LAST_7_DAYS, name: "Last 7 days"}, + {id: FILTER_LAST_MONTH, name: "Last month"}, + {id: FILTER_LAST_3_MONTHS, name: "Last 3 months"}, + {id: FILTER_LAST_YEAR, name: "Last year"} + ] @Input() - dateBefore: NgbDateStruct + dateBefore: string @Input() - dateAfter: NgbDateStruct + dateAfter: string @Input() title: string @@ -26,87 +39,65 @@ export class FilterDropdownDateComponent { @Output() datesSet = new EventEmitter() - @ViewChild('dpAfter') dpAfter: NgbDatepicker - @ViewChild('dpBefore') dpBefore: NgbDatepicker + private datesSetDebounce$ = new Subject() - _dateBefore: NgbDateStruct - _dateAfter: NgbDateStruct - - get _maxDate(): NgbDate { - let date = new Date() - return NgbDate.from({year: date.getFullYear(), month: date.getMonth() + 1, day: date.getDate()}) + private sub: Subscription + + ngOnInit() { + this.sub = this.datesSetDebounce$.pipe( + debounceTime(400) + ).subscribe(() => { + this.onChange() + }) } - isStringRange(range: any) { - return typeof range == 'string' - } - - ngOnChanges(changes: SimpleChange) { - // this is a hacky workaround perhaps because of https://github.com/angular/angular/issues/11097 - let dateString: string = '' - let dateAfterChange: SimpleChange - let dateBeforeChange: SimpleChange - if (changes) { - dateAfterChange = changes['dateAfter'] - dateBeforeChange = changes['dateBefore'] + ngOnDestroy() { + if (this.sub) { + this.sub.unsubscribe() } + } - if (this.dpBefore && this.dpAfter) { - let dpAfterElRef: ElementRef = this.dpAfter['_elRef'] - let dpBeforeElRef: ElementRef = this.dpBefore['_elRef'] + setDateQuickFilter(qf: number) { + this.dateBefore = null + let date = new Date() + switch (qf) { + case FILTER_LAST_7_DAYS: + date.setDate(date.getDate() - 7) + break; - if (dateAfterChange && dateAfterChange.currentValue) { - let dateAfterDate = dateAfterChange.currentValue as NgbDateStruct - dateString = `${dateAfterDate.year}-${dateAfterDate.month.toString().padStart(2,'0')}-${dateAfterDate.day.toString().padStart(2,'0')}` - dpAfterElRef.nativeElement.value = dateString - } else if (dateBeforeChange && dateBeforeChange.currentValue) { - let dateBeforeDate = dateBeforeChange.currentValue as NgbDateStruct - dateString = `${dateBeforeDate.year}-${dateBeforeDate.month.toString().padStart(2,'0')}-${dateBeforeDate.day.toString().padStart(2,'0')}` - dpBeforeElRef.nativeElement.value = dateString - } else { - dpAfterElRef.nativeElement.value = dateString - dpBeforeElRef.nativeElement.value = dateString + case FILTER_LAST_MONTH: + date.setMonth(date.getMonth() - 1) + break; + + case FILTER_LAST_3_MONTHS: + date.setMonth(date.getMonth() - 3) + break + + case FILTER_LAST_YEAR: + date.setFullYear(date.getFullYear() - 1) + break + } - } + this.dateAfter = formatDate(date, 'yyyy-MM-dd', "en-us", "UTC") + this.onChange() } - setDateQuickFilter(range: any) { - let date = new Date() - let newDate: NgbDateStruct = { year: date.getFullYear(), month: date.getMonth() + 1, day: date.getDate() } - switch (typeof range) { - case 'number': - date.setDate(date.getDate() - range) - newDate.year = date.getFullYear() - newDate.month = date.getMonth() + 1 - newDate.day = date.getDate() - break - - case 'string': - newDate.day = 1 - if (range == 'year') newDate.month = 1 - break - - default: - break - } - this._dateAfter = newDate - this._dateBefore = null - this.datesSet.emit({after: newDate, before: null}) + onChange() { + this.datesSet.emit({after: this.dateAfter, before: this.dateBefore}) } - onBeforeSelected(date: NgbDateStruct) { - this._dateBefore = date - this.datesSet.emit({after: this._dateAfter, before: date}) + onChangeDebounce() { + this.datesSetDebounce$.next({after: this.dateAfter, before: this.dateBefore}) } - onAfterSelected(date: NgbDateStruct) { - this._dateAfter = date - this.datesSet.emit({after: date, before: this._dateBefore}) + clearBefore() { + this.dateBefore = null; + this.onChange() } - clear() { - this._dateBefore = null - this._dateAfter = null - this.datesSet.emit({after: null, before: null}) + clearAfter() { + this.dateAfter = null; + this.onChange() } + } diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/filter-editor/filter-editor.component.ts index f762c6138..913c738a5 100644 --- a/src-ui/src/app/components/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/filter-editor/filter-editor.component.ts @@ -179,54 +179,53 @@ export class FilterEditorComponent implements OnInit, OnDestroy { this.applyFilters() } - get dateCreatedBefore(): NgbDateStruct { + get dateCreatedBefore(): string { let createdBeforeRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_CREATED_BEFORE) - return createdBeforeRule ? this.dateParser.parse(createdBeforeRule.value) : null + return createdBeforeRule ? createdBeforeRule.value : null } - get dateCreatedAfter(): NgbDateStruct { + get dateCreatedAfter(): string { let createdAfterRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_CREATED_AFTER) - return createdAfterRule ? this.dateParser.parse(createdAfterRule.value) : null + return createdAfterRule ? createdAfterRule.value : null } - get dateAddedBefore(): NgbDateStruct { + get dateAddedBefore(): string { let addedBeforeRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_ADDED_BEFORE) - return addedBeforeRule ? this.dateParser.parse(addedBeforeRule.value) : null + return addedBeforeRule ? addedBeforeRule.value : null } - get dateAddedAfter(): NgbDateStruct { + get dateAddedAfter(): string { let addedAfterRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_ADDED_AFTER) - return addedAfterRule ? this.dateParser.parse(addedAfterRule.value) : null + return addedAfterRule ? addedAfterRule.value : null } - setDateCreatedBefore(date?: NgbDateStruct) { + setDateCreatedBefore(date?: string) { if (date) this.setDateFilter(date, FILTER_CREATED_BEFORE) else this.clearDateFilter(FILTER_CREATED_BEFORE) } - setDateCreatedAfter(date?: NgbDateStruct) { + setDateCreatedAfter(date?: string) { if (date) this.setDateFilter(date, FILTER_CREATED_AFTER) else this.clearDateFilter(FILTER_CREATED_AFTER) } - setDateAddedBefore(date?: NgbDateStruct) { + setDateAddedBefore(date?: string) { if (date) this.setDateFilter(date, FILTER_ADDED_BEFORE) else this.clearDateFilter(FILTER_ADDED_BEFORE) } - setDateAddedAfter(date?: NgbDateStruct) { + setDateAddedAfter(date?: string) { if (date) this.setDateFilter(date, FILTER_ADDED_AFTER) else this.clearDateFilter(FILTER_ADDED_AFTER) } - setDateFilter(date: NgbDateStruct, dateRuleTypeID: number) { + setDateFilter(date: string, dateRuleTypeID: number) { let existingRule = this.filterRules.find(rule => rule.rule_type == dateRuleTypeID) - let newValue = this.dateParser.format(date) if (existingRule) { - existingRule.value = newValue + existingRule.value = date } else { - this.filterRules.push({rule_type: dateRuleTypeID, value: newValue}) + this.filterRules.push({rule_type: dateRuleTypeID, value: date}) } } From e5fe515b69d570b83081d8cc2686635eb2bf8dff Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Fri, 18 Dec 2020 14:31:09 -0800 Subject: [PATCH 13/37] Add page number to toolbar --- .../document-detail.component.html | 12 +++++++++++- .../document-detail.component.ts | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index a3bc7e1e6..0fec2aa44 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -1,4 +1,14 @@ +
+
+
Page
+
+ +
+
of {{previewNumPages}}
+
+
+
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index b4005b920..a2f80f786 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -15,6 +15,7 @@ import { DocumentService } from 'src/app/services/rest/document.service'; import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component'; import { CorrespondentEditDialogComponent } from '../manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component'; import { DocumentTypeEditDialogComponent } from '../manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component'; +import { PDFDocumentProxy } from 'ng2-pdf-viewer'; @Component({ selector: 'app-document-detail', @@ -47,8 +48,11 @@ export class DocumentDetailComponent implements OnInit { tags: new FormControl([]) }) + currentPreviewPage: number = 1 + previewNumPages: number + constructor( - private documentsService: DocumentService, + private documentsService: DocumentService, private route: ActivatedRoute, private correspondentService: CorrespondentService, private documentTypeService: DocumentTypeService, @@ -113,6 +117,8 @@ export class DocumentDetailComponent implements OnInit { modal.componentInstance.success.subscribe(newCorrespondent => { this.correspondentService.listAll().subscribe(correspondents => { this.correspondents = correspondents.results + console.log(this.documentForm.get('correspondent'), this.documentForm.get('correspondent').setValue); + this.documentForm.get('correspondent').setValue(newCorrespondent.id) }) }) @@ -126,7 +132,7 @@ export class DocumentDetailComponent implements OnInit { }, error => {this.router.navigate(['404'])}) } - save() { + save() { this.documentsService.update(this.document).subscribe(result => { this.close() }) @@ -161,7 +167,7 @@ export class DocumentDetailComponent implements OnInit { modal.componentInstance.btnCaption = "Delete document" modal.componentInstance.confirmClicked.subscribe(() => { this.documentsService.delete(this.document).subscribe(() => { - modal.close() + modal.close() this.close() }) }) @@ -171,4 +177,9 @@ export class DocumentDetailComponent implements OnInit { hasNext() { return this.documentListViewService.hasNext(this.documentId) } + + pdfPreviewLoaded(pdf: PDFDocumentProxy) { + this.previewNumPages = pdf.numPages + } + } From f214fe1b3eb5b8b319d7fabeeb847d7a96cddb50 Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Fri, 18 Dec 2020 14:44:17 -0800 Subject: [PATCH 14/37] Log line --- .../app/components/document-detail/document-detail.component.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index a2f80f786..aa3d4e5b8 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -117,8 +117,6 @@ export class DocumentDetailComponent implements OnInit { modal.componentInstance.success.subscribe(newCorrespondent => { this.correspondentService.listAll().subscribe(correspondents => { this.correspondents = correspondents.results - console.log(this.documentForm.get('correspondent'), this.documentForm.get('correspondent').setValue); - this.documentForm.get('correspondent').setValue(newCorrespondent.id) }) }) From 2d841e71673559e8853e4f853ffec96e004f8e0a Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Fri, 18 Dec 2020 14:47:06 -0800 Subject: [PATCH 15/37] Refactor --- .../components/document-detail/document-detail.component.html | 4 ++-- .../components/document-detail/document-detail.component.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index 0fec2aa44..e5dde2ad0 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -3,7 +3,7 @@
Page
- +
of {{previewNumPages}}
@@ -138,7 +138,7 @@
- +
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index aa3d4e5b8..2b839e969 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -48,7 +48,7 @@ export class DocumentDetailComponent implements OnInit { tags: new FormControl([]) }) - currentPreviewPage: number = 1 + previewCurrentPage: number = 1 previewNumPages: number constructor( From e0293db16dde23829e77717a7e7a4e7432b4a65f Mon Sep 17 00:00:00 2001 From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com> Date: Fri, 18 Dec 2020 15:04:52 -0800 Subject: [PATCH 16/37] Only show page numbers when content type is application/pdf --- .../components/document-detail/document-detail.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html index e5dde2ad0..d47c07de1 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.html +++ b/src-ui/src/app/components/document-detail/document-detail.component.html @@ -1,5 +1,5 @@ -
+
Page
From f184e6b16260f38709c0cc8bb65c4a8b64ab27d2 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 01:15:40 +0100 Subject: [PATCH 17/37] fix some layout issues --- .../app/components/document-detail/document-detail.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts index 2b839e969..75a64f548 100644 --- a/src-ui/src/app/components/document-detail/document-detail.component.ts +++ b/src-ui/src/app/components/document-detail/document-detail.component.ts @@ -49,7 +49,7 @@ export class DocumentDetailComponent implements OnInit { }) previewCurrentPage: number = 1 - previewNumPages: number + previewNumPages: number = 1 constructor( private documentsService: DocumentService, From 37237dfcf68221a919aa808625aee368e2915b6b Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 12:46:11 +0100 Subject: [PATCH 18/37] default title --- src/documents/templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/documents/templates/index.html b/src/documents/templates/index.html index 06dbb678e..d086be0fe 100644 --- a/src/documents/templates/index.html +++ b/src/documents/templates/index.html @@ -5,7 +5,7 @@ - PaperlessUi + Paperless-ng From 57a5a4147bd168a1ef2e5b7e614f43499b312fee Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 14:48:42 +0100 Subject: [PATCH 19/37] test case --- src/documents/tests/test_file_handling.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 2e60065f1..b24f52aa2 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -14,7 +14,7 @@ from django.utils import timezone from .utils import DirectoriesMixin from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \ generate_unique_filename -from ..models import Document, Correspondent, Tag +from ..models import Document, Correspondent, Tag, DocumentType class TestFileHandling(DirectoriesMixin, TestCase): @@ -190,6 +190,17 @@ class TestFileHandling(DirectoriesMixin, TestCase): self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), True) self.assertTrue(os.path.isfile(important_file)) + @override_settings(PAPERLESS_FILENAME_FORMAT="{document_type} - {title}") + def test_document_type(self): + dt = DocumentType.objects.create(name="my_doc_type") + d = Document.objects.create(title="the_doc", mime_type="application/pdf") + + self.assertEqual(generate_filename(d), "none - the_doc.pdf") + + d.document_type = dt + + self.assertEqual(generate_filename(d), "my_doc_type - the_doc.pdf") + @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}") def test_tags_with_underscore(self): document = Document() From 1b1b57eb6a5c77ceb3128c44f75ea17ac1865586 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 15:16:42 +0100 Subject: [PATCH 20/37] more tests --- src/paperless/checks.py | 16 +++--- src/paperless/tests/test_checks.py | 55 ++++++++++++++++++++ src/paperless_tesseract/checks.py | 2 +- src/paperless_tesseract/tests/test_checks.py | 26 +++++++++ 4 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 src/paperless/tests/test_checks.py create mode 100644 src/paperless_tesseract/tests/test_checks.py diff --git a/src/paperless/checks.py b/src/paperless/checks.py index 819582ffc..3e26ae69f 100644 --- a/src/paperless/checks.py +++ b/src/paperless/checks.py @@ -13,18 +13,18 @@ writeable_hint = ( ) -def path_check(env_var): +def path_check(var, directory): messages = [] - directory = os.getenv(env_var) + print(directory) if directory: if not os.path.exists(directory): messages.append(Error( - exists_message.format(env_var), + exists_message.format(var), exists_hint.format(directory) )) elif not os.access(directory, os.W_OK | os.X_OK): messages.append(Error( - writeable_message.format(env_var), + writeable_message.format(var), writeable_hint.format(directory) )) return messages @@ -36,10 +36,10 @@ def paths_check(app_configs, **kwargs): Check the various paths for existence, readability and writeability """ - check_messages = path_check("PAPERLESS_DATA_DIR") + \ - path_check("PAPERLESS_MEDIA_ROOT") + \ - path_check("PAPERLESS_CONSUMPTION_DIR") + \ - path_check("PAPERLESS_STATICDIR") + check_messages = path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \ + path_check("PAPERLESS_MEDIA_ROOT", settings.MEDIA_ROOT) + \ + path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR) + \ + path_check("PAPERLESS_STATICDIR", settings.STATIC_ROOT) return check_messages diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py new file mode 100644 index 000000000..61cc05bf0 --- /dev/null +++ b/src/paperless/tests/test_checks.py @@ -0,0 +1,55 @@ +import os +import shutil + +from django.test import TestCase, override_settings + +from documents.tests.utils import DirectoriesMixin +from paperless import binaries_check, paths_check +from paperless.checks import debug_mode_check + + +class TestChecks(DirectoriesMixin, TestCase): + + def test_binaries(self): + self.assertEqual(binaries_check(None), []) + + @override_settings(CONVERT_BINARY="uuuhh", OPTIPNG_BINARY="forgot") + def test_binaries_fail(self): + self.assertEqual(len(binaries_check(None)), 2) + + def test_paths_check(self): + self.assertEqual(paths_check(None), []) + + @override_settings(MEDIA_ROOT="uuh", + STATIC_ROOT="somewhere", + DATA_DIR="whatever", + CONSUMPTION_DIR="idontcare") + def test_paths_check_dont_exist(self): + msgs = paths_check(None) + self.assertEqual(len(msgs), 4) + + for msg in msgs: + self.assertTrue(msg.msg.endswith("is set but doesn't exist.")) + + def test_paths_check_no_access(self): + os.chmod(self.dirs.data_dir, 0o000) + os.chmod(self.dirs.media_dir, 0o000) + os.chmod(self.dirs.consumption_dir, 0o000) + + self.addCleanup(os.chmod, self.dirs.data_dir, 0o777) + self.addCleanup(os.chmod, self.dirs.media_dir, 0o777) + self.addCleanup(os.chmod, self.dirs.consumption_dir, 0o777) + + msgs = paths_check(None) + self.assertEqual(len(msgs), 3) + + for msg in msgs: + self.assertTrue(msg.msg.endswith("is not writeable")) + + @override_settings(DEBUG=False) + def test_debug_disabled(self): + self.assertEqual(debug_mode_check(None), []) + + @override_settings(DEBUG=True) + def test_debug_enabled(self): + self.assertEqual(len(debug_mode_check(None)), 1) diff --git a/src/paperless_tesseract/checks.py b/src/paperless_tesseract/checks.py index 41ea3c9b5..d58b7ac6d 100644 --- a/src/paperless_tesseract/checks.py +++ b/src/paperless_tesseract/checks.py @@ -1,7 +1,7 @@ import subprocess from django.conf import settings -from django.core.checks import Error, register +from django.core.checks import Error, Warning, register def get_tesseract_langs(): diff --git a/src/paperless_tesseract/tests/test_checks.py b/src/paperless_tesseract/tests/test_checks.py new file mode 100644 index 000000000..c4f15764e --- /dev/null +++ b/src/paperless_tesseract/tests/test_checks.py @@ -0,0 +1,26 @@ +from unittest import mock + +from django.core.checks import ERROR +from django.test import TestCase, override_settings + +from paperless_tesseract import check_default_language_available + + +class TestChecks(TestCase): + + def test_default_language(self): + msgs = check_default_language_available(None) + + @override_settings(OCR_LANGUAGE="") + def test_no_language(self): + msgs = check_default_language_available(None) + self.assertEqual(len(msgs), 1) + self.assertTrue(msgs[0].msg.startswith("No OCR language has been specified with PAPERLESS_OCR_LANGUAGE")) + + @override_settings(OCR_LANGUAGE="ita") + @mock.patch("paperless_tesseract.checks.get_tesseract_langs") + def test_invalid_language(self, m): + m.return_value = ["deu", "eng"] + msgs = check_default_language_available(None) + self.assertEqual(len(msgs), 1) + self.assertEqual(msgs[0].level, ERROR) From fad3df1e39037af3ea4d7dcaf8220dae9f97ef93 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 16:46:04 +0100 Subject: [PATCH 21/37] removed x-frame-options, since that was only used for the pdf display tag. --- src/paperless/settings.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 1a6b80a0c..c6f7c9357 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -160,13 +160,6 @@ if AUTO_LOGIN_USERNAME: MIDDLEWARE.insert(_index+1, 'paperless.auth.AutoLoginMiddleware') -if DEBUG: - X_FRAME_OPTIONS = '' - # this should really be 'allow-from uri' but its not supported in any mayor - # browser. -else: - X_FRAME_OPTIONS = 'SAMEORIGIN' - # We allow CORS from localhost:8080 CORS_ALLOWED_ORIGINS = tuple(os.getenv("PAPERLESS_CORS_ALLOWED_HOSTS", "http://localhost:8000").split(",")) From bb814da95b1e5efecdfc3cdafd0a5fb772ed2281 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 16:46:09 +0100 Subject: [PATCH 22/37] more test. --- src/documents/tests/test_management.py | 135 ++++++++++++++++++ .../tests/test_management_archiver.py | 40 ------ .../tests/test_management_decrypt.py | 57 -------- 3 files changed, 135 insertions(+), 97 deletions(-) create mode 100644 src/documents/tests/test_management.py delete mode 100644 src/documents/tests/test_management_archiver.py delete mode 100644 src/documents/tests/test_management_decrypt.py diff --git a/src/documents/tests/test_management.py b/src/documents/tests/test_management.py new file mode 100644 index 000000000..58aaf9342 --- /dev/null +++ b/src/documents/tests/test_management.py @@ -0,0 +1,135 @@ +import hashlib +import tempfile +import filecmp +import os +import shutil +from pathlib import Path +from unittest import mock + +from django.test import TestCase, override_settings + + +from django.core.management import call_command + +from documents.file_handling import generate_filename +from documents.management.commands.document_archiver import handle_document +from documents.models import Document +from documents.tests.utils import DirectoriesMixin + + +sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf") + + +class TestArchiver(DirectoriesMixin, TestCase): + + def make_models(self): + return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf") + + def test_archiver(self): + + doc = self.make_models() + shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) + + call_command('document_archiver') + + def test_handle_document(self): + + doc = self.make_models() + shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) + + handle_document(doc.pk) + + doc = Document.objects.get(id=doc.id) + + self.assertIsNotNone(doc.checksum) + self.assertTrue(os.path.isfile(doc.archive_path)) + self.assertTrue(os.path.isfile(doc.source_path)) + self.assertTrue(filecmp.cmp(sample_file, doc.source_path)) + + +class TestDecryptDocuments(TestCase): + + @override_settings( + ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"), + THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"), + PASSPHRASE="test", + PAPERLESS_FILENAME_FORMAT=None + ) + @mock.patch("documents.management.commands.decrypt_documents.input") + def test_decrypt(self, m): + + media_dir = tempfile.mkdtemp() + originals_dir = os.path.join(media_dir, "documents", "originals") + thumb_dir = os.path.join(media_dir, "documents", "thumbnails") + os.makedirs(originals_dir, exist_ok=True) + os.makedirs(thumb_dir, exist_ok=True) + + override_settings( + ORIGINALS_DIR=originals_dir, + THUMBNAIL_DIR=thumb_dir, + PASSPHRASE="test" + ).enable() + + doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) + + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg")) + + call_command('decrypt_documents') + + doc.refresh_from_db() + + self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED) + self.assertEqual(doc.filename, "0000002.pdf") + self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf"))) + self.assertTrue(os.path.isfile(doc.source_path)) + self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png"))) + self.assertTrue(os.path.isfile(doc.thumbnail_path)) + + with doc.source_file as f: + checksum = hashlib.md5(f.read()).hexdigest() + self.assertEqual(checksum, doc.checksum) + + +class TestMakeIndex(TestCase): + + @mock.patch("documents.management.commands.document_index.index_reindex") + def test_reindex(self, m): + call_command("document_index", "reindex") + m.assert_called_once() + + @mock.patch("documents.management.commands.document_index.index_optimize") + def test_optimize(self, m): + call_command("document_index", "optimize") + m.assert_called_once() + + +class TestRenamer(DirectoriesMixin, TestCase): + + def test_rename(self): + doc = Document.objects.create(title="test", mime_type="application/pdf") + doc.filename = generate_filename(doc) + doc.save() + + Path(doc.source_path).touch() + + old_source_path = doc.source_path + + with override_settings(PAPERLESS_FILENAME_FORMAT="{title}"): + call_command("document_renamer") + + doc2 = Document.objects.get(id=doc.id) + + self.assertEqual(doc2.filename, "test.pdf") + self.assertFalse(os.path.isfile(old_source_path)) + self.assertFalse(os.path.isfile(doc.source_path)) + self.assertTrue(os.path.isfile(doc2.source_path)) + + +class TestCreateClassifier(TestCase): + + @mock.patch("documents.management.commands.document_create_classifier.train_classifier") + def test_create_classifier(self, m): + call_command("document_create_classifier") + + m.assert_called_once() diff --git a/src/documents/tests/test_management_archiver.py b/src/documents/tests/test_management_archiver.py deleted file mode 100644 index 0828f05ff..000000000 --- a/src/documents/tests/test_management_archiver.py +++ /dev/null @@ -1,40 +0,0 @@ -import filecmp -import os -import shutil - -from django.core.management import call_command -from django.test import TestCase - -from documents.management.commands.document_archiver import handle_document -from documents.models import Document -from documents.tests.utils import DirectoriesMixin - - -sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf") - - -class TestArchiver(DirectoriesMixin, TestCase): - - def make_models(self): - return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf") - - def test_archiver(self): - - doc = self.make_models() - shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) - - call_command('document_archiver') - - def test_handle_document(self): - - doc = self.make_models() - shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf")) - - handle_document(doc.pk) - - doc = Document.objects.get(id=doc.id) - - self.assertIsNotNone(doc.checksum) - self.assertTrue(os.path.isfile(doc.archive_path)) - self.assertTrue(os.path.isfile(doc.source_path)) - self.assertTrue(filecmp.cmp(sample_file, doc.source_path)) diff --git a/src/documents/tests/test_management_decrypt.py b/src/documents/tests/test_management_decrypt.py deleted file mode 100644 index 1d64b1105..000000000 --- a/src/documents/tests/test_management_decrypt.py +++ /dev/null @@ -1,57 +0,0 @@ -import hashlib -import json -import os -import shutil -import tempfile -from unittest import mock - -from django.core.management import call_command -from django.test import TestCase, override_settings - -from documents.management.commands import document_exporter -from documents.models import Document, Tag, DocumentType, Correspondent - - -class TestDecryptDocuments(TestCase): - - @override_settings( - ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"), - THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"), - PASSPHRASE="test", - PAPERLESS_FILENAME_FORMAT=None - ) - @mock.patch("documents.management.commands.decrypt_documents.input") - def test_decrypt(self, m): - - media_dir = tempfile.mkdtemp() - originals_dir = os.path.join(media_dir, "documents", "originals") - thumb_dir = os.path.join(media_dir, "documents", "thumbnails") - os.makedirs(originals_dir, exist_ok=True) - os.makedirs(thumb_dir, exist_ok=True) - - override_settings( - ORIGINALS_DIR=originals_dir, - THUMBNAIL_DIR=thumb_dir, - PASSPHRASE="test" - ).enable() - - doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) - - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg")) - shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg")) - - call_command('decrypt_documents') - - doc.refresh_from_db() - - self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED) - self.assertEqual(doc.filename, "0000002.pdf") - self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf"))) - self.assertTrue(os.path.isfile(doc.source_path)) - self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png"))) - self.assertTrue(os.path.isfile(doc.thumbnail_path)) - - with doc.source_file as f: - checksum = hashlib.md5(f.read()).hexdigest() - self.assertEqual(checksum, doc.checksum) - From e79c45c98d5164de4c99888c1a9cbd7553ab3bcc Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 20:39:56 +0100 Subject: [PATCH 23/37] fix test cases --- src/paperless/checks.py | 7 ++----- src/paperless/tests/test_checks.py | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/paperless/checks.py b/src/paperless/checks.py index 3e26ae69f..1e74b30a9 100644 --- a/src/paperless/checks.py +++ b/src/paperless/checks.py @@ -36,12 +36,9 @@ def paths_check(app_configs, **kwargs): Check the various paths for existence, readability and writeability """ - check_messages = path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \ + return path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \ path_check("PAPERLESS_MEDIA_ROOT", settings.MEDIA_ROOT) + \ - path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR) + \ - path_check("PAPERLESS_STATICDIR", settings.STATIC_ROOT) - - return check_messages + path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR) @register() diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py index 61cc05bf0..e1525cab8 100644 --- a/src/paperless/tests/test_checks.py +++ b/src/paperless/tests/test_checks.py @@ -21,12 +21,11 @@ class TestChecks(DirectoriesMixin, TestCase): self.assertEqual(paths_check(None), []) @override_settings(MEDIA_ROOT="uuh", - STATIC_ROOT="somewhere", DATA_DIR="whatever", CONSUMPTION_DIR="idontcare") def test_paths_check_dont_exist(self): msgs = paths_check(None) - self.assertEqual(len(msgs), 4) + self.assertEqual(len(msgs), 3, str(msgs)) for msg in msgs: self.assertTrue(msg.msg.endswith("is set but doesn't exist.")) From 3f94fc2618e5f543c3cbaeed46cd01749a6e8f66 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sat, 19 Dec 2020 21:52:58 +0100 Subject: [PATCH 24/37] fix & test a migration --- src/documents/migrations/1003_mime_types.py | 1 + src/documents/tests/test_migrations.py | 129 ++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 src/documents/tests/test_migrations.py diff --git a/src/documents/migrations/1003_mime_types.py b/src/documents/migrations/1003_mime_types.py index 78ecced2b..c196f29f4 100644 --- a/src/documents/migrations/1003_mime_types.py +++ b/src/documents/migrations/1003_mime_types.py @@ -11,6 +11,7 @@ from paperless.db import GnuPG STORAGE_TYPE_UNENCRYPTED = "unencrypted" STORAGE_TYPE_GPG = "gpg" + def source_path(self): if self.filename: fname = str(self.filename) diff --git a/src/documents/tests/test_migrations.py b/src/documents/tests/test_migrations.py new file mode 100644 index 000000000..33ba41444 --- /dev/null +++ b/src/documents/tests/test_migrations.py @@ -0,0 +1,129 @@ +import os +import shutil +from pathlib import Path + +from django.apps import apps +from django.conf import settings +from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.test import TestCase, TransactionTestCase, override_settings + +from documents.models import Document +from documents.parsers import get_default_file_extension +from documents.tests.utils import DirectoriesMixin + + +class TestMigrations(TransactionTestCase): + + @property + def app(self): + return apps.get_containing_app_config(type(self).__module__).name + + migrate_from = None + migrate_to = None + + def setUp(self): + super(TestMigrations, self).setUp() + + assert self.migrate_from and self.migrate_to, \ + "TestCase '{}' must define migrate_from and migrate_to properties".format(type(self).__name__) + self.migrate_from = [(self.app, self.migrate_from)] + self.migrate_to = [(self.app, self.migrate_to)] + executor = MigrationExecutor(connection) + old_apps = executor.loader.project_state(self.migrate_from).apps + + # Reverse to the original migration + executor.migrate(self.migrate_from) + + self.setUpBeforeMigration(old_apps) + + # Run the migration to test + executor = MigrationExecutor(connection) + executor.loader.build_graph() # reload. + executor.migrate(self.migrate_to) + + self.apps = executor.loader.project_state(self.migrate_to).apps + + def setUpBeforeMigration(self, apps): + pass + + +STORAGE_TYPE_UNENCRYPTED = "unencrypted" +STORAGE_TYPE_GPG = "gpg" + + +def source_path_before(self): + if self.filename: + fname = str(self.filename) + else: + fname = "{:07}.{}".format(self.pk, self.file_type) + if self.storage_type == STORAGE_TYPE_GPG: + fname += ".gpg" + + return os.path.join( + settings.ORIGINALS_DIR, + fname + ) + + +def file_type_after(self): + return get_default_file_extension(self.mime_type) + + +def source_path_after(doc): + if doc.filename: + fname = str(doc.filename) + else: + fname = "{:07}{}".format(doc.pk, file_type_after(doc)) + if doc.storage_type == STORAGE_TYPE_GPG: + fname += ".gpg" # pragma: no cover + + return os.path.join( + settings.ORIGINALS_DIR, + fname + ) + + +@override_settings(PASSPHRASE="test") +class TestMigrateMimeType(DirectoriesMixin, TestMigrations): + + migrate_from = '1002_auto_20201111_1105' + migrate_to = '1003_mime_types' + + def setUpBeforeMigration(self, apps): + Document = apps.get_model("documents", "Document") + doc = Document.objects.create(title="test", file_type="pdf", filename="file1.pdf") + self.doc_id = doc.id + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_before(doc)) + + doc2 = Document.objects.create(checksum="B", file_type="pdf", storage_type=STORAGE_TYPE_GPG) + self.doc2_id = doc2.id + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), source_path_before(doc2)) + + def testMimeTypesMigrated(self): + Document = self.apps.get_model('documents', 'Document') + + doc = Document.objects.get(id=self.doc_id) + self.assertEqual(doc.mime_type, "application/pdf") + + doc2 = Document.objects.get(id=self.doc2_id) + self.assertEqual(doc2.mime_type, "application/pdf") + + +@override_settings(PASSPHRASE="test") +class TestMigrateMimeTypeBackwards(DirectoriesMixin, TestMigrations): + + migrate_from = '1003_mime_types' + migrate_to = '1002_auto_20201111_1105' + + def setUpBeforeMigration(self, apps): + Document = apps.get_model("documents", "Document") + doc = Document.objects.create(title="test", mime_type="application/pdf", filename="file1.pdf") + self.doc_id = doc.id + shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_after(doc)) + + def testMimeTypesReverted(self): + Document = self.apps.get_model('documents', 'Document') + + doc = Document.objects.get(id=self.doc_id) + self.assertEqual(doc.file_type, "pdf") From 32224f187dd8fa8ec4f6f6ad019c6014a33af0e8 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 00:06:33 +0100 Subject: [PATCH 25/37] test CONSUMER_DELETE_DUPLICATES --- src/documents/tests/test_consumer.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index b4b19be4c..75d6aa16b 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -642,3 +642,31 @@ class TestConsumer(DirectoriesMixin, TestCase): self.assertEqual(document.document_type, dtype) self.assertIn(t1, document.tags.all()) self.assertNotIn(t2, document.tags.all()) + + @override_settings(CONSUMER_DELETE_DUPLICATES=True) + def test_delete_duplicate(self): + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + doc = self.consumer.try_consume_file(dst) + + self.assertFalse(os.path.isfile(dst)) + self.assertIsNotNone(doc) + + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst) + self.assertFalse(os.path.isfile(dst)) + + @override_settings(CONSUMER_DELETE_DUPLICATES=False) + def test_no_delete_duplicate(self): + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + doc = self.consumer.try_consume_file(dst) + + self.assertFalse(os.path.isfile(dst)) + self.assertIsNotNone(doc) + + dst = self.get_test_file() + self.assertTrue(os.path.isfile(dst)) + self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst) + self.assertTrue(os.path.isfile(dst)) From 7f9a0204b59239088d2e47aec8d797d12d1a581a Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 20 Dec 2020 00:08:05 +0100 Subject: [PATCH 26/37] removed most of the logic that extracts data from filename patterns #156 --- src/documents/consumer.py | 7 - src/documents/models.py | 66 --------- src/documents/tests/test_consumer.py | 212 +-------------------------- 3 files changed, 4 insertions(+), 281 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index e4da51f1d..ab4912a36 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -247,7 +247,6 @@ class Consumer(LoggingMixin): with open(self.path, "rb") as f: document = Document.objects.create( - correspondent=file_info.correspondent, title=(self.override_title or file_info.title)[:127], content=text, mime_type=mime_type, @@ -257,12 +256,6 @@ class Consumer(LoggingMixin): storage_type=storage_type ) - relevant_tags = set(file_info.tags) - if relevant_tags: - tag_names = ", ".join([t.name for t in relevant_tags]) - self.log("debug", "Tagging with {}".format(tag_names)) - document.tags.add(*relevant_tags) - self.apply_overrides(document) document.save() diff --git a/src/documents/models.py b/src/documents/models.py index 3a6d155ed..168dd8c7b 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -357,54 +357,12 @@ class SavedViewFilterRule(models.Model): # TODO: why is this in the models file? class FileInfo: - # This epic regex *almost* worked for our needs, so I'm keeping it here for - # posterity, in the hopes that we might find a way to make it work one day. - ALMOST_REGEX = re.compile( - r"^((?P\d\d\d\d\d\d\d\d\d\d\d\d\d\dZ){separator})?" - r"((?P{non_separated_word}+){separator})??" - r"(?P{non_separated_word}+)" - r"({separator}(?P<tags>[a-z,0-9-]+))?" - r"\.(?P<extension>[a-zA-Z.-]+)$".format( - separator=r"\s+-\s+", - non_separated_word=r"([\w,. ]|([^\s]-))" - ) - ) REGEXES = OrderedDict([ - ("created-correspondent-title-tags", re.compile( - r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " - r"(?P<correspondent>.*) - " - r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)$", - flags=re.IGNORECASE - )), - ("created-title-tags", re.compile( - r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " - r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)$", - flags=re.IGNORECASE - )), - ("created-correspondent-title", re.compile( - r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " - r"(?P<correspondent>.*) - " - r"(?P<title>.*)$", - flags=re.IGNORECASE - )), ("created-title", re.compile( r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - " r"(?P<title>.*)$", flags=re.IGNORECASE )), - ("correspondent-title-tags", re.compile( - r"(?P<correspondent>.*) - " - r"(?P<title>.*) - " - r"(?P<tags>[a-z0-9\-,]*)$", - flags=re.IGNORECASE - )), - ("correspondent-title", re.compile( - r"(?P<correspondent>.*) - " - r"(?P<title>.*)?$", - flags=re.IGNORECASE - )), ("title", re.compile( r"(?P<title>.*)$", flags=re.IGNORECASE @@ -427,23 +385,10 @@ class FileInfo: except ValueError: return None - @classmethod - def _get_correspondent(cls, name): - if not name: - return None - return Correspondent.objects.get_or_create(name=name)[0] - @classmethod def _get_title(cls, title): return title - @classmethod - def _get_tags(cls, tags): - r = [] - for t in tags.split(","): - r.append(Tag.objects.get_or_create(name=t)[0]) - return tuple(r) - @classmethod def _mangle_property(cls, properties, name): if name in properties: @@ -453,15 +398,6 @@ class FileInfo: @classmethod def from_filename(cls, filename): - """ - We use a crude naming convention to make handling the correspondent, - title, and tags easier: - "<date> - <correspondent> - <title> - <tags>" - "<correspondent> - <title> - <tags>" - "<correspondent> - <title>" - "<title>" - """ - # Mutate filename in-place before parsing its components # by applying at most one of the configured transformations. for (pattern, repl) in settings.FILENAME_PARSE_TRANSFORMS: @@ -492,7 +428,5 @@ class FileInfo: if m: properties = m.groupdict() cls._mangle_property(properties, "created") - cls._mangle_property(properties, "correspondent") cls._mangle_property(properties, "title") - cls._mangle_property(properties, "tags") return cls(**properties) diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index 75d6aa16b..f53981850 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -29,81 +29,6 @@ class TestAttributes(TestCase): self.assertEqual(tuple([t.name for t in file_info.tags]), tags, filename) - def test_guess_attributes_from_name0(self): - self._test_guess_attributes_from_name( - "Sender - Title.pdf", "Sender", "Title", ()) - - def test_guess_attributes_from_name1(self): - self._test_guess_attributes_from_name( - "Spaced Sender - Title.pdf", "Spaced Sender", "Title", ()) - - def test_guess_attributes_from_name2(self): - self._test_guess_attributes_from_name( - "Sender - Spaced Title.pdf", "Sender", "Spaced Title", ()) - - def test_guess_attributes_from_name3(self): - self._test_guess_attributes_from_name( - "Dashed-Sender - Title.pdf", "Dashed-Sender", "Title", ()) - - def test_guess_attributes_from_name4(self): - self._test_guess_attributes_from_name( - "Sender - Dashed-Title.pdf", "Sender", "Dashed-Title", ()) - - def test_guess_attributes_from_name5(self): - self._test_guess_attributes_from_name( - "Sender - Title - tag1,tag2,tag3.pdf", - "Sender", - "Title", - self.TAGS - ) - - def test_guess_attributes_from_name6(self): - self._test_guess_attributes_from_name( - "Spaced Sender - Title - tag1,tag2,tag3.pdf", - "Spaced Sender", - "Title", - self.TAGS - ) - - def test_guess_attributes_from_name7(self): - self._test_guess_attributes_from_name( - "Sender - Spaced Title - tag1,tag2,tag3.pdf", - "Sender", - "Spaced Title", - self.TAGS - ) - - def test_guess_attributes_from_name8(self): - self._test_guess_attributes_from_name( - "Dashed-Sender - Title - tag1,tag2,tag3.pdf", - "Dashed-Sender", - "Title", - self.TAGS - ) - - def test_guess_attributes_from_name9(self): - self._test_guess_attributes_from_name( - "Sender - Dashed-Title - tag1,tag2,tag3.pdf", - "Sender", - "Dashed-Title", - self.TAGS - ) - - def test_guess_attributes_from_name10(self): - self._test_guess_attributes_from_name( - "Σενδερ - Τιτλε - tag1,tag2,tag3.pdf", - "Σενδερ", - "Τιτλε", - self.TAGS - ) - - def test_guess_attributes_from_name_when_correspondent_empty(self): - self._test_guess_attributes_from_name( - ' - weird empty correspondent but should not break.pdf', - None, - 'weird empty correspondent but should not break', - () - ) def test_guess_attributes_from_name_when_title_starts_with_dash(self): self._test_guess_attributes_from_name( @@ -121,28 +46,6 @@ class TestAttributes(TestCase): () ) - def test_guess_attributes_from_name_when_title_is_empty(self): - self._test_guess_attributes_from_name( - 'weird correspondent but should not break - .pdf', - 'weird correspondent but should not break', - '', - () - ) - - def test_case_insensitive_tag_creation(self): - """ - Tags should be detected and created as lower case. - :return: - """ - - filename = "Title - Correspondent - tAg1,TAG2.pdf" - self.assertEqual(len(FileInfo.from_filename(filename).tags), 2) - - path = "Title - Correspondent - tag1,tag2.pdf" - self.assertEqual(len(FileInfo.from_filename(filename).tags), 2) - - self.assertEqual(Tag.objects.all().count(), 2) - class TestFieldPermutations(TestCase): @@ -199,69 +102,7 @@ class TestFieldPermutations(TestCase): filename = template.format(**spec) self._test_guessed_attributes(filename, **spec) - def test_title_and_correspondent(self): - template = '{correspondent} - {title}.pdf' - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - spec = dict(correspondent=correspondent, title=title) - filename = template.format(**spec) - self._test_guessed_attributes(filename, **spec) - - def test_title_and_correspondent_and_tags(self): - template = '{correspondent} - {title} - {tags}.pdf' - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - for tags in self.valid_tags: - spec = dict(correspondent=correspondent, title=title, - tags=tags) - filename = template.format(**spec) - self._test_guessed_attributes(filename, **spec) - - def test_created_and_correspondent_and_title_and_tags(self): - - template = ( - "{created} - " - "{correspondent} - " - "{title} - " - "{tags}.pdf" - ) - - for created in self.valid_dates: - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - for tags in self.valid_tags: - spec = { - "created": created, - "correspondent": correspondent, - "title": title, - "tags": tags, - } - self._test_guessed_attributes( - template.format(**spec), **spec) - - def test_created_and_correspondent_and_title(self): - - template = "{created} - {correspondent} - {title}.pdf" - - for created in self.valid_dates: - for correspondent in self.valid_correspondents: - for title in self.valid_titles: - - # Skip cases where title looks like a tag as we can't - # accommodate such cases. - if title.lower() == title: - continue - - spec = { - "created": created, - "correspondent": correspondent, - "title": title - } - self._test_guessed_attributes( - template.format(**spec), **spec) - def test_created_and_title(self): - template = "{created} - {title}.pdf" for created in self.valid_dates: @@ -273,21 +114,6 @@ class TestFieldPermutations(TestCase): self._test_guessed_attributes( template.format(**spec), **spec) - def test_created_and_title_and_tags(self): - - template = "{created} - {title} - {tags}.pdf" - - for created in self.valid_dates: - for title in self.valid_titles: - for tags in self.valid_tags: - spec = { - "created": created, - "title": title, - "tags": tags - } - self._test_guessed_attributes( - template.format(**spec), **spec) - def test_invalid_date_format(self): info = FileInfo.from_filename("06112017Z - title.pdf") self.assertEqual(info.title, "title") @@ -336,32 +162,6 @@ class TestFieldPermutations(TestCase): info = FileInfo.from_filename(filename) self.assertEqual(info.title, "anotherall") - # Complex transformation without date in replacement string - with self.settings( - FILENAME_PARSE_TRANSFORMS=[(exact_patt, repl1)]): - info = FileInfo.from_filename(filename) - self.assertEqual(info.title, "0001") - self.assertEqual(len(info.tags), 2) - self.assertEqual(info.tags[0].name, "tag1") - self.assertEqual(info.tags[1].name, "tag2") - self.assertIsNone(info.created) - - # Complex transformation with date in replacement string - with self.settings( - FILENAME_PARSE_TRANSFORMS=[ - (none_patt, "none.gif"), - (exact_patt, repl2), # <-- matches - (exact_patt, repl1), - (all_patt, "all.gif")]): - info = FileInfo.from_filename(filename) - self.assertEqual(info.title, "0001") - self.assertEqual(len(info.tags), 2) - self.assertEqual(info.tags[0].name, "tag1") - self.assertEqual(info.tags[1].name, "tag2") - self.assertEqual(info.created.year, 2019) - self.assertEqual(info.created.month, 9) - self.assertEqual(info.created.day, 8) - class DummyParser(DocumentParser): @@ -476,15 +276,13 @@ class TestConsumer(DirectoriesMixin, TestCase): def testOverrideFilename(self): filename = self.get_test_file() - override_filename = "My Bank - Statement for November.pdf" + override_filename = "Statement for November.pdf" document = self.consumer.try_consume_file(filename, override_filename=override_filename) - self.assertEqual(document.correspondent.name, "My Bank") self.assertEqual(document.title, "Statement for November") def testOverrideTitle(self): - document = self.consumer.try_consume_file(self.get_test_file(), override_title="Override Title") self.assertEqual(document.title, "Override Title") @@ -594,11 +392,10 @@ class TestConsumer(DirectoriesMixin, TestCase): def testFilenameHandling(self): filename = self.get_test_file() - document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs") + document = self.consumer.try_consume_file(filename, override_title="new docs") self.assertEqual(document.title, "new docs") - self.assertEqual(document.correspondent.name, "Bank") - self.assertEqual(document.filename, "Bank/new docs.pdf") + self.assertEqual(document.filename, "none/new docs.pdf") @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}") @mock.patch("documents.signals.handlers.generate_unique_filename") @@ -617,10 +414,9 @@ class TestConsumer(DirectoriesMixin, TestCase): Tag.objects.create(name="test", is_inbox_tag=True) - document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs") + document = self.consumer.try_consume_file(filename, override_title="new docs") self.assertEqual(document.title, "new docs") - self.assertEqual(document.correspondent.name, "Bank") self.assertIsNotNone(os.path.isfile(document.title)) self.assertTrue(os.path.isfile(document.source_path)) From ee31fdc650c6f9df91ec2331b24c289cb03b466b Mon Sep 17 00:00:00 2001 From: jonaswinkler <jonas.winkler@jpwinkler.de> Date: Sun, 20 Dec 2020 13:59:25 +0100 Subject: [PATCH 27/37] removed unused code --- src/paperless_tesseract/languages.py | 194 --------------------------- src/paperless_text/parsers.py | 12 -- 2 files changed, 206 deletions(-) delete mode 100644 src/paperless_tesseract/languages.py diff --git a/src/paperless_tesseract/languages.py b/src/paperless_tesseract/languages.py deleted file mode 100644 index 5ea560654..000000000 --- a/src/paperless_tesseract/languages.py +++ /dev/null @@ -1,194 +0,0 @@ -# Thanks to the Library of Congress and some creative use of sed and awk: -# http://www.loc.gov/standards/iso639-2/php/English_list.php - -ISO639 = { - - "aa": "aar", - "ab": "abk", - "ae": "ave", - "af": "afr", - "ak": "aka", - "am": "amh", - "an": "arg", - "ar": "ara", - "as": "asm", - "av": "ava", - "ay": "aym", - "az": "aze", - "ba": "bak", - "be": "bel", - "bg": "bul", - "bh": "bih", - "bi": "bis", - "bm": "bam", - "bn": "ben", - "bo": "bod", - "br": "bre", - "bs": "bos", - "ca": "cat", - "ce": "che", - "ch": "cha", - "co": "cos", - "cr": "cre", - "cs": "ces", - "cu": "chu", - "cv": "chv", - "cy": "cym", - "da": "dan", - "de": "deu", - "dv": "div", - "dz": "dzo", - "ee": "ewe", - "el": "ell", - "en": "eng", - "eo": "epo", - "es": "spa", - "et": "est", - "eu": "eus", - "fa": "fas", - "ff": "ful", - "fi": "fin", - "fj": "fij", - "fo": "fao", - "fr": "fra", - "fy": "fry", - "ga": "gle", - "gd": "gla", - "gl": "glg", - "gn": "grn", - "gu": "guj", - "gv": "glv", - "ha": "hau", - "he": "heb", - "hi": "hin", - "ho": "hmo", - "hr": "hrv", - "ht": "hat", - "hu": "hun", - "hy": "hye", - "hz": "her", - "ia": "ina", - "id": "ind", - "ie": "ile", - "ig": "ibo", - "ii": "iii", - "ik": "ipk", - "io": "ido", - "is": "isl", - "it": "ita", - "iu": "iku", - "ja": "jpn", - "jv": "jav", - "ka": "kat", - "kg": "kon", - "ki": "kik", - "kj": "kua", - "kk": "kaz", - "kl": "kal", - "km": "khm", - "kn": "kan", - "ko": "kor", - "kr": "kau", - "ks": "kas", - "ku": "kur", - "kv": "kom", - "kw": "cor", - "ky": "kir", - "la": "lat", - "lb": "ltz", - "lg": "lug", - "li": "lim", - "ln": "lin", - "lo": "lao", - "lt": "lit", - "lu": "lub", - "lv": "lav", - "mg": "mlg", - "mh": "mah", - "mi": "mri", - "mk": "mkd", - "ml": "mal", - "mn": "mon", - "mr": "mar", - "ms": "msa", - "mt": "mlt", - "my": "mya", - "na": "nau", - "nb": "nob", - "nd": "nde", - "ne": "nep", - "ng": "ndo", - "nl": "nld", - "no": "nor", - "nr": "nbl", - "nv": "nav", - "ny": "nya", - "oc": "oci", - "oj": "oji", - "om": "orm", - "or": "ori", - "os": "oss", - "pa": "pan", - "pi": "pli", - "pl": "pol", - "ps": "pus", - "pt": "por", - "qu": "que", - "rm": "roh", - "rn": "run", - "ro": "ron", - "ru": "rus", - "rw": "kin", - "sa": "san", - "sc": "srd", - "sd": "snd", - "se": "sme", - "sg": "sag", - "si": "sin", - "sk": "slk", - "sl": "slv", - "sm": "smo", - "sn": "sna", - "so": "som", - "sq": "sqi", - "sr": "srp", - "ss": "ssw", - "st": "sot", - "su": "sun", - "sv": "swe", - "sw": "swa", - "ta": "tam", - "te": "tel", - "tg": "tgk", - "th": "tha", - "ti": "tir", - "tk": "tuk", - "tl": "tgl", - "tn": "tsn", - "to": "ton", - "tr": "tur", - "ts": "tso", - "tt": "tat", - "tw": "twi", - "ty": "tah", - "ug": "uig", - "uk": "ukr", - "ur": "urd", - "uz": "uzb", - "ve": "ven", - "vi": "vie", - "vo": "vol", - "wa": "wln", - "wo": "wol", - "xh": "xho", - "yi": "yid", - "yo": "yor", - "za": "zha", - - # Tessdata contains two values for Chinese, "chi_sim" and "chi_tra". I - # have no idea which one is better, so I just picked the bigger file. - "zh": "chi_tra", - - "zu": "zul" - -} diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py index 7e488ca37..030c2c2c2 100644 --- a/src/paperless_text/parsers.py +++ b/src/paperless_text/parsers.py @@ -35,15 +35,3 @@ class TextDocumentParser(DocumentParser): def parse(self, document_path, mime_type): with open(document_path, 'r') as f: self.text = f.read() - - -def run_command(*args): - environment = os.environ.copy() - if settings.CONVERT_MEMORY_LIMIT: - environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT - if settings.CONVERT_TMPDIR: - environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR - - if not subprocess.Popen(' '.join(args), env=environment, - shell=True).wait() == 0: - raise ParseError("Convert failed at {}".format(args)) From 01c2fe508e084e9b310ba7693a094f249fa78dbe Mon Sep 17 00:00:00 2001 From: jonaswinkler <jonas.winkler@jpwinkler.de> Date: Sun, 20 Dec 2020 14:39:17 +0100 Subject: [PATCH 28/37] more tests --- src/documents/tests/test_admin.py | 57 +++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 src/documents/tests/test_admin.py diff --git a/src/documents/tests/test_admin.py b/src/documents/tests/test_admin.py new file mode 100644 index 000000000..b280c43ea --- /dev/null +++ b/src/documents/tests/test_admin.py @@ -0,0 +1,57 @@ +from unittest import mock + +from django.contrib.admin.sites import AdminSite +from django.test import TestCase +from django.utils import timezone + +from documents.admin import DocumentAdmin +from documents.models import Document, Tag + + +class TestDocumentAdmin(TestCase): + + def setUp(self) -> None: + self.doc_admin = DocumentAdmin(model=Document, admin_site=AdminSite()) + + @mock.patch("documents.admin.index.add_or_update_document") + def test_save_model(self, m): + doc = Document.objects.create(title="test") + doc.title = "new title" + self.doc_admin.save_model(None, doc, None, None) + self.assertEqual(Document.objects.get(id=doc.id).title, "new title") + m.assert_called_once() + + def test_tags(self): + doc = Document.objects.create(title="test") + doc.tags.create(name="t1") + doc.tags.create(name="t2") + + self.assertEqual(self.doc_admin.tags_(doc), "<span >t1, </span><span >t2, </span>") + + def test_tags_empty(self): + doc = Document.objects.create(title="test") + + self.assertEqual(self.doc_admin.tags_(doc), "") + + @mock.patch("documents.admin.index.remove_document") + def test_delete_model(self, m): + doc = Document.objects.create(title="test") + self.doc_admin.delete_model(None, doc) + self.assertRaises(Document.DoesNotExist, Document.objects.get, id=doc.id) + m.assert_called_once() + + @mock.patch("documents.admin.index.remove_document") + def test_delete_queryset(self, m): + for i in range(42): + Document.objects.create(title="Many documents with the same title", checksum=f"{i:02}") + + self.assertEqual(Document.objects.count(), 42) + + self.doc_admin.delete_queryset(None, Document.objects.all()) + + self.assertEqual(m.call_count, 42) + self.assertEqual(Document.objects.count(), 0) + + def test_created(self): + doc = Document.objects.create(title="test", created=timezone.datetime(2020, 4, 12)) + self.assertEqual(self.doc_admin.created_(doc), "2020-04-12") From b10e7abbe8f42af1ea241dc8adcdd2cfb58b342a Mon Sep 17 00:00:00 2001 From: jonaswinkler <jonas.winkler@jpwinkler.de> Date: Sun, 20 Dec 2020 15:25:47 +0100 Subject: [PATCH 29/37] don't know how that got in here --- src/paperless/checks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/paperless/checks.py b/src/paperless/checks.py index 1e74b30a9..1329ad679 100644 --- a/src/paperless/checks.py +++ b/src/paperless/checks.py @@ -15,7 +15,6 @@ writeable_hint = ( def path_check(var, directory): messages = [] - print(directory) if directory: if not os.path.exists(directory): messages.append(Error( From 240d5b9da214f5d56fc1b4e3229ceb3578e76ab1 Mon Sep 17 00:00:00 2001 From: jonaswinkler <jonas.winkler@jpwinkler.de> Date: Sun, 20 Dec 2020 15:59:37 +0100 Subject: [PATCH 30/37] reorganized docker build. --- docker-compose.env.example | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docker-compose.env.example diff --git a/docker-compose.env.example b/docker-compose.env.example new file mode 100644 index 000000000..4271bce6e --- /dev/null +++ b/docker-compose.env.example @@ -0,0 +1,34 @@ +# The UID and GID of the user used to run paperless in the container. Set this +# to your UID and GID on the host so that you have write access to the +# consumption directory. +#USERMAP_UID=1000 +#USERMAP_GID=1000 + +# Additional languages to install for text recognition, separated by a +# whitespace. Note that this is +# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the +# default language used when guessing the language from the OCR output. +# The container installs English, German, Italian, Spanish and French by +# default. +# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster +# for available languages. +#PAPERLESS_OCR_LANGUAGES=tur ces + +############################################################################### +# Paperless-specific settings # +############################################################################### + +# All settings defined in the paperless.conf.example can be used here. The +# Docker setup does not use the configuration file. +# A few commonly adjusted settings are provided below. + +# Adjust this key if you plan to make paperless available publicly. It should +# be a very long sequence of random characters. You don't need to remember it. +#PAPERLESS_SECRET_KEY=change-me + +# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. +#PAPERLESS_TIME_ZONE=America/Los_Angeles + +# The default language to use for OCR. Set this to the language most of your +# documents are written in. +#PAPERLESS_OCR_LANGUAGE=eng From c6b9e2b5447a4c4021ec48ac8ea6ac590eb20b7e Mon Sep 17 00:00:00 2001 From: jonaswinkler <jonas.winkler@jpwinkler.de> Date: Sun, 20 Dec 2020 16:00:11 +0100 Subject: [PATCH 31/37] revert last commit --- docker-compose.env.example | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 docker-compose.env.example diff --git a/docker-compose.env.example b/docker-compose.env.example deleted file mode 100644 index 4271bce6e..000000000 --- a/docker-compose.env.example +++ /dev/null @@ -1,34 +0,0 @@ -# The UID and GID of the user used to run paperless in the container. Set this -# to your UID and GID on the host so that you have write access to the -# consumption directory. -#USERMAP_UID=1000 -#USERMAP_GID=1000 - -# Additional languages to install for text recognition, separated by a -# whitespace. Note that this is -# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the -# default language used when guessing the language from the OCR output. -# The container installs English, German, Italian, Spanish and French by -# default. -# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster -# for available languages. -#PAPERLESS_OCR_LANGUAGES=tur ces - -############################################################################### -# Paperless-specific settings # -############################################################################### - -# All settings defined in the paperless.conf.example can be used here. The -# Docker setup does not use the configuration file. -# A few commonly adjusted settings are provided below. - -# Adjust this key if you plan to make paperless available publicly. It should -# be a very long sequence of random characters. You don't need to remember it. -#PAPERLESS_SECRET_KEY=change-me - -# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC. -#PAPERLESS_TIME_ZONE=America/Los_Angeles - -# The default language to use for OCR. Set this to the language most of your -# documents are written in. -#PAPERLESS_OCR_LANGUAGE=eng From 665863e3953a3696f9d87d4585847e5841588f84 Mon Sep 17 00:00:00 2001 From: jonaswinkler <jonas.winkler@jpwinkler.de> Date: Sun, 20 Dec 2020 17:18:23 +0100 Subject: [PATCH 32/37] Display name of current user on the dashboard --- .../dashboard/dashboard.component.html | 2 +- .../dashboard/dashboard.component.ts | 24 ++++++++++++++++++- src/documents/templates/index.html | 2 ++ src/documents/views.py | 2 ++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html index 627e7ff22..541255a68 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.html +++ b/src-ui/src/app/components/dashboard/dashboard.component.html @@ -1,4 +1,4 @@ -<app-page-header title="Dashboard" subTitle="Welcome to paperless-ng!"> +<app-page-header title="Dashboard" [subTitle]="subtitle"> <img src="assets/logo.svg" height="80" class="m-2 d-none d-md-block"> </app-page-header> diff --git a/src-ui/src/app/components/dashboard/dashboard.component.ts b/src-ui/src/app/components/dashboard/dashboard.component.ts index a14ec5e90..db9b5d425 100644 --- a/src-ui/src/app/components/dashboard/dashboard.component.ts +++ b/src-ui/src/app/components/dashboard/dashboard.component.ts @@ -1,4 +1,5 @@ import { Component, OnInit } from '@angular/core'; +import { Meta } from '@angular/platform-browser'; import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'; import { SavedViewService } from 'src/app/services/rest/saved-view.service'; @@ -11,8 +12,29 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service'; export class DashboardComponent implements OnInit { constructor( - private savedViewService: SavedViewService) { } + private savedViewService: SavedViewService, + private meta: Meta + ) { } + get displayName() { + let tagFullName = this.meta.getTag('name=full_name') + let tagUsername = this.meta.getTag('name=username') + if (tagFullName && tagFullName.content) { + return tagFullName.content + } else if (tagUsername && tagUsername.content) { + return tagUsername.content + } else { + return null + } + } + + get subtitle() { + if (this.displayName) { + return `Hello ${this.displayName}, welcome to Paperless-ng!` + } else { + return `Welcome to Paperless-ng!` + } + } savedViews: PaperlessSavedView[] = [] diff --git a/src/documents/templates/index.html b/src/documents/templates/index.html index d086be0fe..47a352cd5 100644 --- a/src/documents/templates/index.html +++ b/src/documents/templates/index.html @@ -8,6 +8,8 @@ <title>Paperless-ng + + diff --git a/src/documents/views.py b/src/documents/views.py index 54d0de3f6..43e06065f 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -57,6 +57,8 @@ class IndexView(TemplateView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['cookie_prefix'] = settings.COOKIE_PREFIX + context['username'] = self.request.user.username + context['full_name'] = self.request.user.get_full_name() return context From e75534c0f222ad256c4525c43691cbb49bb0efc5 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 17:35:05 +0100 Subject: [PATCH 33/37] fixes #165 --- .../management/commands/document_importer.py | 31 +++++++++++++------ .../tests/test_management_exporter.py | 30 +++++++++++++++--- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 70d05d98b..8e9a79219 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -1,18 +1,29 @@ import json import os import shutil +from contextlib import contextmanager from django.conf import settings from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError +from django.db.models.signals import post_save, m2m_changed from filelock import FileLock from documents.models import Document from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME, \ EXPORTER_ARCHIVE_NAME -from ...file_handling import create_source_path_directory, \ - generate_unique_filename +from ...file_handling import create_source_path_directory from ...mixins import Renderable +from ...signals.handlers import update_filename_and_move_files + + +@contextmanager +def disable_signal(sig, receiver, sender): + try: + sig.disconnect(receiver=receiver, sender=sender) + yield + finally: + sig.connect(receiver=receiver, sender=sender) class Command(Renderable, BaseCommand): @@ -47,11 +58,16 @@ class Command(Renderable, BaseCommand): self.manifest = json.load(f) self._check_manifest() + with disable_signal(post_save, + receiver=update_filename_and_move_files, + sender=Document): + with disable_signal(m2m_changed, + receiver=update_filename_and_move_files, + sender=Document.tags.through): + # Fill up the database with whatever is in the manifest + call_command("loaddata", manifest_path) - # Fill up the database with whatever is in the manifest - call_command("loaddata", manifest_path) - - self._import_files_from_manifest() + self._import_files_from_manifest() @staticmethod def _check_manifest_exists(path): @@ -117,9 +133,6 @@ class Command(Renderable, BaseCommand): document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED with FileLock(settings.MEDIA_LOCK): - document.filename = generate_unique_filename( - document, settings.ORIGINALS_DIR) - if os.path.isfile(document.source_path): raise FileExistsError(document.source_path) diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 22d6fc7f6..d6ab7eadd 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -24,11 +24,17 @@ class TestExportImport(DirectoriesMixin, TestCase): file = os.path.join(self.dirs.originals_dir, "0000001.pdf") - Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf") - Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) - Tag.objects.create(name="t") - DocumentType.objects.create(name="dt") - Correspondent.objects.create(name="c") + d1 = Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf") + d2 = Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG) + t1 = Tag.objects.create(name="t") + dt1 = DocumentType.objects.create(name="dt") + c1 = Correspondent.objects.create(name="c") + + d1.tags.add(t1) + d1.correspondents = c1 + d1.document_type = dt1 + d1.save() + d2.save() target = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, target) @@ -59,11 +65,25 @@ class TestExportImport(DirectoriesMixin, TestCase): self.assertEqual(checksum, element['fields']['archive_checksum']) with paperless_environment() as dirs: + self.assertEqual(Document.objects.count(), 2) + Document.objects.all().delete() + Correspondent.objects.all().delete() + DocumentType.objects.all().delete() + Tag.objects.all().delete() + self.assertEqual(Document.objects.count(), 0) + call_command('document_importer', target) + self.assertEqual(Document.objects.count(), 2) messages = check_sanity() # everything is alright after the test self.assertEqual(len(messages), 0, str([str(m) for m in messages])) + @override_settings( + PAPERLESS_FILENAME_FORMAT="{title}" + ) + def test_exporter_with_filename_format(self): + self.test_exporter() + def test_export_missing_files(self): target = tempfile.mkdtemp() From b653e44f65bfbe9a80e5e1a4e21cb364bfc6abb7 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 18:15:28 +0100 Subject: [PATCH 34/37] changed field order, updated ng-select for tag color selection --- .../common/input/select/select.component.html | 2 +- .../correspondent-edit-dialog.component.html | 5 ++--- .../document-type-edit-dialog.component.html | 4 ++-- .../tag-edit-dialog.component.html | 16 +++++++++++++--- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html index 655adbe74..d33dae425 100644 --- a/src-ui/src/app/components/common/input/select/select.component.html +++ b/src-ui/src/app/components/common/input/select/select.component.html @@ -1,7 +1,7 @@
- - - - + +
diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html index 013c5a947..3338c40c3 100644 --- a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html @@ -8,9 +8,9 @@
- - + +
diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html index 8048b0c80..2d9f93252 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html @@ -7,11 +7,21 @@
- + + +
+ + + + {{item.name}} + + +
+ - - + +
From 5892941d8a7c9b3dd227666334ebbb5825d45a70 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 18:17:21 +0100 Subject: [PATCH 35/37] updated tag coloring for easier readability --- src-ui/src/app/data/paperless-tag.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/data/paperless-tag.ts b/src-ui/src/app/data/paperless-tag.ts index 551c6e03a..979a200a8 100644 --- a/src-ui/src/app/data/paperless-tag.ts +++ b/src-ui/src/app/data/paperless-tag.ts @@ -6,14 +6,14 @@ export const TAG_COLOURS = [ {id: 1, value: "#a6cee3", name: "Light Blue", textColor: "#000000"}, {id: 2, value: "#1f78b4", name: "Blue", textColor: "#ffffff"}, {id: 3, value: "#b2df8a", name: "Light Green", textColor: "#000000"}, - {id: 4, value: "#33a02c", name: "Green", textColor: "#000000"}, + {id: 4, value: "#33a02c", name: "Green", textColor: "#ffffff"}, {id: 5, value: "#fb9a99", name: "Light Red", textColor: "#000000"}, {id: 6, value: "#e31a1c", name: "Red ", textColor: "#ffffff"}, {id: 7, value: "#fdbf6f", name: "Light Orange", textColor: "#000000"}, {id: 8, value: "#ff7f00", name: "Orange", textColor: "#000000"}, {id: 9, value: "#cab2d6", name: "Light Violet", textColor: "#000000"}, {id: 10, value: "#6a3d9a", name: "Violet", textColor: "#ffffff"}, - {id: 11, value: "#b15928", name: "Brown", textColor: "#000000"}, + {id: 11, value: "#b15928", name: "Brown", textColor: "#ffffff"}, {id: 12, value: "#000000", name: "Black", textColor: "#ffffff"}, {id: 13, value: "#cccccc", name: "Light Grey", textColor: "#000000"} ] From c527b274b65fb4f32617cabb0848c18649a9451d Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 18:23:06 +0100 Subject: [PATCH 36/37] disallow clearing tag color --- .../tag-list/tag-edit-dialog/tag-edit-dialog.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html index 2d9f93252..138d3e7cd 100644 --- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html +++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html @@ -11,7 +11,7 @@
- + {{item.name}} From ae3e2a70632642ce63abdd534aead833a45d7868 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 21 Dec 2020 21:28:21 +0100 Subject: [PATCH 37/37] updated documentation --- docs/advanced_usage.rst | 79 ----------------------------------------- docs/configuration.rst | 5 --- 2 files changed, 84 deletions(-) diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index 48a86384c..8f6b91b4c 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -5,85 +5,6 @@ Advanced topics Paperless offers a couple features that automate certain tasks and make your life easier. -Guesswork -######### - - -Any document you put into the consumption directory will be consumed, but if -you name the file right, it'll automatically set some values in the database -for you. This is is the logic the consumer follows: - -1. Try to find the correspondent, title, and tags in the file name following - the pattern: ``Date - Correspondent - Title - tag,tag,tag.pdf``. Note that - the format of the date is **rigidly defined** as ``YYYYMMDDHHMMSSZ`` or - ``YYYYMMDDZ``. The ``Z`` refers "Zulu time" AKA "UTC". - The tags are optional, so the format ``Date - Correspondent - Title.pdf`` - works as well. -2. If that doesn't work, we skip the date and try this pattern: - ``Correspondent - Title - tag,tag,tag.pdf``. -3. If that doesn't work, we try to find the correspondent and title in the file - name following the pattern: ``Correspondent - Title.pdf``. -4. If that doesn't work, just assume that the name of the file is the title. - -So given the above, the following examples would work as you'd expect: - -* ``20150314000700Z - Some Company Name - Invoice 2016-01-01 - money,invoices.pdf`` -* ``20150314Z - Some Company Name - Invoice 2016-01-01 - money,invoices.pdf`` -* ``Some Company Name - Invoice 2016-01-01 - money,invoices.pdf`` -* ``Another Company - Letter of Reference.jpg`` -* ``Dad's Recipe for Pancakes.png`` - -These however wouldn't work: - -* ``2015-03-14 00:07:00 UTC - Some Company Name, Invoice 2016-01-01, money, invoices.pdf`` -* ``2015-03-14 - Some Company Name, Invoice 2016-01-01, money, invoices.pdf`` -* ``Some Company Name, Invoice 2016-01-01, money, invoices.pdf`` -* ``Another Company- Letter of Reference.jpg`` - -Do I have to be so strict about naming? -======================================= - -Rather than using the strict document naming rules, one can also set the option -``PAPERLESS_FILENAME_DATE_ORDER`` in ``paperless.conf`` to any date order -that is accepted by dateparser_. Doing so will cause ``paperless`` to default -to any date format that is found in the title, instead of a date pulled from -the document's text, without requiring the strict formatting of the document -filename as described above. - -.. _dateparser: https://github.com/scrapinghub/dateparser/blob/v0.7.0/docs/usage.rst#settings - -.. _advanced-transforming_filenames: - -Transforming filenames for parsing -================================== - -Some devices can't produce filenames that can be parsed by the default -parser. By configuring the option ``PAPERLESS_FILENAME_PARSE_TRANSFORMS`` in -``paperless.conf`` one can add transformations that are applied to the filename -before it's parsed. - -The option contains a list of dictionaries of regular expressions (key: -``pattern``) and replacements (key: ``repl``) in JSON format, which are -applied in order by passing them to ``re.subn``. Transformation stops -after the first match, so at most one transformation is applied. The general -syntax is - -.. code:: python - - [{"pattern":"pattern1", "repl":"repl1"}, {"pattern":"pattern2", "repl":"repl2"}, ..., {"pattern":"patternN", "repl":"replN"}] - -The example below is for a Brother ADS-2400N, a scanner that allows -different names to different hardware buttons (useful for handling -multiple entities in one instance), but insists on adding ``_`` -to the filename. - -.. code:: python - - # Brother profile configuration, support "Name_Date_Count" (the default - # setting) and "Name_Count" (use "Name" as tag and "Count" as title). - PAPERLESS_FILENAME_PARSE_TRANSFORMS=[{"pattern":"^([a-z]+)_(\\d{8})_(\\d{6})_([0-9]+)\\.", "repl":"\\2\\3Z - \\4 - \\1."}, {"pattern":"^([a-z]+)_([0-9]+)\\.", "repl":" - \\2 - \\1."}] - - .. _advanced-matching: Matching tags, correspondents and document types diff --git a/docs/configuration.rst b/docs/configuration.rst index d3f47215b..efc1a9db1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -400,11 +400,6 @@ PAPERLESS_FILENAME_DATE_ORDER= Defaults to none, which disables this feature. -PAPERLESS_FILENAME_PARSE_TRANSFORMS - Transforms filenames before they are processed by paperless. See - :ref:`advanced-transforming_filenames` for details. - - Defaults to none, which disables this feature. Binaries ########