paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-02 13:45:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

lots of changes for the new unified search

Browse Source
This commit is contained in:
jonaswinkler 2021-03-17 22:25:22 +01:00
parent 630cd814e2
commit b6ff88645b
29 changed files with 302 additions and 543 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src-ui/src/app/app-routing.module.ts
View File

@ -10,7 +10,6 @@ import { LogsComponent } from './components/manage/logs/logs.component';
import { SettingsComponent } from './components/manage/settings/settings.component';
import { TagListComponent } from './components/manage/tag-list/tag-list.component';
import { NotFoundComponent } from './components/not-found/not-found.component';
import { SearchComponent } from './components/search/search.component';
const routes: Routes = [
{path: '', redirectTo: 'dashboard', pathMatch: 'full'},
@ -18,7 +17,6 @@ const routes: Routes = [
{path: 'dashboard', component: DashboardComponent },
{path: 'documents', component: DocumentListComponent },
{path: 'view/:id', component: DocumentListComponent },
{path: 'search', component: SearchComponent },
{path: 'documents/:id', component: DocumentDetailComponent },
{path: 'tags', component: TagListComponent },

4
src-ui/src/app/app.module.ts
View File

@ -21,8 +21,6 @@ import { CorrespondentEditDialogComponent } from './components/manage/correspond
import { TagEditDialogComponent } from './components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component';
import { DocumentTypeEditDialogComponent } from './components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
import { TagComponent } from './components/common/tag/tag.component';
import { SearchComponent } from './components/search/search.component';
import { ResultHighlightComponent } from './components/search/result-highlight/result-highlight.component';
import { PageHeaderComponent } from './components/common/page-header/page-header.component';
import { AppFrameComponent } from './components/app-frame/app-frame.component';
import { ToastsComponent } from './components/common/toasts/toasts.component';
@ -104,8 +102,6 @@ registerLocaleData(localeEs)
TagEditDialogComponent,
DocumentTypeEditDialogComponent,
TagComponent,
SearchComponent,
ResultHighlightComponent,
PageHeaderComponent,
AppFrameComponent,
ToastsComponent,

5
src-ui/src/app/components/app-frame/app-frame.component.ts
View File

@ -10,6 +10,8 @@ import { SearchService } from 'src/app/services/rest/search.service';
import { environment } from 'src/environments/environment';
import { DocumentDetailComponent } from '../document-detail/document-detail.component';
import { Meta } from '@angular/platform-browser';
import { DocumentListViewService } from 'src/app/services/document-list-view.service';
import { FILTER_FULLTEXT_QUERY } from 'src/app/data/filter-rule-type';
@Component({
selector: 'app-app-frame',
@ -24,6 +26,7 @@ export class AppFrameComponent implements OnInit {
private openDocumentsService: OpenDocumentsService,
private searchService: SearchService,
public savedViewService: SavedViewService,
private list: DocumentListViewService,
private meta: Meta
) {
@ -74,7 +77,7 @@ export class AppFrameComponent implements OnInit {
search() {
this.closeMenu()
this.router.navigate(['search'], {queryParams: {query: this.searchField.value}})
this.list.quickFilter([{rule_type: FILTER_FULLTEXT_QUERY, value: this.searchField.value}])
}
closeDocument(d: PaperlessDocument) {

3
src-ui/src/app/components/document-detail/document-detail.component.ts
View File

@ -20,6 +20,7 @@ import { ToastService } from 'src/app/services/toast.service';
import { TextComponent } from '../common/input/text/text.component';
import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
import { PaperlessDocumentSuggestions } from 'src/app/data/paperless-document-suggestions';
import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type';
@Component({
selector: 'app-document-detail',
@ -219,7 +220,7 @@ export class DocumentDetailComponent implements OnInit {
}
moreLike() {
this.router.navigate(["search"], {queryParams: {more_like:this.document.id}})
this.documentListViewService.quickFilter([{rule_type: FILTER_FULLTEXT_MORELIKE, value: this.documentId.toString()}])
}
hasNext() {

10
src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
View File

@ -25,14 +25,14 @@
</h5>
</div>
<p class="card-text">
<app-result-highlight *ngIf="getDetailsAsHighlight()" class="result-content" [highlights]="getDetailsAsHighlight()"></app-result-highlight>
<span *ngIf="getDetailsAsString()" class="result-content">{{getDetailsAsString()}}</span>
<span *ngIf="document.__search_hit__" [innerHtml]="document.__search_hit__.highlights"></span>
<span *ngIf="!document.__search_hit__" class="result-content">{{contentTrimmed}}</span>
</p>
<div class="d-flex flex-column flex-md-row align-items-md-center">
<div class="btn-group">
<a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary" *ngIf="moreLikeThis">
<a class="btn btn-sm btn-outline-secondary" (click)="clickMoreLike.emit()">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-three-dots" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
</svg>&nbsp;<span class="d-block d-md-inline" i18n>More like this</span>
@ -62,9 +62,9 @@
</div>
<div class="list-group list-group-horizontal border-0 card-info ml-md-auto mt-2 mt-md-0">
<div *ngIf="searchScore" class="list-group-item bg-light text-dark p-1 mr-5 border-0 d-flex search-score">
<div *ngIf="document.__search_hit__" class="list-group-item bg-light text-dark p-1 mr-5 border-0 d-flex search-score">
<small class="text-muted" i18n>Score:</small>
<ngb-progressbar [type]="searchScoreClass" [value]="searchScore" class="search-score-bar mx-2 mt-1" [max]="1"></ngb-progressbar>
<ngb-progressbar [type]="searchScoreClass" [value]="document.__search_hit__.score" class="search-score-bar mx-2 mt-1" [max]="1"></ngb-progressbar>
</div>
<button *ngIf="document.document_type" type="button" class="list-group-item btn btn-sm bg-light text-dark p-1 border-0 mr-2" title="Filter by document type"
(click)="clickDocumentType.emit(document.document_type);$event.stopPropagation()">

5
src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
View File

@ -60,3 +60,8 @@
padding-top: 0.35rem !important;
}
}
span ::ng-deep .match {
color: black;
background-color: rgb(255, 211, 66);
}

28
src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
View File

@ -4,6 +4,8 @@ import { PaperlessDocument } from 'src/app/data/paperless-document';
import { DocumentService } from 'src/app/services/rest/document.service';
import { SettingsService, SETTINGS_KEYS } from 'src/app/services/settings.service';
import { NgbPopover } from '@ng-bootstrap/ng-bootstrap';
import { DocumentListViewService } from 'src/app/services/document-list-view.service';
import { FILTER_FULLTEXT_MORELIKE } from 'src/app/data/filter-rule-type';
@Component({
selector: 'app-document-card-large',
@ -24,15 +26,9 @@ export class DocumentCardLargeComponent implements OnInit {
return this.toggleSelected.observers.length > 0
}
@Input()
moreLikeThis: boolean = false
@Input()
document: PaperlessDocument
@Input()
details: any
@Output()
clickTag = new EventEmitter<number>()
@ -42,6 +38,9 @@ export class DocumentCardLargeComponent implements OnInit {
@Output()
clickDocumentType = new EventEmitter<number>()
@Output()
clickMoreLike= new EventEmitter()
@Input()
searchScore: number
@ -67,19 +66,6 @@ export class DocumentCardLargeComponent implements OnInit {
return this.settingsService.get(SETTINGS_KEYS.DARK_MODE_THUMB_INVERTED)
}
getDetailsAsString() {
if (typeof this.details === 'string') {
return this.details.substring(0, 500)
}
}
getDetailsAsHighlight() {
//TODO: this is not an exact typecheck, can we do better
if (this.details instanceof Array) {
return this.details
}
}
getThumbUrl() {
return this.documentService.getThumbUrl(this.document.id)
}
@ -116,4 +102,8 @@ export class DocumentCardLargeComponent implements OnInit {
mouseLeaveCard() {
this.popover.close()
}
get contentTrimmed() {
return this.document.content.substr(0, 500)
}
}

2
src-ui/src/app/components/document-list/document-list.component.html
View File

@ -90,7 +90,7 @@
</div>
<div *ngIf="displayMode == 'largeCards'">
<app-document-card-large [selected]="list.isSelected(d)" (toggleSelected)="toggleSelected(d, $event)" *ngFor="let d of list.documents; trackBy: trackByDocumentId" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)" (clickDocumentType)="clickDocumentType($event)">
<app-document-card-large [selected]="list.isSelected(d)" (toggleSelected)="toggleSelected(d, $event)" *ngFor="let d of list.documents; trackBy: trackByDocumentId" [document]="d" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)" (clickDocumentType)="clickDocumentType($event)" (clickMoreLike)="clickMoreLike(d.id)">
</app-document-card-large>
</div>

7
src-ui/src/app/components/document-list/document-list.component.ts
View File

@ -207,6 +207,13 @@ export class DocumentListComponent implements OnInit, OnDestroy {
})
}
clickMoreLike(documentID: number) {
this.list.selectNone()
setTimeout(() => {
//this.filterEditor.moreLikeThis(doc)
})
}
trackByDocumentId(index, item: PaperlessDocument) {
return item.id
}

4
src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html
View File

@ -1,7 +1,6 @@
<div class="row">
<div class="col mb-2 mb-xl-0">
<div class="form-inline d-flex align-items-center">
<label class="text-muted mr-2 mb-0" i18n>Filter by:</label>
<div class="input-group input-group-sm flex-fill w-auto">
<div class="input-group-prepend" ngbDropdown>
<button class="btn btn-outline-primary" ngbDropdownToggle>{{textFilterTargetName}}</button>
@ -9,7 +8,8 @@
<button *ngFor="let t of textFilterTargets" ngbDropdownItem [class.active]="textFilterTarget == t.id" (click)="changeTextFilterTarget(t.id)">{{t.name}}</button>
</div>
</div>
<input class="form-control form-control-sm" type="text" [(ngModel)]="textFilter">
<input class="form-control form-control-sm" type="text" [(ngModel)]="textFilter" *ngIf="textFilterTarget != 'fulltext-morelike'">
<span class="form-control form-control-sm text-truncate" *ngIf="textFilterTarget == 'fulltext-morelike'">{{_moreLikeDoc?.title}}</span>
</div>
</div>
</div>

47
src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts
View File

@ -8,13 +8,17 @@ import { DocumentTypeService } from 'src/app/services/rest/document-type.service
import { TagService } from 'src/app/services/rest/tag.service';
import { CorrespondentService } from 'src/app/services/rest/correspondent.service';
import { FilterRule } from 'src/app/data/filter-rule';
import { FILTER_ADDED_AFTER, FILTER_ADDED_BEFORE, FILTER_ASN, FILTER_CORRESPONDENT, FILTER_CREATED_AFTER, FILTER_CREATED_BEFORE, FILTER_DOCUMENT_TYPE, FILTER_HAS_ANY_TAG, FILTER_HAS_TAG, FILTER_TITLE, FILTER_TITLE_CONTENT } from 'src/app/data/filter-rule-type';
import { FILTER_ADDED_AFTER, FILTER_ADDED_BEFORE, FILTER_ASN, FILTER_CORRESPONDENT, FILTER_CREATED_AFTER, FILTER_CREATED_BEFORE, FILTER_DOCUMENT_TYPE, FILTER_FULLTEXT_MORELIKE, FILTER_FULLTEXT_QUERY, FILTER_HAS_ANY_TAG, FILTER_HAS_TAG, FILTER_TITLE, FILTER_TITLE_CONTENT } from 'src/app/data/filter-rule-type';
import { FilterableDropdownSelectionModel } from '../../common/filterable-dropdown/filterable-dropdown.component';
import { ToggleableItemState } from '../../common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component';
import { DocumentService } from 'src/app/services/rest/document.service';
import { PaperlessDocument } from 'src/app/data/paperless-document';
const TEXT_FILTER_TARGET_TITLE = "title"
const TEXT_FILTER_TARGET_TITLE_CONTENT = "title-content"
const TEXT_FILTER_TARGET_ASN = "asn"
const TEXT_FILTER_TARGET_FULLTEXT_QUERY = "fulltext-query"
const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = "fulltext-morelike"
@Component({
selector: 'app-filter-editor',
@ -64,7 +68,8 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
constructor(
private documentTypeService: DocumentTypeService,
private tagService: TagService,
private correspondentService: CorrespondentService
private correspondentService: CorrespondentService,
private documentService: DocumentService
) { }
tags: PaperlessTag[] = []
@ -72,12 +77,21 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
documentTypes: PaperlessDocumentType[] = []
_textFilter = ""
_moreLikeId: number
_moreLikeDoc: PaperlessDocument
textFilterTargets = [
{id: TEXT_FILTER_TARGET_TITLE, name: $localize`Title`},
{id: TEXT_FILTER_TARGET_TITLE_CONTENT, name: $localize`Title & content`},
{id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN`}
]
get textFilterTargets() {
let targets = [
{id: TEXT_FILTER_TARGET_TITLE, name: $localize`Title`},
{id: TEXT_FILTER_TARGET_TITLE_CONTENT, name: $localize`Title & content`},
{id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN`},
{id: TEXT_FILTER_TARGET_FULLTEXT_QUERY, name: $localize`Fulltext search`}
]
if (this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_MORELIKE) {
targets.push({id: TEXT_FILTER_TARGET_FULLTEXT_MORELIKE, name: $localize`More like`})
}
return targets
}
textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT
@ -101,6 +115,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
this.tagSelectionModel.clear(false)
this.correspondentSelectionModel.clear(false)
this._textFilter = null
this._moreLikeId = null
this.dateAddedBefore = null
this.dateAddedAfter = null
this.dateCreatedBefore = null
@ -120,6 +135,17 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
this._textFilter = rule.value
this.textFilterTarget = TEXT_FILTER_TARGET_ASN
break
case FILTER_FULLTEXT_QUERY:
this._textFilter = rule.value
this.textFilterTarget = TEXT_FILTER_TARGET_FULLTEXT_QUERY
break
case FILTER_FULLTEXT_MORELIKE:
this._moreLikeId = +rule.value
this.textFilterTarget = TEXT_FILTER_TARGET_FULLTEXT_MORELIKE
this.documentService.get(this._moreLikeId).subscribe(result => {
this._moreLikeDoc = result
})
break
case FILTER_CREATED_AFTER:
this.dateCreatedAfter = rule.value
break
@ -159,6 +185,12 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
if (this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_ASN) {
filterRules.push({rule_type: FILTER_ASN, value: this._textFilter})
}
if (this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_QUERY) {
filterRules.push({rule_type: FILTER_FULLTEXT_QUERY, value: this._textFilter})
}
if (this._moreLikeId && this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_MORELIKE) {
filterRules.push({rule_type: FILTER_FULLTEXT_MORELIKE, value: this._moreLikeId?.toString()})
}
if (this.tagSelectionModel.isNoneSelected()) {
filterRules.push({rule_type: FILTER_HAS_ANY_TAG, value: "false"})
} else {
@ -232,6 +264,7 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
}
resetSelected() {
this.textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT
this.reset.next()
}

3
src-ui/src/app/components/search/result-highlight/result-highlight.component.html
View File

@ -1,3 +0,0 @@
... <span *ngFor="let fragment of highlights">
<span *ngFor="let token of fragment" [class.match]="token.highlight">{{token.text}}</span> ...
</span>

4
src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
View File

@ -1,4 +0,0 @@
.match {
color: black;
background-color: rgb(255, 211, 66);
}

25
src-ui/src/app/components/search/result-highlight/result-highlight.component.spec.ts
View File

@ -1,25 +0,0 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { ResultHighlightComponent } from './result-highlight.component';
describe('ResultHighlightComponent', () => {
let component: ResultHighlightComponent;
let fixture: ComponentFixture<ResultHighlightComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ ResultHighlightComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(ResultHighlightComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

19
src-ui/src/app/components/search/result-highlight/result-highlight.component.ts
View File

@ -1,19 +0,0 @@
import { Component, Input, OnInit } from '@angular/core';
import { SearchHitHighlight } from 'src/app/data/search-result';
@Component({
selector: 'app-result-highlight',
templateUrl: './result-highlight.component.html',
styleUrls: ['./result-highlight.component.scss']
})
export class ResultHighlightComponent implements OnInit {
constructor() { }
@Input()
highlights: SearchHitHighlight[][]
ngOnInit(): void {
}
}

26
src-ui/src/app/components/search/search.component.html
View File

@ -1,26 +0,0 @@
<app-page-header i18n-title title="Search results">
</app-page-header>
<div *ngIf="errorMessage" class="alert alert-danger" i18n>Invalid search query: {{errorMessage}}</div>
<p *ngIf="more_like" i18n>Showing documents similar to <a routerLink="/documents/{{more_like}}">{{more_like_doc?.original_file_name}}</a></p>
<p *ngIf="query">
<ng-container i18n>Search query: <i>{{query}}</i></ng-container>
<ng-container *ngIf="correctedQuery">
- <ng-container i18n>Did you mean "<a [routerLink]="" (click)="searchCorrectedQuery()">{{correctedQuery}}</a>"?</ng-container>
</ng-container>
</p>
<div *ngIf="!errorMessage" [class.result-content-searching]="searching" infiniteScroll (scrolled)="onScroll()">
<p i18n>{resultCount, plural, =0 {No results} =1 {One result} other {{{resultCount}} results}}</p>
<ng-container *ngFor="let result of results">
<app-document-card-large *ngIf="result.document"
[document]="result.document"
[details]="result.highlights"
[searchScore]="result.score / maxScore"
[moreLikeThis]="true">
</app-document-card-large>
</ng-container>
</div>

15
src-ui/src/app/components/search/search.component.scss
View File

@ -1,15 +0,0 @@
.result-content {
color: darkgray;
}
.doc-img {
object-fit: cover;
object-position: top;
height: 100%;
position: absolute;
}
.result-content-searching {
opacity: 0.3;
}

25
src-ui/src/app/components/search/search.component.spec.ts
View File

@ -1,25 +0,0 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { SearchComponent } from './search.component';
describe('SearchComponent', () => {
let component: SearchComponent;
let fixture: ComponentFixture<SearchComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ SearchComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(SearchComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

95
src-ui/src/app/components/search/search.component.ts
View File

@ -1,95 +0,0 @@
import { Component, OnInit } from '@angular/core';
import { ActivatedRoute, Router } from '@angular/router';
import { PaperlessDocument } from 'src/app/data/paperless-document';
import { PaperlessDocumentType } from 'src/app/data/paperless-document-type';
import { SearchHit } from 'src/app/data/search-result';
import { DocumentService } from 'src/app/services/rest/document.service';
import { SearchService } from 'src/app/services/rest/search.service';
@Component({
selector: 'app-search',
templateUrl: './search.component.html',
styleUrls: ['./search.component.scss']
})
export class SearchComponent implements OnInit {
results: SearchHit[] = []
query: string = ""
more_like: number
more_like_doc: PaperlessDocument
searching = false
currentPage = 1
pageCount = 1
resultCount
correctedQuery: string = null
errorMessage: string
get maxScore() {
return this.results?.length > 0 ? this.results[0].score : 100
}
constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router, private documentService: DocumentService) { }
ngOnInit(): void {
this.route.queryParamMap.subscribe(paramMap => {
window.scrollTo(0, 0)
this.query = paramMap.get('query')
this.more_like = paramMap.has('more_like') ? +paramMap.get('more_like') : null
if (this.more_like) {
this.documentService.get(this.more_like).subscribe(r => {
this.more_like_doc = r
})
} else {
this.more_like_doc = null
}
this.searching = true
this.currentPage = 1
this.loadPage()
})
}
searchCorrectedQuery() {
this.router.navigate(["search"], {queryParams: {query: this.correctedQuery, more_like: this.more_like}})
}
loadPage(append: boolean = false) {
this.errorMessage = null
this.correctedQuery = null
this.searchService.search(this.query, this.currentPage, this.more_like).subscribe(result => {
if (append) {
this.results.push(...result.results)
} else {
this.results = result.results
}
this.pageCount = result.page_count
this.searching = false
this.resultCount = result.count
this.correctedQuery = result.corrected_query
}, error => {
this.searching = false
this.resultCount = 1
this.pageCount = 1
this.results = []
this.errorMessage = error.error
})
}
onScroll() {
if (this.currentPage < this.pageCount) {
this.currentPage += 1
this.loadPage(true)
}
}
}

9
src-ui/src/app/data/filter-rule-type.ts
View File

@ -22,6 +22,9 @@ export const FILTER_ASN_ISNULL = 18
export const FILTER_TITLE_CONTENT = 19
export const FILTER_FULLTEXT_QUERY = 20
export const FILTER_FULLTEXT_MORELIKE = 21
export const FILTER_RULE_TYPES: FilterRuleType[] = [
{id: FILTER_TITLE, filtervar: "title__icontains", datatype: "string", multi: false, default: ""},
@ -51,7 +54,11 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [
{id: FILTER_MODIFIED_AFTER, filtervar: "modified__date__gt", datatype: "date", multi: false},
{id: FILTER_ASN_ISNULL, filtervar: "archive_serial_number__isnull", datatype: "boolean", multi: false},
{id: FILTER_TITLE_CONTENT, filtervar: "title_content", datatype: "string", multi: false}
{id: FILTER_TITLE_CONTENT, filtervar: "title_content", datatype: "string", multi: false},
{id: FILTER_FULLTEXT_QUERY, filtervar: "query", datatype: "string", multi: false},
{id: FILTER_FULLTEXT_MORELIKE, filtervar: "more_like_id", datatype: "number", multi: false},
]
export interface FilterRuleType {

11
src-ui/src/app/data/paperless-document.ts
View File

@ -4,6 +4,15 @@ import { PaperlessTag } from './paperless-tag'
import { PaperlessDocumentType } from './paperless-document-type'
import { Observable } from 'rxjs'
export interface SearchHit {
score?: number
rank?: number
highlights?: string
}
export interface PaperlessDocument extends ObjectWithId {
correspondent$?: Observable<PaperlessCorrespondent>
@ -40,4 +49,6 @@ export interface PaperlessDocument extends ObjectWithId {
archive_serial_number?: number
__search_hit__?: SearchHit
}

29
src-ui/src/app/data/search-result.ts
View File

@ -1,29 +0,0 @@
import { PaperlessDocument } from './paperless-document'
export class SearchHitHighlight {
text?: string
term?: number
}
export interface SearchHit {
id?: number
title?: string
score?: number
rank?: number
highlights?: SearchHitHighlight[][]
document?: PaperlessDocument
}
export interface SearchResult {
count?: number
page?: number
page_count?: number
corrected_query?: string
results?: SearchHit[]
}

12
src-ui/src/app/services/document-list-view.service.ts
View File

@ -1,7 +1,9 @@
import { Route } from '@angular/compiler/src/core';
import { Injectable } from '@angular/core';
import { Router } from '@angular/router';
import { ActivatedRoute, Router } from '@angular/router';
import { Observable } from 'rxjs';
import { cloneFilterRules, FilterRule } from '../data/filter-rule';
import { FILTER_FULLTEXT_MORELIKE, FILTER_FULLTEXT_QUERY } from '../data/filter-rule-type';
import { PaperlessDocument } from '../data/paperless-document';
import { PaperlessSavedView } from '../data/paperless-saved-view';
import { DOCUMENT_LIST_SERVICE } from '../data/storage-keys';
@ -207,7 +209,11 @@ export class DocumentListViewService {
this.activeListViewState.currentPage = 1
this.reduceSelectionToFilter()
this.saveDocumentListView()
this.router.navigate(["documents"])
if (this.router.url == "/documents") {
this.reload()
} else {
this.router.navigate(["documents"])
}
}
getLastPage(): number {
@ -317,7 +323,7 @@ export class DocumentListViewService {
return this.documents.map(d => d.id).indexOf(documentID)
}
constructor(private documentService: DocumentService, private settings: SettingsService, private router: Router) {
constructor(private documentService: DocumentService, private settings: SettingsService, private router: Router, private route: ActivatedRoute) {
let documentListViewConfigJson = sessionStorage.getItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG)
if (documentListViewConfigJson) {
try {

27
src-ui/src/app/services/rest/search.service.ts
View File

@ -2,8 +2,6 @@ import { HttpClient, HttpParams } from '@angular/common/http';
import { Injectable } from '@angular/core';
import { Observable } from 'rxjs';
import { map } from 'rxjs/operators';
import { PaperlessDocument } from 'src/app/data/paperless-document';
import { SearchResult } from 'src/app/data/search-result';
import { environment } from 'src/environments/environment';
import { DocumentService } from './document.service';
@ -13,30 +11,7 @@ import { DocumentService } from './document.service';
})
export class SearchService {
constructor(private http: HttpClient, private documentService: DocumentService) { }
search(query: string, page?: number, more_like?: number): Observable<SearchResult> {
let httpParams = new HttpParams()
if (query) {
httpParams = httpParams.set('query', query)
}
if (page) {
httpParams = httpParams.set('page', page.toString())
}
if (more_like) {
httpParams = httpParams.set('more_like', more_like.toString())
}
return this.http.get<SearchResult>(`${environment.apiBaseUrl}search/`, {params: httpParams}).pipe(
map(result => {
result.results.forEach(hit => {
if (hit.document) {
this.documentService.addObservablesToDocument(hit.document)
}
})
return result
})
)
}
constructor(private http: HttpClient) { }
autocomplete(term: string): Observable<string[]> {
return this.http.get<string[]>(`${environment.apiBaseUrl}search/autocomplete/`, {params: new HttpParams().set('term', term)})

289
src/documents/index.py
View File

@ -5,12 +5,12 @@ from contextlib import contextmanager
import math
from django.conf import settings
from whoosh import highlight, classify, query
from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME
from whoosh.highlight import Formatter, get_text
from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME, BOOLEAN
from whoosh.highlight import Formatter, get_text, HtmlFormatter
from whoosh.index import create_in, exists_in, open_dir
from whoosh.qparser import MultifieldParser
from whoosh.qparser.dateparse import DateParserPlugin
from whoosh.searching import ResultsPage
from whoosh.searching import ResultsPage, Searcher
from whoosh.writing import AsyncWriter
from documents.models import Document
@ -18,63 +18,53 @@ from documents.models import Document
logger = logging.getLogger("paperless.index")
class JsonFormatter(Formatter):
def __init__(self):
self.seen = {}
def format_token(self, text, token, replace=False):
ttext = self._text(get_text(text, token, replace))
return {'text': ttext, 'highlight': 'true'}
def format_fragment(self, fragment, replace=False):
output = []
index = fragment.startchar
text = fragment.text
amend_token = None
for t in fragment.matches:
if t.startchar is None:
continue
if t.startchar < index:
continue
if t.startchar > index:
text_inbetween = text[index:t.startchar]
if amend_token and t.startchar - index < 10:
amend_token['text'] += text_inbetween
else:
output.append({'text': text_inbetween,
'highlight': False})
amend_token = None
token = self.format_token(text, t, replace)
if amend_token:
amend_token['text'] += token['text']
else:
output.append(token)
amend_token = token
index = t.endchar
if index < fragment.endchar:
output.append({'text': text[index:fragment.endchar],
'highlight': False})
return output
def format(self, fragments, replace=False):
output = []
for fragment in fragments:
output.append(self.format_fragment(fragment, replace=replace))
return output
def get_schema():
return Schema(
id=NUMERIC(stored=True, unique=True, numtype=int),
title=TEXT(stored=True),
id=NUMERIC(
stored=True,
unique=True
),
title=TEXT(
sortable=True
),
content=TEXT(),
correspondent=TEXT(stored=True),
correspondent_id=NUMERIC(stored=True, numtype=int),
tag=KEYWORD(stored=True, commas=True, scorable=True, lowercase=True),
type=TEXT(stored=True),
created=DATETIME(stored=True, sortable=True),
modified=DATETIME(stored=True, sortable=True),
added=DATETIME(stored=True, sortable=True),
archive_serial_number=NUMERIC(
sortable=True
),
correspondent=TEXT(
sortable=True
),
correspondent_id=NUMERIC(),
has_correspondent=BOOLEAN(),
tag=KEYWORD(
commas=True,
scorable=True,
lowercase=True
),
tag_id=KEYWORD(
commas=True,
scorable=True
),
has_tag=BOOLEAN(),
type=TEXT(
sortable=True
),
type_id=NUMERIC(),
has_type=BOOLEAN(),
created=DATETIME(
sortable=True
),
modified=DATETIME(
sortable=True
),
added=DATETIME(
sortable=True
),
)
@ -106,18 +96,38 @@ def open_index_writer(ix=None, optimize=False):
writer.commit(optimize=optimize)
@contextmanager
def open_index_searcher(ix=None):
if ix:
searcher = ix.searcher()
else:
searcher = open_index().searcher()
try:
yield searcher
finally:
searcher.close()
def update_document(writer, doc):
tags = ",".join([t.name for t in doc.tags.all()])
tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
writer.update_document(
id=doc.pk,
title=doc.title,
content=doc.content,
correspondent=doc.correspondent.name if doc.correspondent else None,
correspondent_id=doc.correspondent.id if doc.correspondent else None,
has_correspondent=doc.correspondent is not None,
tag=tags if tags else None,
tag_id=tags_ids if tags_ids else None,
has_tag=len(tags) > 0,
type=doc.document_type.name if doc.document_type else None,
type_id=doc.document_type.id if doc.document_type else None,
has_type=doc.document_type is not None,
created=doc.created,
added=doc.added,
archive_serial_number=doc.archive_serial_number,
modified=doc.modified,
)
@ -140,78 +150,11 @@ def remove_document_from_index(document):
remove_document(writer, document)
@contextmanager
def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content):
searcher = ix.searcher()
try:
if querystring:
qp = MultifieldParser(
["content", "title", "correspondent", "tag", "type"],
ix.schema)
qp.add_plugin(DateParserPlugin())
str_q = qp.parse(querystring)
corrected = searcher.correct_query(str_q, querystring)
else:
str_q = None
corrected = None
if more_like_doc_id:
docnum = searcher.document_number(id=more_like_doc_id)
kts = searcher.key_terms_from_text(
'content', more_like_doc_content, numterms=20,
model=classify.Bo1Model, normalize=False)
more_like_q = query.Or(
[query.Term('content', word, boost=weight)
for word, weight in kts])
result_page = searcher.search_page(
more_like_q, page, filter=str_q, mask={docnum})
elif str_q:
result_page = searcher.search_page(str_q, page)
else:
raise ValueError(
"Either querystring or more_like_doc_id is required."
)
result_page.results.fragmenter = highlight.ContextFragmenter(
surround=50)
result_page.results.formatter = JsonFormatter()
if corrected and corrected.query != str_q:
corrected_query = corrected.string
else:
corrected_query = None
yield result_page, corrected_query
finally:
searcher.close()
class DelayedQuery:
@property
def _query(self):
if 'query' in self.query_params:
qp = MultifieldParser(
["content", "title", "correspondent", "tag", "type"],
self.ix.schema)
qp.add_plugin(DateParserPlugin())
q = qp.parse(self.query_params['query'])
elif 'more_like_id' in self.query_params:
more_like_doc_id = int(self.query_params['more_like_id'])
content = Document.objects.get(id=more_like_doc_id).content
docnum = self.searcher.document_number(id=more_like_doc_id)
kts = self.searcher.key_terms_from_text(
'content', content, numterms=20,
model=classify.Bo1Model, normalize=False)
q = query.Or(
[query.Term('content', word, boost=weight)
for word, weight in kts])
else:
raise ValueError(
"Either query or more_like_id is required."
)
return q
raise NotImplementedError()
@property
def _query_filter(self):
@ -219,32 +162,114 @@ class DelayedQuery:
for k, v in self.query_params.items():
if k == 'correspondent__id':
criterias.append(query.Term('correspondent_id', v))
elif k == 'tags__id__all':
for tag_id in v.split(","):
criterias.append(query.Term('tag_id', tag_id))
elif k == 'document_type__id':
criterias.append(query.Term('type_id', v))
elif k == 'correspondent__isnull':
criterias.append(query.Term("has_correspondent", v == "false"))
elif k == 'is_tagged':
criterias.append(query.Term("has_tag", v == "true"))
elif k == 'document_type__isnull':
criterias.append(query.Term("has_type", v == "false"))
elif k == 'created__date__lt':
pass
elif k == 'created__date__gt':
pass
elif k == 'added__date__gt':
pass
elif k == 'added__date__lt':
pass
if len(criterias) > 0:
return query.And(criterias)
else:
return None
def __init__(self, ix, searcher, query_params, page_size):
self.ix = ix
@property
def _query_sortedby(self):
if not 'ordering' in self.query_params:
return None, False
o: str = self.query_params['ordering']
if o.startswith('-'):
return o[1:], True
else:
return o, False
def __init__(self, searcher: Searcher, query_params, page_size):
self.searcher = searcher
self.query_params = query_params
self.page_size = page_size
self.saved_results = dict()
def __len__(self):
results = self.searcher.search(self._query, limit=1, filter=self._query_filter)
return len(results)
#return 1000
page = self[0:1]
return len(page)
def __getitem__(self, item):
if item.start in self.saved_results:
return self.saved_results[item.start]
q, mask = self._query
sortedby, reverse = self._query_sortedby
print("OY", self.page_size)
page: ResultsPage = self.searcher.search_page(
self._query,
q,
mask=mask,
filter=self._query_filter,
pagenum=math.floor(item.start / self.page_size) + 1,
pagelen=self.page_size
pagelen=self.page_size,
sortedby=sortedby,
reverse=reverse
)
page.results.fragmenter = highlight.ContextFragmenter(
surround=50)
page.results.formatter = HtmlFormatter(tagname="span", between=" ... ")
self.saved_results[item.start] = page
return page
class DelayedFullTextQuery(DelayedQuery):
@property
def _query(self):
q_str = self.query_params['query']
qp = MultifieldParser(
["content", "title", "correspondent", "tag", "type"],
self.searcher.ixreader.schema)
qp.add_plugin(DateParserPlugin())
q = qp.parse(q_str)
corrected = self.searcher.correct_query(q, q_str)
if corrected.query != q:
corrected_query = corrected.string
return q, None
class DelayedMoreLikeThisQuery(DelayedQuery):
@property
def _query(self):
more_like_doc_id = int(self.query_params['more_like_id'])
content = Document.objects.get(id=more_like_doc_id).content
docnum = self.searcher.document_number(id=more_like_doc_id)
kts = self.searcher.key_terms_from_text(
'content', content, numterms=20,
model=classify.Bo1Model, normalize=False)
q = query.Or(
[query.Term('content', word, boost=weight)
for word, weight in kts])
mask = {docnum}
return q, mask
def autocomplete(ix, term, limit=10):
with ix.reader() as reader:
terms = []

7
src/documents/models.py
View File

@ -359,7 +359,10 @@ class SavedView(models.Model):
sort_field = models.CharField(
_("sort field"),
max_length=128)
max_length=128,
null=True,
blank=True
)
sort_reverse = models.BooleanField(
_("sort reverse"),
default=False)
@ -387,6 +390,8 @@ class SavedViewFilterRule(models.Model):
(17, _("does not have tag")),
(18, _("does not have ASN")),
(19, _("title or content contains")),
(20, _("fulltext query")),
(21, _("more like this"))
]
saved_view = models.ForeignKey(

10
src/documents/tests/test_index.py
View File

@ -1,20 +1,10 @@
from django.test import TestCase
from documents import index
from documents.index import JsonFormatter
from documents.models import Document
from documents.tests.utils import DirectoriesMixin
class JsonFormatterTest(TestCase):
def setUp(self) -> None:
self.formatter = JsonFormatter()
def test_empty_fragments(self):
self.assertListEqual(self.formatter.format([]), [])
class TestAutoComplete(DirectoriesMixin, TestCase):
def test_auto_complete(self):

117
src/documents/views.py
View File

@ -36,7 +36,6 @@ from rest_framework.viewsets import (
from paperless.db import GnuPG
from paperless.views import StandardPagination
from . import index
from .bulk_download import OriginalAndArchiveStrategy, OriginalsOnlyStrategy, \
ArchiveOnlyStrategy
from .classifier import load_classifier
@ -332,15 +331,23 @@ class SearchResultSerializer(DocumentSerializer):
def to_representation(self, instance):
doc = Document.objects.get(id=instance['id'])
# repressentation = super(SearchResultSerializer, self).to_representation(doc)
# repressentation['__search_hit__'] = {
# "score": instance.score
# }
return super(SearchResultSerializer, self).to_representation(doc)
representation = super(SearchResultSerializer, self).to_representation(doc)
representation['__search_hit__'] = {
"score": instance.score,
"highlights": instance.highlights("content",
text=doc.content) if doc else None, # NOQA: E501
"rank": instance.rank
}
return representation
class UnifiedSearchViewSet(DocumentViewSet):
def __init__(self, *args, **kwargs):
super(UnifiedSearchViewSet, self).__init__(*args, **kwargs)
self.searcher = None
def get_serializer_class(self):
if self._is_search_request():
return SearchResultSerializer
@ -348,25 +355,39 @@ class UnifiedSearchViewSet(DocumentViewSet):
return DocumentSerializer
def _is_search_request(self):
return "query" in self.request.query_params
return "query" in self.request.query_params or "more_like_id" in self.request.query_params
def filter_queryset(self, queryset):
if self._is_search_request():
ix = index.open_index()
return index.DelayedQuery(ix, self.searcher, self.request.query_params, self.paginator.page_size)
from documents import index
if "query" in self.request.query_params:
query_class = index.DelayedFullTextQuery
elif "more_like_id" in self.request.query_params:
query_class = index.DelayedMoreLikeThisQuery
else:
raise ValueError()
return query_class(
self.searcher,
self.request.query_params,
self.paginator.get_page_size(self.request))
else:
return super(UnifiedSearchViewSet, self).filter_queryset(queryset)
def list(self, request, *args, **kwargs):
if self._is_search_request():
ix = index.open_index()
with ix.searcher() as s:
self.searcher = s
return super(UnifiedSearchViewSet, self).list(request)
from documents import index
try:
with index.open_index_searcher() as s:
self.searcher = s
return super(UnifiedSearchViewSet, self).list(request)
except Exception as e:
return HttpResponseBadRequest(str(e))
else:
return super(UnifiedSearchViewSet, self).list(request)
class LogViewSet(ViewSet):
permission_classes = (IsAuthenticated,)
@ -518,74 +539,6 @@ class SelectionDataView(GenericAPIView):
return r
class SearchView(APIView):
permission_classes = (IsAuthenticated,)
def add_infos_to_hit(self, r):
try:
doc = Document.objects.get(id=r['id'])
except Document.DoesNotExist:
logger.warning(
f"Search index returned a non-existing document: "
f"id: {r['id']}, title: {r['title']}. "
f"Search index needs reindex."
)
doc = None
return {'id': r['id'],
'highlights': r.highlights("content", text=doc.content) if doc else None, # NOQA: E501
'score': r.score,
'rank': r.rank,
'document': DocumentSerializer(doc).data if doc else None,
'title': r['title']
}
def get(self, request, format=None):
from documents import index
if 'query' in request.query_params:
query = request.query_params['query']
else:
query = None
if 'more_like' in request.query_params:
more_like_id = request.query_params['more_like']
more_like_content = Document.objects.get(id=more_like_id).content
else:
more_like_id = None
more_like_content = None
if not query and not more_like_id:
return Response({
'count': 0,
'page': 0,
'page_count': 0,
'corrected_query': None,
'results': []})
try:
page = int(request.query_params.get('page', 1))
except (ValueError, TypeError):
page = 1
if page < 1:
page = 1
ix = index.open_index()
try:
with index.query_page(ix, page, query, more_like_id, more_like_content) as (result_page, corrected_query): # NOQA: E501
return Response(
{'count': len(result_page),
'page': result_page.pagenum,
'page_count': result_page.pagecount,
'corrected_query': corrected_query,
'results': list(map(self.add_infos_to_hit, result_page))})
except Exception as e:
return HttpResponseBadRequest(str(e))
class SearchAutoCompleteView(APIView):
permission_classes = (IsAuthenticated,)

5
src/paperless/urls.py
View File

@ -16,7 +16,6 @@ from documents.views import (
LogViewSet,
TagViewSet,
DocumentTypeViewSet,
SearchView,
IndexView,
SearchAutoCompleteView,
StatisticsView,
@ -47,10 +46,6 @@ urlpatterns = [
SearchAutoCompleteView.as_view(),
name="autocomplete"),
re_path(r"^search/",
SearchView.as_view(),
name="search"),
re_path(r"^statistics/",
StatisticsView.as_view(),
name="statistics"),