From 164418880a93301b61e577363d4211b8142dcf0e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 17 Dec 2020 21:36:21 +0100
Subject: [PATCH] more like this searching

---
 .../document-detail.component.html            |  6 +++
 .../document-detail.component.ts              |  4 ++
 .../document-card-large.component.html        | 11 +++++-
 .../document-card-large.component.ts          | 13 +++++++
 .../result-highlight.component.scss           |  2 +-
 .../components/search/search.component.html   | 10 ++++-
 .../app/components/search/search.component.ts | 26 +++++++++++--
 .../src/app/services/rest/search.service.ts   | 10 ++++-
 src/documents/index.py                        | 37 ++++++++++++++-----
 src/documents/views.py                        | 21 +++++++----
 10 files changed, 113 insertions(+), 27 deletions(-)

diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index f4a64c2cc..f7e1ff855 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -24,6 +24,12 @@
 
     </div>
 
+    <button type="button" class="btn btn-sm btn-outline-primary mr-2" (click)="moreLike()">
+        <svg class="buttonicon" fill="currentColor">
+            <use xlink:href="assets/bootstrap-icons.svg#three-dots" />
+        </svg>
+        <span class="d-none d-lg-inline"> More like this</span>
+    </button>
 
     <button type="button" class="btn btn-sm btn-outline-primary" (click)="close()">
         <svg class="buttonicon" fill="currentColor">
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts
index b4005b920..90594bd0a 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -168,6 +168,10 @@ export class DocumentDetailComponent implements OnInit {
 
   }
 
+  moreLike() {
+    this.router.navigate(["search"], {queryParams: {more_like:this.document.id}})
+  }
+
   hasNext() {
     return this.documentListViewService.hasNext(this.documentId)
   }
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
index c2645db5e..32abaaef1 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
@@ -25,6 +25,12 @@
 
         <div class="d-flex justify-content-between align-items-center">
           <div class="btn-group">
+            <a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary">
+              <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-three-dots" viewBox="0 0 16 16">
+                <path fill-rule="evenodd" d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
+              </svg>
+              More like this
+            </a>
             <a routerLink="/documents/{{document.id}}" class="btn btn-sm btn-outline-secondary">
               <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-pencil" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
                 <path fill-rule="evenodd" d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5L13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175l-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
@@ -45,10 +51,13 @@
               </svg>
               Download
             </a>
+            <ngb-progressbar [type]="searchScoreClass" [value]="searchScore" style="width: 100px; height: 5px; margin: 10px;" [max]="1"></ngb-progressbar>
+            
           </div>
+          
           <small class="text-muted">Created: {{document.created | date}}</small>
         </div>
-
+        
       </div>
     </div>
   </div>
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
index 2e056cc70..44f9cb906 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
@@ -24,6 +24,19 @@ export class DocumentCardLargeComponent implements OnInit {
   @Output()
   clickCorrespondent = new EventEmitter<number>()
 
+  @Input()
+  searchScore: number
+
+  get searchScoreClass() {
+    if (this.searchScore > 0.7) {
+      return "success"
+    } else if (this.searchScore > 0.3) {
+      return "warning"
+    } else {
+      return "danger"
+    }
+  }
+
   ngOnInit(): void {
   }
 
diff --git a/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss b/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
index 645fb0426..e04dd13b2 100644
--- a/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
+++ b/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
@@ -1,4 +1,4 @@
 .match {
     color: black;
-    background-color: orange;
+    background-color: rgb(255, 211, 66);
 }
\ No newline at end of file
diff --git a/src-ui/src/app/components/search/search.component.html b/src-ui/src/app/components/search/search.component.html
index 55fcee900..609bea9e5 100644
--- a/src-ui/src/app/components/search/search.component.html
+++ b/src-ui/src/app/components/search/search.component.html
@@ -3,7 +3,12 @@
 
 <div *ngIf="errorMessage" class="alert alert-danger">Invalid search query: {{errorMessage}}</div>
 
-<p>
+<p *ngIf="more_like">
+    Showing documents similar to
+    <a routerLink="/documents/{{more_like}}">{{more_like_doc?.original_file_name}}</a>
+</p>
+
+<p *ngIf="query">
     Search string: <i>{{query}}</i>
     <ng-container *ngIf="correctedQuery">
         - Did you mean "<a [routerLink]="" (click)="searchCorrectedQuery()">{{correctedQuery}}</a>"?
@@ -15,7 +20,8 @@
     <p>{{resultCount}} result(s)</p>
     <app-document-card-large *ngFor="let result of results"
         [document]="result.document"
-        [details]="result.highlights">
+        [details]="result.highlights"
+        [searchScore]="result.score / maxScore">
 
 </app-document-card-large>
 </div>
diff --git a/src-ui/src/app/components/search/search.component.ts b/src-ui/src/app/components/search/search.component.ts
index de8b4652f..b2b10d632 100644
--- a/src-ui/src/app/components/search/search.component.ts
+++ b/src-ui/src/app/components/search/search.component.ts
@@ -1,6 +1,9 @@
 import { Component, OnInit } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
+import { PaperlessDocument } from 'src/app/data/paperless-document';
+import { PaperlessDocumentType } from 'src/app/data/paperless-document-type';
 import { SearchHit } from 'src/app/data/search-result';
+import { DocumentService } from 'src/app/services/rest/document.service';
 import { SearchService } from 'src/app/services/rest/search.service';
 
 @Component({
@@ -14,6 +17,10 @@ export class SearchComponent implements OnInit {
 
   query: string = ""
 
+  more_like: number
+
+  more_like_doc: PaperlessDocument
+
   searching = false
 
   currentPage = 1
@@ -26,11 +33,23 @@ export class SearchComponent implements OnInit {
 
   errorMessage: string
 
-  constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router) { }
+  get maxScore() {
+    return this.results?.length > 0 ? this.results[0].score : 100
+  }
+
+  constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router, private documentService: DocumentService) { }
 
   ngOnInit(): void {
     this.route.queryParamMap.subscribe(paramMap => {
       this.query = paramMap.get('query')
+      this.more_like = paramMap.has('more_like') ? +paramMap.get('more_like') : null
+      if (this.more_like) {
+        this.documentService.get(this.more_like).subscribe(r => {
+          this.more_like_doc = r
+        })
+      } else {
+        this.more_like_doc = null
+      }
       this.searching = true
       this.currentPage = 1
       this.loadPage()
@@ -39,13 +58,14 @@ export class SearchComponent implements OnInit {
   }
 
   searchCorrectedQuery() {
-    this.router.navigate(["search"], {queryParams: {query: this.correctedQuery}})
+    this.router.navigate(["search"], {queryParams: {query: this.correctedQuery, more_like: this.more_like}})
   }
 
   loadPage(append: boolean = false) {
     this.errorMessage = null
     this.correctedQuery = null
-    this.searchService.search(this.query, this.currentPage).subscribe(result => {
+
+    this.searchService.search(this.query, this.currentPage, this.more_like).subscribe(result => {
       if (append) {
         this.results.push(...result.results)
       } else {
diff --git a/src-ui/src/app/services/rest/search.service.ts b/src-ui/src/app/services/rest/search.service.ts
index b19a55769..3799f3dc7 100644
--- a/src-ui/src/app/services/rest/search.service.ts
+++ b/src-ui/src/app/services/rest/search.service.ts
@@ -15,11 +15,17 @@ export class SearchService {
   
   constructor(private http: HttpClient, private documentService: DocumentService) { }
 
-  search(query: string, page?: number): Observable<SearchResult> {
-    let httpParams = new HttpParams().set('query', query)
+  search(query: string, page?: number, more_like?: number): Observable<SearchResult> {
+    let httpParams = new HttpParams()
+    if (query) {
+      httpParams = httpParams.set('query', query)
+    }
     if (page) {
       httpParams = httpParams.set('page', page.toString())
     }
+    if (more_like) {
+      httpParams = httpParams.set('more_like', more_like.toString())
+    }
     return this.http.get<SearchResult>(`${environment.apiBaseUrl}search/`, {params: httpParams}).pipe(
       map(result => {
         result.results.forEach(hit => this.documentService.addObservablesToDocument(hit.document))
diff --git a/src/documents/index.py b/src/documents/index.py
index 53bf34542..7d022182f 100644
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -3,7 +3,7 @@ import os
 from contextlib import contextmanager
 
 from django.conf import settings
-from whoosh import highlight
+from whoosh import highlight, classify, query
 from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME
 from whoosh.highlight import Formatter, get_text
 from whoosh.index import create_in, exists_in, open_dir
@@ -120,22 +120,39 @@ def remove_document_from_index(document):
 
 
 @contextmanager
-def query_page(ix, querystring, page):
+def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content):
     searcher = ix.searcher()
     try:
-        qp = MultifieldParser(
-            ["content", "title", "correspondent", "tag", "type"],
-            ix.schema)
-        qp.add_plugin(DateParserPlugin())
+        if querystring:
+            qp = MultifieldParser(
+                ["content", "title", "correspondent", "tag", "type"],
+                ix.schema)
+            qp.add_plugin(DateParserPlugin())
+            str_q = qp.parse(querystring)
+            corrected = searcher.correct_query(str_q, querystring)
+        else:
+            str_q = None
+            corrected = None
+
+        if more_like_doc_id:
+            docnum = searcher.document_number(id=more_like_doc_id)
+            kts = searcher.key_terms_from_text('content', more_like_doc_content, numterms=20,
+                                           model=classify.Bo1Model, normalize=False)
+            more_like_q = query.Or([query.Term('content', word, boost=weight)
+                          for word, weight in kts])
+            result_page = searcher.search_page(more_like_q, page, filter=str_q, mask={docnum})
+        elif str_q:
+            result_page = searcher.search_page(str_q, page)
+        else:
+            raise ValueError(
+                "Either querystring or more_like_doc_id is required."
+            )
 
-        q = qp.parse(querystring)
-        result_page = searcher.search_page(q, page)
         result_page.results.fragmenter = highlight.ContextFragmenter(
             surround=50)
         result_page.results.formatter = JsonFormatter()
 
-        corrected = searcher.correct_query(q, querystring)
-        if corrected.query != q:
+        if corrected and corrected.query != str_q:
             corrected_query = corrected.string
         else:
             corrected_query = None
diff --git a/src/documents/views.py b/src/documents/views.py
index bf31c749b..bd9a748e8 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -335,14 +335,19 @@ class SearchView(APIView):
                 }
 
     def get(self, request, format=None):
-        if 'query' not in request.query_params:
-            return Response({
-                'count': 0,
-                'page': 0,
-                'page_count': 0,
-                'results': []})
 
-        query = request.query_params['query']
+        if 'query' in request.query_params:
+            query = request.query_params['query']
+        else:
+            query = None
+
+        if 'more_like' in request.query_params:
+            more_like_id = request.query_params['more_like']
+            more_like_content = Document.objects.get(id=more_like_id).content
+        else:
+            more_like_id = None
+            more_like_content = None
+
         try:
             page = int(request.query_params.get('page', 1))
         except (ValueError, TypeError):
@@ -352,7 +357,7 @@ class SearchView(APIView):
             page = 1
 
         try:
-            with index.query_page(self.ix, query, page) as (result_page,
+            with index.query_page(self.ix, page, query, more_like_id, more_like_content) as (result_page,
                                                             corrected_query):
                 return Response(
                     {'count': len(result_page),