more like this searching

2025-10-30 03:56:23 -05:00 · 2020-12-17 21:36:21 +01:00
parent 9d88e3ee07
commit 29c094e407
10 changed files with 113 additions and 27 deletions
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -24,6 +24,12 @@

    </div>

+    <button type="button" class="btn btn-sm btn-outline-primary mr-2" (click)="moreLike()">
+        <svg class="buttonicon" fill="currentColor">
+            <use xlink:href="assets/bootstrap-icons.svg#three-dots" />
+        </svg>
+        <span class="d-none d-lg-inline"> More like this</span>
+    </button>

    <button type="button" class="btn btn-sm btn-outline-primary" (click)="close()">
        <svg class="buttonicon" fill="currentColor">
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -168,6 +168,10 @@ export class DocumentDetailComponent implements OnInit {

  }

+  moreLike() {
+    this.router.navigate(["search"], {queryParams: {more_like:this.document.id}})
+  }
+
  hasNext() {
    return this.documentListViewService.hasNext(this.documentId)
  }
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
@@ -25,6 +25,12 @@

        <div class="d-flex justify-content-between align-items-center">
          <div class="btn-group">
+            <a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary">
+              <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-three-dots" viewBox="0 0 16 16">
+                <path fill-rule="evenodd" d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
+              </svg>
+              More like this
+            </a>
            <a routerLink="/documents/{{document.id}}" class="btn btn-sm btn-outline-secondary">
              <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-pencil" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
                <path fill-rule="evenodd" d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5L13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175l-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
@@ -45,10 +51,13 @@
              </svg>
              Download
            </a>
+            <ngb-progressbar [type]="searchScoreClass" [value]="searchScore" style="width: 100px; height: 5px; margin: 10px;" [max]="1"></ngb-progressbar>
+            
          </div>
+          
          <small class="text-muted">Created: {{document.created | date}}</small>
        </div>
-
+        
      </div>
    </div>
  </div>
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
@@ -24,6 +24,19 @@ export class DocumentCardLargeComponent implements OnInit {
  @Output()
  clickCorrespondent = new EventEmitter<number>()

+  @Input()
+  searchScore: number
+
+  get searchScoreClass() {
+    if (this.searchScore > 0.7) {
+      return "success"
+    } else if (this.searchScore > 0.3) {
+      return "warning"
+    } else {
+      return "danger"
+    }
+  }
+
  ngOnInit(): void {
  }

--- a/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
+++ b/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
@@ -1,4 +1,4 @@
 .match {
    color: black;
-    background-color: orange;
+    background-color: rgb(255, 211, 66);
 }
--- a/src-ui/src/app/components/search/search.component.html
+++ b/src-ui/src/app/components/search/search.component.html
@@ -3,7 +3,12 @@

 <div *ngIf="errorMessage" class="alert alert-danger">Invalid search query: {{errorMessage}}</div>

-<p>
+<p *ngIf="more_like">
+    Showing documents similar to
+    <a routerLink="/documents/{{more_like}}">{{more_like_doc?.original_file_name}}</a>
+</p>
+
+<p *ngIf="query">
    Search string: <i>{{query}}</i>
    <ng-container *ngIf="correctedQuery">
        - Did you mean "<a [routerLink]="" (click)="searchCorrectedQuery()">{{correctedQuery}}</a>"?
@@ -15,7 +20,8 @@
    <p>{{resultCount}} result(s)</p>
    <app-document-card-large *ngFor="let result of results"
        [document]="result.document"
-        [details]="result.highlights">
+        [details]="result.highlights"
+        [searchScore]="result.score / maxScore">

 </app-document-card-large>
 </div>
--- a/src-ui/src/app/components/search/search.component.ts
+++ b/src-ui/src/app/components/search/search.component.ts
@@ -1,6 +1,9 @@
 import { Component, OnInit } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
+import { PaperlessDocument } from 'src/app/data/paperless-document';
+import { PaperlessDocumentType } from 'src/app/data/paperless-document-type';
 import { SearchHit } from 'src/app/data/search-result';
+import { DocumentService } from 'src/app/services/rest/document.service';
 import { SearchService } from 'src/app/services/rest/search.service';

@Component({
@@ -14,6 +17,10 @@ export class SearchComponent implements OnInit {

  query: string = ""

+  more_like: number
+
+  more_like_doc: PaperlessDocument
+
  searching = false

  currentPage = 1
@@ -26,11 +33,23 @@ export class SearchComponent implements OnInit {

  errorMessage: string

-  constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router) { }
+  get maxScore() {
+    return this.results?.length > 0 ? this.results[0].score : 100
+  }
+
+  constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router, private documentService: DocumentService) { }

  ngOnInit(): void {
    this.route.queryParamMap.subscribe(paramMap => {
      this.query = paramMap.get('query')
+      this.more_like = paramMap.has('more_like') ? +paramMap.get('more_like') : null
+      if (this.more_like) {
+        this.documentService.get(this.more_like).subscribe(r => {
+          this.more_like_doc = r
+        })
+      } else {
+        this.more_like_doc = null
+      }
      this.searching = true
      this.currentPage = 1
      this.loadPage()
@@ -39,13 +58,14 @@ export class SearchComponent implements OnInit {
  }

  searchCorrectedQuery() {
-    this.router.navigate(["search"], {queryParams: {query: this.correctedQuery}})
+    this.router.navigate(["search"], {queryParams: {query: this.correctedQuery, more_like: this.more_like}})
  }

  loadPage(append: boolean = false) {
    this.errorMessage = null
    this.correctedQuery = null
-    this.searchService.search(this.query, this.currentPage).subscribe(result => {
+
+    this.searchService.search(this.query, this.currentPage, this.more_like).subscribe(result => {
      if (append) {
        this.results.push(...result.results)
      } else {
--- a/src-ui/src/app/services/rest/search.service.ts
+++ b/src-ui/src/app/services/rest/search.service.ts
@@ -15,11 +15,17 @@ export class SearchService {
  
  constructor(private http: HttpClient, private documentService: DocumentService) { }

-  search(query: string, page?: number): Observable<SearchResult> {
-    let httpParams = new HttpParams().set('query', query)
+  search(query: string, page?: number, more_like?: number): Observable<SearchResult> {
+    let httpParams = new HttpParams()
+    if (query) {
+      httpParams = httpParams.set('query', query)
+    }
    if (page) {
      httpParams = httpParams.set('page', page.toString())
    }
+    if (more_like) {
+      httpParams = httpParams.set('more_like', more_like.toString())
+    }
    return this.http.get<SearchResult>(`${environment.apiBaseUrl}search/`, {params: httpParams}).pipe(
      map(result => {
        result.results.forEach(hit => this.documentService.addObservablesToDocument(hit.document))
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -3,7 +3,7 @@ import os
 from contextlib import contextmanager

 from django.conf import settings
-from whoosh import highlight
+from whoosh import highlight, classify, query
 from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME
 from whoosh.highlight import Formatter, get_text
 from whoosh.index import create_in, exists_in, open_dir
@@ -120,22 +120,39 @@ def remove_document_from_index(document):


@contextmanager
-def query_page(ix, querystring, page):
+def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content):
    searcher = ix.searcher()
    try:
-        qp = MultifieldParser(
-            ["content", "title", "correspondent", "tag", "type"],
-            ix.schema)
-        qp.add_plugin(DateParserPlugin())
+        if querystring:
+            qp = MultifieldParser(
+                ["content", "title", "correspondent", "tag", "type"],
+                ix.schema)
+            qp.add_plugin(DateParserPlugin())
+            str_q = qp.parse(querystring)
+            corrected = searcher.correct_query(str_q, querystring)
+        else:
+            str_q = None
+            corrected = None
+
+        if more_like_doc_id:
+            docnum = searcher.document_number(id=more_like_doc_id)
+            kts = searcher.key_terms_from_text('content', more_like_doc_content, numterms=20,
+                                           model=classify.Bo1Model, normalize=False)
+            more_like_q = query.Or([query.Term('content', word, boost=weight)
+                          for word, weight in kts])
+            result_page = searcher.search_page(more_like_q, page, filter=str_q, mask={docnum})
+        elif str_q:
+            result_page = searcher.search_page(str_q, page)
+        else:
+            raise ValueError(
+                "Either querystring or more_like_doc_id is required."
+            )

-        q = qp.parse(querystring)
-        result_page = searcher.search_page(q, page)
        result_page.results.fragmenter = highlight.ContextFragmenter(
            surround=50)
        result_page.results.formatter = JsonFormatter()

-        corrected = searcher.correct_query(q, querystring)
-        if corrected.query != q:
+        if corrected and corrected.query != str_q:
            corrected_query = corrected.string
        else:
            corrected_query = None
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -335,14 +335,19 @@ class SearchView(APIView):
                }

    def get(self, request, format=None):
-        if 'query' not in request.query_params:
-            return Response({
-                'count': 0,
-                'page': 0,
-                'page_count': 0,
-                'results': []})

-        query = request.query_params['query']
+        if 'query' in request.query_params:
+            query = request.query_params['query']
+        else:
+            query = None
+
+        if 'more_like' in request.query_params:
+            more_like_id = request.query_params['more_like']
+            more_like_content = Document.objects.get(id=more_like_id).content
+        else:
+            more_like_id = None
+            more_like_content = None
+
        try:
            page = int(request.query_params.get('page', 1))
        except (ValueError, TypeError):
@@ -352,7 +357,7 @@ class SearchView(APIView):
            page = 1

        try:
-            with index.query_page(self.ix, query, page) as (result_page,
+            with index.query_page(self.ix, page, query, more_like_id, more_like_content) as (result_page,
                                                            corrected_query):
                return Response(
                    {'count': len(result_page),