From fcc0cb7293e332b4ff140337f6b82132715ee641 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 29 Nov 2020 23:32:03 +0100 Subject: [PATCH 01/10] fixes #71 --- docs/changelog.rst | 10 +++++++--- .../saved-view-widget.component.html | 5 ++++- .../saved-view-widget/saved-view-widget.component.ts | 12 +++++++++++- .../statistics-widget.component.html | 6 ++++-- .../upload-file-widget.component.html | 2 +- .../widgets/widget-frame/widget-frame.component.html | 8 ++++++-- 6 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d6ad73ce2..eef6055c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,11 +8,15 @@ Changelog paperless-ng 0.9.4 ################## -* Front end: Clickable tags, correspondents and types allow quick filtering for related documents. -* Front end: Saved views are now editable. -* Front end: Preview documents directly in the browser. +* Front end: + * Clickable tags, correspondents and types allow quick filtering for related documents. + * Saved views are now editable. + * Preview documents directly in the browser. + * Navigation from the dashboard to saved views. + * Fixes: * A severe error when trying to use post consume scripts. + * An error in the consumer that cause invalid messages of missing files to show up in the log. * The documentation now contains information about bare metal installs. paperless-ng 0.9.3 diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html index 712e4dec7..2dfbe4481 100644 --- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html +++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.html @@ -1,6 +1,9 @@ - + Show all + + +
diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts index 9b124715f..413df0ae4 100644 --- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts +++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts @@ -1,6 +1,8 @@ import { Component, Input, OnInit } from '@angular/core'; +import { Router } from '@angular/router'; import { PaperlessDocument } from 'src/app/data/paperless-document'; import { SavedViewConfig } from 'src/app/data/saved-view-config'; +import { DocumentListViewService } from 'src/app/services/document-list-view.service'; import { DocumentService } from 'src/app/services/rest/document.service'; @Component({ @@ -10,7 +12,10 @@ import { DocumentService } from 'src/app/services/rest/document.service'; }) export class SavedViewWidgetComponent implements OnInit { - constructor(private documentService: DocumentService) { } + constructor( + private documentService: DocumentService, + private router: Router, + private list: DocumentListViewService) { } @Input() savedView: SavedViewConfig @@ -23,4 +28,9 @@ export class SavedViewWidgetComponent implements OnInit { }) } + showAll() { + this.list.load(this.savedView) + this.router.navigate(["documents"]) + } + } diff --git a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html index 693935290..50d844b36 100644 --- a/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html +++ b/src-ui/src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html @@ -1,4 +1,6 @@ -

Documents in inbox: {{statistics.documents_inbox}}

-

Total documents: {{statistics.documents_total}}

+ +

Documents in inbox: {{statistics.documents_inbox}}

+

Total documents: {{statistics.documents_total}}

+ \ No newline at end of file diff --git a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html index fa7faab31..cb114e49e 100644 --- a/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html +++ b/src-ui/src/app/components/dashboard/widgets/upload-file-widget/upload-file-widget.component.html @@ -1,6 +1,6 @@ -
+
-
{{title}}
+
+
{{title}}
+ +
+
- +
\ No newline at end of file From 2224540b711c3deae901a5ba10b2c7d401990dfb Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 29 Nov 2020 23:32:12 +0100 Subject: [PATCH 02/10] don't show links in the search results. --- .../document-card-large/document-card-large.component.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html index 63a8bf710..4e86b6ddc 100644 --- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html +++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html @@ -9,9 +9,11 @@
- {{document.correspondent.name}}: + {{document.correspondent.name}} + {{document.correspondent.name}}: - {{document.title}} + {{document.title}} +
#{{document.archive_serial_number}}
From 64ee8eab2f2cfc12f669172cc96b3d41a14cf181 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Sun, 29 Nov 2020 23:34:09 +0100 Subject: [PATCH 03/10] changelog --- docs/changelog.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index eef6055c1..45817aa1a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,15 +9,19 @@ paperless-ng 0.9.4 ################## * Front end: + * Clickable tags, correspondents and types allow quick filtering for related documents. * Saved views are now editable. * Preview documents directly in the browser. * Navigation from the dashboard to saved views. * Fixes: + * A severe error when trying to use post consume scripts. * An error in the consumer that cause invalid messages of missing files to show up in the log. -* The documentation now contains information about bare metal installs. + +* The documentation now contains information about bare metal installs and a section about + how to setup the development environment. paperless-ng 0.9.3 ################## From f51207fc32df05d04f6f48a9bdd60de988fb1481 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 30 Nov 2020 00:40:04 +0100 Subject: [PATCH 04/10] added file type checks to the parsers to prevent temporary files from being consumed. Also: parsers announce file types they wish to use as default for each mime type. --- src/documents/consumer.py | 21 ++++++++++++++++++-- src/documents/models.py | 6 +++--- src/documents/parsers.py | 24 +++++++++++++++++++++++ src/documents/tests/test_consumer.py | 6 +++--- src/documents/tests/test_parsers.py | 29 ++++++++++++++++++++++++---- src/paperless_tesseract/signals.py | 10 +++++----- src/paperless_text/signals.py | 8 ++++---- 7 files changed, 83 insertions(+), 21 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 1842fbb56..37b2c032d 100755 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -9,10 +9,11 @@ from django.db import transaction from django.utils import timezone from .classifier import DocumentClassifier, IncompatibleClassifierVersionError -from .file_handling import generate_filename, create_source_path_directory +from .file_handling import create_source_path_directory from .loggers import LoggingMixin from .models import Document, FileInfo, Correspondent, DocumentType, Tag -from .parsers import ParseError, get_parser_class_for_mime_type +from .parsers import ParseError, get_parser_class_for_mime_type, \ + get_supported_file_extensions from .signals import ( document_consumption_finished, document_consumption_started @@ -39,6 +40,21 @@ class Consumer(LoggingMixin): raise ConsumerError("Cannot consume {}: It is not a file".format( self.path)) + def pre_check_file_extension(self): + extensions = get_supported_file_extensions() + _, ext = os.path.splitext(self.filename) + + if not ext: + raise ConsumerError( + f"Not consuming {self.filename}: File type unknown." + ) + + if ext not in extensions: + raise ConsumerError( + f"Not consuming {self.filename}: File extension {ext} does " + f"not map to any known file type ({str(extensions)})" + ) + def pre_check_duplicate(self): with open(self.path, "rb") as f: checksum = hashlib.md5(f.read()).hexdigest() @@ -80,6 +96,7 @@ class Consumer(LoggingMixin): # Make sure that preconditions for consuming the file are met. self.pre_check_file_exists() + self.pre_check_file_extension() self.pre_check_directories() self.pre_check_duplicate() diff --git a/src/documents/models.py b/src/documents/models.py index cd4517a3d..ae6665b76 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -1,7 +1,6 @@ # coding=utf-8 import logging -import mimetypes import os import re from collections import OrderedDict @@ -12,6 +11,8 @@ from django.db import models from django.utils import timezone from django.utils.text import slugify +from documents.parsers import get_default_file_extension + class MatchingModel(models.Model): @@ -230,8 +231,7 @@ class Document(models.Model): @property def file_type(self): - # TODO: this is not stable across python versions - return mimetypes.guess_extension(str(self.mime_type)) + return get_default_file_extension(self.mime_type) @property def thumbnail_path(self): diff --git a/src/documents/parsers.py b/src/documents/parsers.py index eb8ccf45e..ad9bbdde6 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -1,4 +1,5 @@ import logging +import mimetypes import os import re import shutil @@ -42,6 +43,29 @@ def is_mime_type_supported(mime_type): return get_parser_class_for_mime_type(mime_type) is not None +def get_default_file_extension(mime_type): + for response in document_consumer_declaration.send(None): + parser_declaration = response[1] + supported_mime_types = parser_declaration["mime_types"] + + if mime_type in supported_mime_types: + return supported_mime_types[mime_type] + + return None + + +def get_supported_file_extensions(): + extensions = set() + for response in document_consumer_declaration.send(None): + parser_declaration = response[1] + supported_mime_types = parser_declaration["mime_types"] + + for mime_type in supported_mime_types: + extensions.update(mimetypes.guess_all_extensions(mime_type)) + + return extensions + + def get_parser_class_for_mime_type(mime_type): options = [] diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index b436f76a1..1b2e3e649 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -423,7 +423,7 @@ class TestConsumer(DirectoriesMixin, TestCase): m = patcher.start() m.return_value = [(None, { "parser": self.make_dummy_parser, - "mime_types": ["application/pdf"], + "mime_types": {"application/pdf": ".pdf"}, "weight": 0 })] @@ -519,7 +519,7 @@ class TestConsumer(DirectoriesMixin, TestCase): try: self.consumer.try_consume_file(self.get_test_file()) except ConsumerError as e: - self.assertTrue(str(e).startswith("No parsers abvailable")) + self.assertTrue("File extension .pdf does not map to any" in str(e)) return self.fail("Should throw exception") @@ -528,7 +528,7 @@ class TestConsumer(DirectoriesMixin, TestCase): def testFaultyParser(self, m): m.return_value = [(None, { "parser": self.make_faulty_parser, - "mime_types": ["application/pdf"], + "mime_types": {"application/pdf": ".pdf"}, "weight": 0 })] diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index 239203186..3636671a3 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -4,7 +4,10 @@ from unittest import mock from django.test import TestCase -from documents.parsers import get_parser_class +from documents.parsers import get_parser_class, get_supported_file_extensions, get_default_file_extension, \ + get_parser_class_for_mime_type +from paperless_tesseract.parsers import RasterisedDocumentParser +from paperless_text.parsers import TextDocumentParser def fake_magic_from_file(file, mime=False): @@ -27,7 +30,7 @@ class TestParserDiscovery(TestCase): pass m.return_value = ( - (None, {"weight": 0, "parser": DummyParser, "mime_types": ["application/pdf"]}), + (None, {"weight": 0, "parser": DummyParser, "mime_types": {"application/pdf": ".pdf"}}), ) self.assertEqual( @@ -45,8 +48,8 @@ class TestParserDiscovery(TestCase): pass m.return_value = ( - (None, {"weight": 0, "parser": DummyParser1, "mime_types": ["application/pdf"]}), - (None, {"weight": 1, "parser": DummyParser2, "mime_types": ["application/pdf"]}), + (None, {"weight": 0, "parser": DummyParser1, "mime_types": {"application/pdf": ".pdf"}}), + (None, {"weight": 1, "parser": DummyParser2, "mime_types": {"application/pdf": ".pdf"}}), ) self.assertEqual( @@ -61,3 +64,21 @@ class TestParserDiscovery(TestCase): self.assertIsNone( get_parser_class("doc.pdf") ) + + +class TestParserAvailability(TestCase): + + def test_file_extensions(self): + + for ext in [".pdf", ".jpe", ".jpg", ".jpeg", ".txt", ".csv"]: + self.assertIn(ext, get_supported_file_extensions()) + self.assertEqual(get_default_file_extension('application/pdf'), ".pdf") + self.assertEqual(get_default_file_extension('image/png'), ".png") + self.assertEqual(get_default_file_extension('image/jpeg'), ".jpg") + self.assertEqual(get_default_file_extension('text/plain'), ".txt") + self.assertEqual(get_default_file_extension('text/csv'), ".csv") + self.assertEqual(get_default_file_extension('aasdasd/dgfgf'), None) + + self.assertEqual(get_parser_class_for_mime_type('application/pdf'), RasterisedDocumentParser) + self.assertEqual(get_parser_class_for_mime_type('text/plain'), TextDocumentParser) + self.assertEqual(get_parser_class_for_mime_type('text/sdgsdf'), None) diff --git a/src/paperless_tesseract/signals.py b/src/paperless_tesseract/signals.py index 712034038..57363b65e 100644 --- a/src/paperless_tesseract/signals.py +++ b/src/paperless_tesseract/signals.py @@ -5,9 +5,9 @@ def tesseract_consumer_declaration(sender, **kwargs): return { "parser": RasterisedDocumentParser, "weight": 0, - "mime_types": [ - "application/pdf", - "image/jpeg", - "image/png" - ] + "mime_types": { + "application/pdf": ".pdf", + "image/jpeg": ".jpg", + "image/png": ".png" + } } diff --git a/src/paperless_text/signals.py b/src/paperless_text/signals.py index f9ac9ad23..1e0493f4f 100644 --- a/src/paperless_text/signals.py +++ b/src/paperless_text/signals.py @@ -5,8 +5,8 @@ def text_consumer_declaration(sender, **kwargs): return { "parser": TextDocumentParser, "weight": 10, - "mime_types": [ - "text/plain", - "text/comma-separated-values" - ] + "mime_types": { + "text/plain": ".txt", + "text/csv": ".csv", + } } From 0d8688515c264ef8654368d37dd90fe539b4b6b7 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 30 Nov 2020 00:52:21 +0100 Subject: [PATCH 05/10] filename changes: don't include time. --- src/documents/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/documents/models.py b/src/documents/models.py index ae6665b76..a85128c07 100755 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -199,7 +199,7 @@ class Document(models.Model): ordering = ("correspondent", "title") def __str__(self): - created = self.created.strftime("%Y%m%d%H%M%S") + created = self.created.strftime("%Y%m%d") if self.correspondent and self.title: return "{}: {} - {}".format( created, self.correspondent, self.title) From 1ef12d2cbc8e29f32b8ee2d993e25e25d57e10f3 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 30 Nov 2020 15:13:53 +0100 Subject: [PATCH 06/10] searching for tags, spelling corrections fixes #74 --- docs/administration.rst | 1 + docs/changelog.rst | 9 +++++++ .../components/search/search.component.html | 10 ++++++-- .../app/components/search/search.component.ts | 18 ++++++++----- src-ui/src/app/data/search-result.ts | 4 ++- src/documents/index.py | 25 +++++++++++++------ src/documents/tests/test_api.py | 16 ++++++++++++ src/documents/views.py | 4 ++- 8 files changed, 70 insertions(+), 17 deletions(-) diff --git a/docs/administration.rst b/docs/administration.rst index 610d2c9d3..3284f7141 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -274,6 +274,7 @@ management command: This command takes no arguments. +.. _`administration-index`: Managing the document search index ================================== diff --git a/docs/changelog.rst b/docs/changelog.rst index 45817aa1a..f326b95ce 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -8,6 +8,15 @@ Changelog paperless-ng 0.9.4 ################## +* Searching: + + * Paperless now supports searching by tags. In order to have this applied to your + existing documents, you need to perform a ``document_index reindex`` management command + (see :ref:`administration-index`) + that adds tags to your search index. Paperless keeps your index updated after that whenever + something changes. + * Paperless now has spelling corrections ("Did you mean") for misstyped queries. + * Front end: * Clickable tags, correspondents and types allow quick filtering for related documents. diff --git a/src-ui/src/app/components/search/search.component.html b/src-ui/src/app/components/search/search.component.html index 59c24fa04..cb5c1a8e8 100644 --- a/src-ui/src/app/components/search/search.component.html +++ b/src-ui/src/app/components/search/search.component.html @@ -1,7 +1,13 @@ -

Search string: {{query}}

+

+ Search string: {{query}} + + - Did you mean "{{correctedQuery}}"? + + +

{{resultCount}} result(s)

@@ -10,4 +16,4 @@ [details]="result.highlights"> -
\ No newline at end of file + diff --git a/src-ui/src/app/components/search/search.component.ts b/src-ui/src/app/components/search/search.component.ts index f8c5d6cdc..8320ac545 100644 --- a/src-ui/src/app/components/search/search.component.ts +++ b/src-ui/src/app/components/search/search.component.ts @@ -1,5 +1,5 @@ import { Component, OnInit } from '@angular/core'; -import { ActivatedRoute } from '@angular/router'; +import { ActivatedRoute, Router } from '@angular/router'; import { SearchHit } from 'src/app/data/search-result'; import { SearchService } from 'src/app/services/rest/search.service'; @@ -9,7 +9,7 @@ import { SearchService } from 'src/app/services/rest/search.service'; styleUrls: ['./search.component.scss'] }) export class SearchComponent implements OnInit { - + results: SearchHit[] = [] query: string = "" @@ -22,7 +22,9 @@ export class SearchComponent implements OnInit { resultCount - constructor(private searchService: SearchService, private route: ActivatedRoute) { } + correctedQuery: string = null + + constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router) { } ngOnInit(): void { this.route.queryParamMap.subscribe(paramMap => { @@ -31,7 +33,12 @@ export class SearchComponent implements OnInit { this.currentPage = 1 this.loadPage() }) - + + } + + searchCorrectedQuery() { + this.router.navigate(["search"], {queryParams: {query: this.correctedQuery}}) + this.correctedQuery = null } loadPage(append: boolean = false) { @@ -44,12 +51,11 @@ export class SearchComponent implements OnInit { this.pageCount = result.page_count this.searching = false this.resultCount = result.count + this.correctedQuery = result.corrected_query }) } onScroll() { - console.log(this.currentPage) - console.log(this.pageCount) if (this.currentPage < this.pageCount) { this.currentPage += 1 this.loadPage(true) diff --git a/src-ui/src/app/data/search-result.ts b/src-ui/src/app/data/search-result.ts index b22dc64af..a769a8351 100644 --- a/src-ui/src/app/data/search-result.ts +++ b/src-ui/src/app/data/search-result.ts @@ -21,7 +21,9 @@ export interface SearchResult { page?: number page_count?: number + corrected_query?: string + results?: SearchHit[] -} \ No newline at end of file +} diff --git a/src/documents/index.py b/src/documents/index.py index ffa3e688f..822ac2e8a 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -4,7 +4,7 @@ from contextlib import contextmanager from django.conf import settings from whoosh import highlight -from whoosh.fields import Schema, TEXT, NUMERIC +from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD from whoosh.highlight import Formatter, get_text from whoosh.index import create_in, exists_in, open_dir from whoosh.qparser import MultifieldParser @@ -59,14 +59,15 @@ def get_schema(): id=NUMERIC(stored=True, unique=True, numtype=int), title=TEXT(stored=True), content=TEXT(), - correspondent=TEXT(stored=True) + correspondent=TEXT(stored=True), + tag=KEYWORD(stored=True, commas=True, scorable=True, lowercase=True) ) def open_index(recreate=False): try: if exists_in(settings.INDEX_DIR) and not recreate: - return open_dir(settings.INDEX_DIR) + return open_dir(settings.INDEX_DIR, schema=get_schema()) except Exception as e: logger.error(f"Error while opening the index: {e}, recreating.") @@ -77,11 +78,13 @@ def open_index(recreate=False): def update_document(writer, doc): logger.debug("Indexing {}...".format(doc)) + tags = ",".join([t.name for t in doc.tags.all()]) writer.update_document( id=doc.pk, title=doc.title, content=doc.content, - correspondent=doc.correspondent.name if doc.correspondent else None + correspondent=doc.correspondent.name if doc.correspondent else None, + tag=tags if tags else None ) @@ -106,13 +109,21 @@ def remove_document_from_index(document): def query_page(ix, query, page): searcher = ix.searcher() try: - query_parser = MultifieldParser(["content", "title", "correspondent"], - ix.schema).parse(query) + query_parser = MultifieldParser( + ["content", "title", "correspondent", "tag"], + ix.schema).parse(query) result_page = searcher.search_page(query_parser, page) result_page.results.fragmenter = highlight.ContextFragmenter( surround=50) result_page.results.formatter = JsonFormatter() - yield result_page + + corrected = searcher.correct_query(query_parser, query) + if corrected.query != query_parser: + corrected_query = corrected.string + else: + corrected_query = None + + yield result_page, corrected_query finally: searcher.close() diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index dabae6d82..b9f3dcfba 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -289,6 +289,22 @@ class DocumentApiTest(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, 200) self.assertEqual(len(response.data), 10) + def test_search_spelling_correction(self): + with AsyncWriter(index.open_index()) as writer: + for i in range(55): + doc = Document.objects.create(checksum=str(i), pk=i+1, title=f"Document {i+1}", content=f"Things document {i+1}") + index.update_document(writer, doc) + + response = self.client.get("/api/search/?query=thing") + correction = response.data['corrected_query'] + + self.assertEqual(correction, "things") + + response = self.client.get("/api/search/?query=things") + correction = response.data['corrected_query'] + + self.assertEqual(correction, None) + def test_statistics(self): doc1 = Document.objects.create(title="none1", checksum="A") diff --git a/src/documents/views.py b/src/documents/views.py index 84f4a3999..0ac232436 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -227,11 +227,13 @@ class SearchView(APIView): if page < 1: page = 1 - with index.query_page(self.ix, query, page) as result_page: + with index.query_page(self.ix, query, page) as (result_page, + corrected_query): return Response( {'count': len(result_page), 'page': result_page.pagenum, 'page_count': result_page.pagecount, + 'corrected_query': corrected_query, 'results': list(map(self.add_infos_to_hit, result_page))}) else: From b03d4c7646d870a261665aefa563062dac8fc246 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 30 Nov 2020 16:13:35 +0100 Subject: [PATCH 07/10] searching for types and dates, error catching, documentation and changelog. --- docs/changelog.rst | 6 +- docs/usage_overview.rst | 56 +++++++++++++++++++ .../components/search/search.component.html | 4 +- .../app/components/search/search.component.ts | 11 +++- src/documents/index.py | 32 +++++++---- src/documents/views.py | 32 ++++++----- 6 files changed, 112 insertions(+), 29 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f326b95ce..806d09fe0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,12 +10,14 @@ paperless-ng 0.9.4 * Searching: - * Paperless now supports searching by tags. In order to have this applied to your + * Paperless now supports searching by tags, types and dates. In order to have this applied to your existing documents, you need to perform a ``document_index reindex`` management command (see :ref:`administration-index`) - that adds tags to your search index. Paperless keeps your index updated after that whenever + that adds tags to your search index. You only need to do this once, so that paperless can find + your documents by tags,types and dates. Paperless keeps your index updated after that whenever something changes. * Paperless now has spelling corrections ("Did you mean") for misstyped queries. + * The documentation contains :ref:`information about the query syntax `. * Front end: diff --git a/docs/usage_overview.rst b/docs/usage_overview.rst index 0e50dafc2..4ce7f9b7a 100644 --- a/docs/usage_overview.rst +++ b/docs/usage_overview.rst @@ -156,6 +156,62 @@ REST API You can also submit a document using the REST API, see :ref:`api-file_uploads` for details. +.. _basic-searching: + +Searching +######### + +Paperless offers an extensive searching mechanism that is designed to allow you to quickly +find a document you're looking for (for example, that thing that just broke and you bought +a couple months ago, that contract you signed 8 years ago). + +When you search paperless for a document, it tries to match this query against your documents. +Paperless will look for matching documents by inspecting their content, title, correspondent, +type and tags. Paperless returns a scored list of results, so that documents matching your query +better will appear further up in the search results. + +By default, paperless returns only documents which contain all words typed in the search bar. +However, paperless also offers advanced search syntax if you want to drill down the results +further. + +Matching documents with logical expressions: + +.. code:: none + + shopname AND (product1 OR product2) + +Matching specific tags, correspondents or types: + +.. code:: none + + type:invoice tag:unpaid + correspondent:university certificate + +Matching dates: + +.. code:: none + + created:[2005 to 2009] + added:yesterday + modified:today + +Matching inexact words: + +.. code:: none + + produ*name + +.. note:: + + Inexact terms are hard for search indexes. These queries might take a while to execute. That's why paperless offers + auto complete and query correction. + +All of these constructs can be combined as you see fit. +If you want to learn more about the query language used by paperless, paperless uses Whoosh's default query language. +Head over to `Whoosh query language `_. +For details on what date parsing utilities are available, see +`Date parsing `_. + .. _usage-recommended_workflow: diff --git a/src-ui/src/app/components/search/search.component.html b/src-ui/src/app/components/search/search.component.html index cb5c1a8e8..55fcee900 100644 --- a/src-ui/src/app/components/search/search.component.html +++ b/src-ui/src/app/components/search/search.component.html @@ -1,6 +1,8 @@ +
Invalid search query: {{errorMessage}}
+

Search string: {{query}} @@ -9,7 +11,7 @@

-
+

{{resultCount}} result(s)

{ if (append) { this.results.push(...result.results) @@ -52,6 +55,12 @@ export class SearchComponent implements OnInit { this.searching = false this.resultCount = result.count this.correctedQuery = result.corrected_query + }, error => { + this.searching = false + this.resultCount = 1 + this.page_count = 1 + this.results = [] + this.errorMessage = error.error }) } diff --git a/src/documents/index.py b/src/documents/index.py index 822ac2e8a..b4d6e1c51 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -4,10 +4,11 @@ from contextlib import contextmanager from django.conf import settings from whoosh import highlight -from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD +from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME from whoosh.highlight import Formatter, get_text from whoosh.index import create_in, exists_in, open_dir from whoosh.qparser import MultifieldParser +from whoosh.qparser.dateparse import DateParserPlugin from whoosh.writing import AsyncWriter @@ -60,7 +61,11 @@ def get_schema(): title=TEXT(stored=True), content=TEXT(), correspondent=TEXT(stored=True), - tag=KEYWORD(stored=True, commas=True, scorable=True, lowercase=True) + tag=KEYWORD(stored=True, commas=True, scorable=True, lowercase=True), + type=TEXT(stored=True), + created=DATETIME(stored=True, sortable=True), + modified=DATETIME(stored=True, sortable=True), + added=DATETIME(stored=True, sortable=True), ) @@ -84,7 +89,11 @@ def update_document(writer, doc): title=doc.title, content=doc.content, correspondent=doc.correspondent.name if doc.correspondent else None, - tag=tags if tags else None + tag=tags if tags else None, + type=doc.document_type.name if doc.document_type else None, + created=doc.created, + added=doc.added, + modified=doc.modified, ) @@ -106,19 +115,22 @@ def remove_document_from_index(document): @contextmanager -def query_page(ix, query, page): +def query_page(ix, querystring, page): searcher = ix.searcher() try: - query_parser = MultifieldParser( - ["content", "title", "correspondent", "tag"], - ix.schema).parse(query) - result_page = searcher.search_page(query_parser, page) + qp = MultifieldParser( + ["content", "title", "correspondent", "tag", "type"], + ix.schema) + qp.add_plugin(DateParserPlugin()) + + q = qp.parse(querystring) + result_page = searcher.search_page(q, page) result_page.results.fragmenter = highlight.ContextFragmenter( surround=50) result_page.results.formatter = JsonFormatter() - corrected = searcher.correct_query(query_parser, query) - if corrected.query != query_parser: + corrected = searcher.correct_query(q, querystring) + if corrected.query != q: corrected_query = corrected.string else: corrected_query = None diff --git a/src/documents/views.py b/src/documents/views.py index 0ac232436..332bdfe8f 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -217,16 +217,23 @@ class SearchView(APIView): } def get(self, request, format=None): - if 'query' in request.query_params: - query = request.query_params['query'] - try: - page = int(request.query_params.get('page', 1)) - except (ValueError, TypeError): - page = 1 + if not 'query' in request.query_params: + return Response({ + 'count': 0, + 'page': 0, + 'page_count': 0, + 'results': []}) - if page < 1: - page = 1 + query = request.query_params['query'] + try: + page = int(request.query_params.get('page', 1)) + except (ValueError, TypeError): + page = 1 + if page < 1: + page = 1 + + try: with index.query_page(self.ix, query, page) as (result_page, corrected_query): return Response( @@ -235,13 +242,8 @@ class SearchView(APIView): 'page_count': result_page.pagecount, 'corrected_query': corrected_query, 'results': list(map(self.add_infos_to_hit, result_page))}) - - else: - return Response({ - 'count': 0, - 'page': 0, - 'page_count': 0, - 'results': []}) + except Exception as e: + return HttpResponseBadRequest(str(e)) class SearchAutoCompleteView(APIView): From f156f05b3769aab18be6907c69f4d8798a4b58ee Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 30 Nov 2020 16:19:32 +0100 Subject: [PATCH 08/10] typo --- src-ui/src/app/components/search/search.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-ui/src/app/components/search/search.component.ts b/src-ui/src/app/components/search/search.component.ts index f3635e31e..de8b4652f 100644 --- a/src-ui/src/app/components/search/search.component.ts +++ b/src-ui/src/app/components/search/search.component.ts @@ -58,7 +58,7 @@ export class SearchComponent implements OnInit { }, error => { this.searching = false this.resultCount = 1 - this.page_count = 1 + this.pageCount = 1 this.results = [] this.errorMessage = error.error }) From e1853583b0ff445aae1793341201990eeabaded8 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 30 Nov 2020 16:25:10 +0100 Subject: [PATCH 09/10] changelog, codestyle --- docs/changelog.rst | 4 ++-- src/documents/views.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 806d09fe0..68198ec49 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,8 +13,8 @@ paperless-ng 0.9.4 * Paperless now supports searching by tags, types and dates. In order to have this applied to your existing documents, you need to perform a ``document_index reindex`` management command (see :ref:`administration-index`) - that adds tags to your search index. You only need to do this once, so that paperless can find - your documents by tags,types and dates. Paperless keeps your index updated after that whenever + that adds the new data to the search index. You only need to do this once, so that paperless can find + your documents by tags,types and dates. Paperless keeps the index updated after that whenever something changes. * Paperless now has spelling corrections ("Did you mean") for misstyped queries. * The documentation contains :ref:`information about the query syntax `. diff --git a/src/documents/views.py b/src/documents/views.py index 332bdfe8f..169350c75 100755 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -217,7 +217,7 @@ class SearchView(APIView): } def get(self, request, format=None): - if not 'query' in request.query_params: + if 'query' not in request.query_params: return Response({ 'count': 0, 'page': 0, From 183d432f8407fbab7e8e02d5eecba759e37be154 Mon Sep 17 00:00:00 2001 From: jonaswinkler Date: Mon, 30 Nov 2020 16:26:20 +0100 Subject: [PATCH 10/10] versions. --- docker/hub/docker-compose.postgres.yml | 2 +- docker/hub/docker-compose.sqlite.yml | 2 +- src/paperless/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml index 5d9e2a7ae..9848b3e05 100644 --- a/docker/hub/docker-compose.postgres.yml +++ b/docker/hub/docker-compose.postgres.yml @@ -15,7 +15,7 @@ services: POSTGRES_PASSWORD: paperless webserver: - image: jonaswinkler/paperless-ng:0.9.3 + image: jonaswinkler/paperless-ng:0.9.4 restart: always depends_on: - db diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml index 95f024061..7331b64ba 100644 --- a/docker/hub/docker-compose.sqlite.yml +++ b/docker/hub/docker-compose.sqlite.yml @@ -5,7 +5,7 @@ services: restart: always webserver: - image: jonaswinkler/paperless-ng:0.9.3 + image: jonaswinkler/paperless-ng:0.9.4 restart: always depends_on: - broker diff --git a/src/paperless/version.py b/src/paperless/version.py index 90680d4b0..23bd5f157 100644 --- a/src/paperless/version.py +++ b/src/paperless/version.py @@ -1 +1 @@ -__version__ = (0, 9, 3) +__version__ = (0, 9, 4)
Created