mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-01-26 22:49:01 -06:00
Compare commits
5 Commits
chore/pyte
...
feature-au
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
496a4035cd | ||
|
|
761044c0d3 | ||
|
|
1b7e4cc286 | ||
|
|
6997a2ab8b | ||
|
|
f82f31f383 |
@@ -3444,7 +3444,7 @@
|
|||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">111</context>
|
<context context-type="linenumber">113</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/input/date/date.component.html</context>
|
<context context-type="sourcefile">src/app/components/common/input/date/date.component.html</context>
|
||||||
@@ -3704,14 +3704,14 @@
|
|||||||
<source>This month</source>
|
<source>This month</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">106</context>
|
<context context-type="linenumber">107</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="4498682414491138092" datatype="html">
|
<trans-unit id="4498682414491138092" datatype="html">
|
||||||
<source>Yesterday</source>
|
<source>Yesterday</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">116</context>
|
<context context-type="linenumber">118</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/pipes/custom-date.pipe.ts</context>
|
<context context-type="sourcefile">src/app/pipes/custom-date.pipe.ts</context>
|
||||||
@@ -3722,28 +3722,28 @@
|
|||||||
<source>Previous week</source>
|
<source>Previous week</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">121</context>
|
<context context-type="linenumber">123</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="8586908745456864217" datatype="html">
|
<trans-unit id="8586908745456864217" datatype="html">
|
||||||
<source>Previous month</source>
|
<source>Previous month</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">135</context>
|
<context context-type="linenumber">137</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="357608474534295480" datatype="html">
|
<trans-unit id="357608474534295480" datatype="html">
|
||||||
<source>Previous quarter</source>
|
<source>Previous quarter</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">141</context>
|
<context context-type="linenumber">143</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="100513227838842152" datatype="html">
|
<trans-unit id="100513227838842152" datatype="html">
|
||||||
<source>Previous year</source>
|
<source>Previous year</source>
|
||||||
<context-group purpose="location">
|
<context-group purpose="location">
|
||||||
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
<context context-type="sourcefile">src/app/components/common/dates-dropdown/dates-dropdown.component.ts</context>
|
||||||
<context context-type="linenumber">155</context>
|
<context context-type="linenumber">157</context>
|
||||||
</context-group>
|
</context-group>
|
||||||
</trans-unit>
|
</trans-unit>
|
||||||
<trans-unit id="8743659855412792665" datatype="html">
|
<trans-unit id="8743659855412792665" datatype="html">
|
||||||
|
|||||||
@@ -164,9 +164,11 @@
|
|||||||
{{ item.name }}
|
{{ item.name }}
|
||||||
<span class="ms-auto text-muted small">
|
<span class="ms-auto text-muted small">
|
||||||
@if (item.dateEnd) {
|
@if (item.dateEnd) {
|
||||||
{{ item.date | customDate:'MMM d' }} – {{ item.dateEnd | customDate:'mediumDate' }}
|
{{ item.date | customDate:'mediumDate' }} – {{ item.dateEnd | customDate:'mediumDate' }}
|
||||||
|
} @else if (item.dateTilNow) {
|
||||||
|
{{ item.dateTilNow | customDate:'mediumDate' }} – <ng-container i18n>now</ng-container>
|
||||||
} @else {
|
} @else {
|
||||||
{{ item.date | customDate:'mediumDate' }} – <ng-container i18n>now</ng-container>
|
{{ item.date | customDate:'mediumDate' }}
|
||||||
}
|
}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -79,32 +79,34 @@ export class DatesDropdownComponent implements OnInit, OnDestroy {
|
|||||||
{
|
{
|
||||||
id: RelativeDate.WITHIN_1_WEEK,
|
id: RelativeDate.WITHIN_1_WEEK,
|
||||||
name: $localize`Within 1 week`,
|
name: $localize`Within 1 week`,
|
||||||
date: new Date().setDate(new Date().getDate() - 7),
|
dateTilNow: new Date().setDate(new Date().getDate() - 7),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: RelativeDate.WITHIN_1_MONTH,
|
id: RelativeDate.WITHIN_1_MONTH,
|
||||||
name: $localize`Within 1 month`,
|
name: $localize`Within 1 month`,
|
||||||
date: new Date().setMonth(new Date().getMonth() - 1),
|
dateTilNow: new Date().setMonth(new Date().getMonth() - 1),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: RelativeDate.WITHIN_3_MONTHS,
|
id: RelativeDate.WITHIN_3_MONTHS,
|
||||||
name: $localize`Within 3 months`,
|
name: $localize`Within 3 months`,
|
||||||
date: new Date().setMonth(new Date().getMonth() - 3),
|
dateTilNow: new Date().setMonth(new Date().getMonth() - 3),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: RelativeDate.WITHIN_1_YEAR,
|
id: RelativeDate.WITHIN_1_YEAR,
|
||||||
name: $localize`Within 1 year`,
|
name: $localize`Within 1 year`,
|
||||||
date: new Date().setFullYear(new Date().getFullYear() - 1),
|
dateTilNow: new Date().setFullYear(new Date().getFullYear() - 1),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: RelativeDate.THIS_YEAR,
|
id: RelativeDate.THIS_YEAR,
|
||||||
name: $localize`This year`,
|
name: $localize`This year`,
|
||||||
date: new Date('1/1/' + new Date().getFullYear()),
|
date: new Date('1/1/' + new Date().getFullYear()),
|
||||||
|
dateEnd: new Date('12/31/' + new Date().getFullYear()),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: RelativeDate.THIS_MONTH,
|
id: RelativeDate.THIS_MONTH,
|
||||||
name: $localize`This month`,
|
name: $localize`This month`,
|
||||||
date: new Date().setDate(1),
|
date: new Date().setDate(1),
|
||||||
|
dateEnd: new Date(new Date().getFullYear(), new Date().getMonth() + 1, 0),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: RelativeDate.TODAY,
|
id: RelativeDate.TODAY,
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
import logging
|
import logging
|
||||||
import shutil
|
import shutil
|
||||||
|
from datetime import timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import faiss
|
import faiss
|
||||||
import llama_index.core.settings as llama_settings
|
import llama_index.core.settings as llama_settings
|
||||||
import tqdm
|
import tqdm
|
||||||
|
from celery import states
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
from django.utils import timezone
|
||||||
from llama_index.core import Document as LlamaDocument
|
from llama_index.core import Document as LlamaDocument
|
||||||
from llama_index.core import StorageContext
|
from llama_index.core import StorageContext
|
||||||
from llama_index.core import VectorStoreIndex
|
from llama_index.core import VectorStoreIndex
|
||||||
@@ -21,6 +24,7 @@ from llama_index.core.text_splitter import TokenTextSplitter
|
|||||||
from llama_index.vector_stores.faiss import FaissVectorStore
|
from llama_index.vector_stores.faiss import FaissVectorStore
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
from documents.models import PaperlessTask
|
||||||
from paperless_ai.embedding import build_llm_index_text
|
from paperless_ai.embedding import build_llm_index_text
|
||||||
from paperless_ai.embedding import get_embedding_dim
|
from paperless_ai.embedding import get_embedding_dim
|
||||||
from paperless_ai.embedding import get_embedding_model
|
from paperless_ai.embedding import get_embedding_model
|
||||||
@@ -28,6 +32,29 @@ from paperless_ai.embedding import get_embedding_model
|
|||||||
logger = logging.getLogger("paperless_ai.indexing")
|
logger = logging.getLogger("paperless_ai.indexing")
|
||||||
|
|
||||||
|
|
||||||
|
def queue_llm_index_update_if_needed(*, rebuild: bool, reason: str) -> bool:
|
||||||
|
from documents.tasks import llmindex_index
|
||||||
|
|
||||||
|
has_running = PaperlessTask.objects.filter(
|
||||||
|
task_name=PaperlessTask.TaskName.LLMINDEX_UPDATE,
|
||||||
|
status__in=[states.PENDING, states.STARTED],
|
||||||
|
).exists()
|
||||||
|
has_recent = PaperlessTask.objects.filter(
|
||||||
|
task_name=PaperlessTask.TaskName.LLMINDEX_UPDATE,
|
||||||
|
date_created__gte=(timezone.now() - timedelta(minutes=5)),
|
||||||
|
).exists()
|
||||||
|
if has_running or has_recent:
|
||||||
|
return False
|
||||||
|
|
||||||
|
llmindex_index.delay(rebuild=rebuild, scheduled=False, auto=True)
|
||||||
|
logger.warning(
|
||||||
|
"Queued LLM index update%s: %s",
|
||||||
|
" (rebuild)" if rebuild else "",
|
||||||
|
reason,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def get_or_create_storage_context(*, rebuild=False):
|
def get_or_create_storage_context(*, rebuild=False):
|
||||||
"""
|
"""
|
||||||
Loads or creates the StorageContext (vector store, docstore, index store).
|
Loads or creates the StorageContext (vector store, docstore, index store).
|
||||||
@@ -93,6 +120,10 @@ def load_or_build_index(nodes=None):
|
|||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logger.warning("Failed to load index from storage: %s", e)
|
logger.warning("Failed to load index from storage: %s", e)
|
||||||
if not nodes:
|
if not nodes:
|
||||||
|
queue_llm_index_update_if_needed(
|
||||||
|
rebuild=vector_store_file_exists(),
|
||||||
|
reason="LLM index missing or invalid while loading.",
|
||||||
|
)
|
||||||
logger.info("No nodes provided for index creation.")
|
logger.info("No nodes provided for index creation.")
|
||||||
raise
|
raise
|
||||||
return VectorStoreIndex(
|
return VectorStoreIndex(
|
||||||
@@ -250,6 +281,13 @@ def query_similar_documents(
|
|||||||
"""
|
"""
|
||||||
Runs a similarity query and returns top-k similar Document objects.
|
Runs a similarity query and returns top-k similar Document objects.
|
||||||
"""
|
"""
|
||||||
|
if not vector_store_file_exists():
|
||||||
|
queue_llm_index_update_if_needed(
|
||||||
|
rebuild=False,
|
||||||
|
reason="LLM index not found for similarity query.",
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
index = load_or_build_index()
|
index = load_or_build_index()
|
||||||
|
|
||||||
# constrain only the node(s) that match the document IDs, if given
|
# constrain only the node(s) that match the document IDs, if given
|
||||||
|
|||||||
@@ -3,11 +3,13 @@ from unittest.mock import MagicMock
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from celery import states
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from llama_index.core.base.embeddings.base import BaseEmbedding
|
from llama_index.core.base.embeddings.base import BaseEmbedding
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
from documents.models import PaperlessTask
|
||||||
from paperless_ai import indexing
|
from paperless_ai import indexing
|
||||||
|
|
||||||
|
|
||||||
@@ -288,6 +290,36 @@ def test_update_llm_index_no_documents(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_queue_llm_index_update_if_needed_enqueues_when_idle_or_skips_recent():
|
||||||
|
# No existing tasks
|
||||||
|
with patch("documents.tasks.llmindex_index") as mock_task:
|
||||||
|
result = indexing.queue_llm_index_update_if_needed(
|
||||||
|
rebuild=True,
|
||||||
|
reason="test enqueue",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
mock_task.delay.assert_called_once_with(rebuild=True, scheduled=False, auto=True)
|
||||||
|
|
||||||
|
PaperlessTask.objects.create(
|
||||||
|
task_id="task-1",
|
||||||
|
task_name=PaperlessTask.TaskName.LLMINDEX_UPDATE,
|
||||||
|
status=states.STARTED,
|
||||||
|
date_created=timezone.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Existing running task
|
||||||
|
with patch("documents.tasks.llmindex_index") as mock_task:
|
||||||
|
result = indexing.queue_llm_index_update_if_needed(
|
||||||
|
rebuild=False,
|
||||||
|
reason="should skip",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
mock_task.delay.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@override_settings(
|
@override_settings(
|
||||||
LLM_EMBEDDING_BACKEND="huggingface",
|
LLM_EMBEDDING_BACKEND="huggingface",
|
||||||
LLM_BACKEND="ollama",
|
LLM_BACKEND="ollama",
|
||||||
@@ -299,11 +331,15 @@ def test_query_similar_documents(
|
|||||||
with (
|
with (
|
||||||
patch("paperless_ai.indexing.get_or_create_storage_context") as mock_storage,
|
patch("paperless_ai.indexing.get_or_create_storage_context") as mock_storage,
|
||||||
patch("paperless_ai.indexing.load_or_build_index") as mock_load_or_build_index,
|
patch("paperless_ai.indexing.load_or_build_index") as mock_load_or_build_index,
|
||||||
|
patch(
|
||||||
|
"paperless_ai.indexing.vector_store_file_exists",
|
||||||
|
) as mock_vector_store_exists,
|
||||||
patch("paperless_ai.indexing.VectorIndexRetriever") as mock_retriever_cls,
|
patch("paperless_ai.indexing.VectorIndexRetriever") as mock_retriever_cls,
|
||||||
patch("paperless_ai.indexing.Document.objects.filter") as mock_filter,
|
patch("paperless_ai.indexing.Document.objects.filter") as mock_filter,
|
||||||
):
|
):
|
||||||
mock_storage.return_value = MagicMock()
|
mock_storage.return_value = MagicMock()
|
||||||
mock_storage.return_value.persist_dir = temp_llm_index_dir
|
mock_storage.return_value.persist_dir = temp_llm_index_dir
|
||||||
|
mock_vector_store_exists.return_value = True
|
||||||
|
|
||||||
mock_index = MagicMock()
|
mock_index = MagicMock()
|
||||||
mock_load_or_build_index.return_value = mock_index
|
mock_load_or_build_index.return_value = mock_index
|
||||||
@@ -332,3 +368,31 @@ def test_query_similar_documents(
|
|||||||
mock_filter.assert_called_once_with(pk__in=[1, 2])
|
mock_filter.assert_called_once_with(pk__in=[1, 2])
|
||||||
|
|
||||||
assert result == mock_filtered_docs
|
assert result == mock_filtered_docs
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_query_similar_documents_triggers_update_when_index_missing(
|
||||||
|
temp_llm_index_dir,
|
||||||
|
real_document,
|
||||||
|
):
|
||||||
|
with (
|
||||||
|
patch(
|
||||||
|
"paperless_ai.indexing.vector_store_file_exists",
|
||||||
|
return_value=False,
|
||||||
|
),
|
||||||
|
patch(
|
||||||
|
"paperless_ai.indexing.queue_llm_index_update_if_needed",
|
||||||
|
) as mock_queue,
|
||||||
|
patch("paperless_ai.indexing.load_or_build_index") as mock_load,
|
||||||
|
):
|
||||||
|
result = indexing.query_similar_documents(
|
||||||
|
real_document,
|
||||||
|
top_k=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_queue.assert_called_once_with(
|
||||||
|
rebuild=False,
|
||||||
|
reason="LLM index not found for similarity query.",
|
||||||
|
)
|
||||||
|
mock_load.assert_not_called()
|
||||||
|
assert result == []
|
||||||
|
|||||||
Reference in New Issue
Block a user