Merge branch 'dev' into 2312-add-pdf-layout-choice

This commit is contained in:
Trenton H
2025-02-07 09:21:52 -08:00
committed by GitHub
126 changed files with 5504 additions and 4078 deletions

View File

@@ -1,7 +1,6 @@
import datetime
import io
import json
import os
import shutil
import zipfile
@@ -15,9 +14,10 @@ from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.tests.utils import DirectoriesMixin
from documents.tests.utils import SampleDirMixin
class TestBulkDownload(DirectoriesMixin, APITestCase):
class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
ENDPOINT = "/api/documents/bulk_download/"
def setUp(self):
@@ -51,22 +51,10 @@ class TestBulkDownload(DirectoriesMixin, APITestCase):
archive_checksum="D",
)
shutil.copy(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
self.doc2.source_path,
)
shutil.copy(
os.path.join(os.path.dirname(__file__), "samples", "simple.png"),
self.doc2b.source_path,
)
shutil.copy(
os.path.join(os.path.dirname(__file__), "samples", "simple.jpg"),
self.doc3.source_path,
)
shutil.copy(
os.path.join(os.path.dirname(__file__), "samples", "test_with_bom.pdf"),
self.doc3.archive_path,
)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", self.doc2.source_path)
shutil.copy(self.SAMPLE_DIR / "simple.png", self.doc2b.source_path)
shutil.copy(self.SAMPLE_DIR / "simple.jpg", self.doc3.source_path)
shutil.copy(self.SAMPLE_DIR / "test_with_bom.pdf", self.doc3.archive_path)
def test_download_originals(self):
response = self.client.post(

View File

@@ -1,5 +1,4 @@
import datetime
import os
import shutil
import tempfile
import uuid
@@ -8,6 +7,7 @@ from binascii import hexlify
from datetime import date
from datetime import timedelta
from pathlib import Path
from typing import TYPE_CHECKING
from unittest import mock
import celery
@@ -171,19 +171,18 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
content = b"This is a test"
content_thumbnail = b"thumbnail content"
with open(filename, "wb") as f:
with Path(filename).open("wb") as f:
f.write(content)
doc = Document.objects.create(
title="none",
filename=os.path.basename(filename),
filename=Path(filename).name,
mime_type="application/pdf",
)
with open(
os.path.join(self.dirs.thumbnail_dir, f"{doc.pk:07d}.webp"),
"wb",
) as f:
if TYPE_CHECKING:
assert isinstance(self.dirs.thumbnail_dir, Path), self.dirs.thumbnail_dir
with (self.dirs.thumbnail_dir / f"{doc.pk:07d}.webp").open("wb") as f:
f.write(content_thumbnail)
response = self.client.get(f"/api/documents/{doc.pk}/download/")
@@ -217,7 +216,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
content = b"This is a test"
content_thumbnail = b"thumbnail content"
with open(filename, "wb") as f:
with Path(filename).open("wb") as f:
f.write(content)
user1 = User.objects.create_user(username="test1")
@@ -229,15 +228,12 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
doc = Document.objects.create(
title="none",
filename=os.path.basename(filename),
filename=Path(filename).name,
mime_type="application/pdf",
owner=user1,
)
with open(
os.path.join(self.dirs.thumbnail_dir, f"{doc.pk:07d}.webp"),
"wb",
) as f:
with (Path(self.dirs.thumbnail_dir) / f"{doc.pk:07d}.webp").open("wb") as f:
f.write(content_thumbnail)
response = self.client.get(f"/api/documents/{doc.pk}/download/")
@@ -272,10 +268,10 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
mime_type="application/pdf",
)
with open(doc.source_path, "wb") as f:
with Path(doc.source_path).open("wb") as f:
f.write(content)
with open(doc.archive_path, "wb") as f:
with Path(doc.archive_path).open("wb") as f:
f.write(content_archive)
response = self.client.get(f"/api/documents/{doc.pk}/download/")
@@ -305,7 +301,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
def test_document_actions_not_existing_file(self):
doc = Document.objects.create(
title="none",
filename=os.path.basename("asd"),
filename=Path("asd").name,
mime_type="application/pdf",
)
@@ -1026,10 +1022,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f},
@@ -1061,10 +1054,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{
@@ -1095,10 +1085,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"documenst": f},
@@ -1111,10 +1098,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.zip"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.zip").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f},
@@ -1127,10 +1111,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "title": "my custom title"},
@@ -1152,10 +1133,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
)
c = Correspondent.objects.create(name="test-corres")
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "correspondent": c.id},
@@ -1176,10 +1154,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "correspondent": 3456},
@@ -1194,10 +1169,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
)
dt = DocumentType.objects.create(name="invoice")
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "document_type": dt.id},
@@ -1218,10 +1190,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "document_type": 34578},
@@ -1236,10 +1205,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
)
sp = StoragePath.objects.create(name="invoices")
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "storage_path": sp.id},
@@ -1260,10 +1226,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "storage_path": 34578},
@@ -1279,10 +1242,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
t1 = Tag.objects.create(name="tag1")
t2 = Tag.objects.create(name="tag2")
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "tags": [t2.id, t1.id]},
@@ -1305,10 +1265,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
t1 = Tag.objects.create(name="tag1")
t2 = Tag.objects.create(name="tag2")
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "tags": [t2.id, t1.id, 734563]},
@@ -1332,10 +1289,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
0,
tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles"),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "created": created},
@@ -1353,10 +1307,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "archive_serial_number": 500},
@@ -1385,10 +1336,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
data_type=CustomField.FieldDataType.STRING,
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{
@@ -1417,10 +1365,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
id=str(uuid.uuid4()),
)
with open(
os.path.join(os.path.dirname(__file__), "samples", "invalid_pdf.pdf"),
"rb",
) as f:
with (Path(__file__).parent / "samples" / "invalid_pdf.pdf").open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f},
@@ -1437,14 +1382,14 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
archive_filename="archive.pdf",
)
source_file = os.path.join(
os.path.dirname(__file__),
"samples",
"documents",
"thumbnails",
"0000001.webp",
source_file: Path = (
Path(__file__).parent
/ "samples"
/ "documents"
/ "thumbnails"
/ "0000001.webp"
)
archive_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
archive_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
shutil.copy(source_file, doc.source_path)
shutil.copy(archive_file, doc.archive_path)
@@ -1460,8 +1405,8 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
self.assertGreater(len(meta["archive_metadata"]), 0)
self.assertEqual(meta["media_filename"], "file.pdf")
self.assertEqual(meta["archive_media_filename"], "archive.pdf")
self.assertEqual(meta["original_size"], os.stat(source_file).st_size)
self.assertEqual(meta["archive_size"], os.stat(archive_file).st_size)
self.assertEqual(meta["original_size"], Path(source_file).stat().st_size)
self.assertEqual(meta["archive_size"], Path(archive_file).stat().st_size)
response = self.client.get(f"/api/documents/{doc.pk}/metadata/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -1477,10 +1422,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
mime_type="application/pdf",
)
shutil.copy(
os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"),
doc.source_path,
)
shutil.copy(Path(__file__).parent / "samples" / "simple.pdf", doc.source_path)
response = self.client.get(f"/api/documents/{doc.pk}/metadata/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -1939,9 +1881,9 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
def test_get_logs(self):
log_data = "test\ntest2\n"
with open(os.path.join(settings.LOGGING_DIR, "mail.log"), "w") as f:
with (Path(settings.LOGGING_DIR) / "mail.log").open("w") as f:
f.write(log_data)
with open(os.path.join(settings.LOGGING_DIR, "paperless.log"), "w") as f:
with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f:
f.write(log_data)
response = self.client.get("/api/logs/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -1949,7 +1891,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
def test_get_logs_only_when_exist(self):
log_data = "test\ntest2\n"
with open(os.path.join(settings.LOGGING_DIR, "paperless.log"), "w") as f:
with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f:
f.write(log_data)
response = self.client.get("/api/logs/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -1966,7 +1908,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
def test_get_log(self):
log_data = "test\ntest2\n"
with open(os.path.join(settings.LOGGING_DIR, "paperless.log"), "w") as f:
with (Path(settings.LOGGING_DIR) / "paperless.log").open("w") as f:
f.write(log_data)
response = self.client.get("/api/logs/paperless/")
self.assertEqual(response.status_code, status.HTTP_200_OK)

View File

@@ -165,6 +165,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
self,
query: list,
reference_predicate: Callable[[DocumentWrapper], bool],
*,
match_nothing_ok=False,
):
"""

View File

@@ -3,6 +3,7 @@ import json
from unittest import mock
from allauth.mfa.models import Authenticator
from allauth.mfa.totp.internal import auth as totp_auth
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
@@ -488,6 +489,71 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
self.assertEqual(response.data["detail"], "MFA required")
@mock.patch("allauth.mfa.totp.internal.auth.TOTP.validate_code")
def test_get_token_mfa_enabled(self, mock_validate_code):
"""
GIVEN:
- User with MFA enabled
WHEN:
- API request is made to obtain an auth token
THEN:
- MFA code is required
"""
user1 = User.objects.create_user(username="user1")
user1.set_password("password")
user1.save()
response = self.client.post(
"/api/token/",
data={
"username": "user1",
"password": "password",
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
secret = totp_auth.generate_totp_secret()
totp_auth.TOTP.activate(
user1,
secret,
)
# no code
response = self.client.post(
"/api/token/",
data={
"username": "user1",
"password": "password",
},
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.data["non_field_errors"][0], "MFA code is required")
# invalid code
mock_validate_code.return_value = False
response = self.client.post(
"/api/token/",
data={
"username": "user1",
"password": "password",
"code": "123456",
},
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.data["non_field_errors"][0], "Invalid MFA code")
# valid code
mock_validate_code.return_value = True
response = self.client.post(
"/api/token/",
data={
"username": "user1",
"password": "password",
"code": "123456",
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
class TestApiUser(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/users/"

View File

@@ -268,7 +268,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
cf3 = CustomField.objects.create(
name="cf3",
data_type=CustomField.FieldDataType.STRING,
data_type=CustomField.FieldDataType.DOCUMENTLINK,
)
CustomFieldInstance.objects.create(
document=self.doc2,
@@ -284,7 +284,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
bulk_edit.modify_custom_fields(
[self.doc1.id, self.doc2.id],
add_custom_fields={cf2.id: None, cf3.id: "value"},
add_custom_fields={cf2.id: None, cf3.id: [self.doc3.id]},
remove_custom_fields=[cf.id],
)
@@ -301,7 +301,7 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.assertEqual(
self.doc1.custom_fields.get(field=cf3).value,
"value",
[self.doc3.id],
)
self.assertEqual(
self.doc2.custom_fields.count(),
@@ -309,13 +309,33 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
)
self.assertEqual(
self.doc2.custom_fields.get(field=cf3).value,
"value",
[self.doc3.id],
)
# assert reflect document link
self.assertEqual(
self.doc3.custom_fields.first().value,
[self.doc2.id, self.doc1.id],
)
self.async_task.assert_called_once()
args, kwargs = self.async_task.call_args
self.assertCountEqual(kwargs["document_ids"], [self.doc1.id, self.doc2.id])
# removal of document link cf, should also remove symmetric link
bulk_edit.modify_custom_fields(
[self.doc3.id],
add_custom_fields={},
remove_custom_fields=[cf3.id],
)
self.assertNotIn(
self.doc3.id,
self.doc1.custom_fields.filter(field=cf3).first().value,
)
self.assertNotIn(
self.doc3.id,
self.doc2.custom_fields.filter(field=cf3).first().value,
)
def test_delete(self):
self.assertEqual(Document.objects.count(), 5)
bulk_edit.delete([self.doc1.id, self.doc2.id])
@@ -515,7 +535,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
metadata_document_id = self.doc1.id
user = User.objects.create(username="test_user")
result = bulk_edit.merge(doc_ids, None, False, user)
result = bulk_edit.merge(
doc_ids,
metadata_document_id=None,
delete_originals=False,
user=user,
)
expected_filename = (
f"{'_'.join([str(doc_id) for doc_id in doc_ids])[:100]}_merged.pdf"
@@ -618,7 +643,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
doc_ids = [self.doc2.id]
pages = [[1, 2], [3]]
user = User.objects.create(username="test_user")
result = bulk_edit.split(doc_ids, pages, False, user)
result = bulk_edit.split(doc_ids, pages, delete_originals=False, user=user)
self.assertEqual(mock_consume_file.call_count, 2)
consume_file_args, _ = mock_consume_file.call_args
self.assertEqual(consume_file_args[1].title, "B (split 2)")

View File

@@ -236,7 +236,7 @@ class FaultyGenericExceptionParser(_BaseTestParser):
raise Exception("Generic exception.")
def fake_magic_from_file(file, mime=False):
def fake_magic_from_file(file, *, mime=False):
if mime:
if file.name.startswith("invalid_pdf"):
return "application/octet-stream"

View File

@@ -10,7 +10,7 @@ class TestDelayedQuery(TestCase):
super().setUp()
# all tests run without permission criteria, so has_no_owner query will always
# be appended.
self.has_no_owner = query.Or([query.Term("has_owner", False)])
self.has_no_owner = query.Or([query.Term("has_owner", text=False)])
def _get_testset__id__in(self, param, field):
return (
@@ -43,12 +43,12 @@ class TestDelayedQuery(TestCase):
def test_get_permission_criteria(self):
# tests contains tuples of user instances and the expected filter
tests = (
(None, [query.Term("has_owner", False)]),
(None, [query.Term("has_owner", text=False)]),
(User(42, username="foo", is_superuser=True), []),
(
User(42, username="foo", is_superuser=False),
[
query.Term("has_owner", False),
query.Term("has_owner", text=False),
query.Term("owner_id", 42),
query.Term("viewer_id", "42"),
],

View File

@@ -93,7 +93,7 @@ class ConsumerThreadMixin(DocumentConsumeDelayMixin):
else:
print("Consumed a perfectly valid file.") # noqa: T201
def slow_write_file(self, target, incomplete=False):
def slow_write_file(self, target, *, incomplete=False):
with open(self.sample_file, "rb") as f:
pdf_bytes = f.read()

View File

@@ -188,7 +188,7 @@ class TestExportImport(
return manifest
def test_exporter(self, use_filename_format=False):
def test_exporter(self, *, use_filename_format=False):
shutil.rmtree(os.path.join(self.dirs.media_dir, "documents"))
shutil.copytree(
os.path.join(os.path.dirname(__file__), "samples", "documents"),

View File

@@ -23,6 +23,7 @@ class _TestMatchingBase(TestCase):
match_algorithm: str,
should_match: Iterable[str],
no_match: Iterable[str],
*,
case_sensitive: bool = False,
):
for klass in (Tag, Correspondent, DocumentType):