paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Chore: Initial conversion to pytest fixtures (#7110)

Browse Source
This commit is contained in:
Trenton H
2024-07-08 07:46:20 -07:00
committed by GitHub
parent 1b9cf5121b
commit 3cf73a77ac
17 changed files with 1051 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/paperless_mail/parsers.py
View File

@@ -52,7 +52,12 @@ class MailDocumentParser(DocumentParser):
return PdfAFormat.A3b
return None
def get_thumbnail(self, document_path: Path, mime_type: str, file_name=None):
def get_thumbnail(
self,
document_path: Path,
mime_type: str,
file_name=None,
) -> Path:
if not self.archive_path:
self.archive_path = self.generate_pdf(
self.parse_file_to_message(document_path),

89
src/paperless_mail/tests/conftest.py Normal file
View File

@@ -0,0 +1,89 @@
import os
from collections.abc import Generator
from pathlib import Path
import pytest
from paperless_mail.mail import MailAccountHandler
from paperless_mail.models import MailAccount
from paperless_mail.parsers import MailDocumentParser
@pytest.fixture(scope="session")
def sample_dir() -> Path:
return (Path(__file__).parent / Path("samples")).resolve()
@pytest.fixture(scope="session")
def broken_email_file(sample_dir: Path) -> Path:
return sample_dir / "broken.eml"
@pytest.fixture(scope="session")
def simple_txt_email_file(sample_dir: Path) -> Path:
return sample_dir / "simple_text.eml"
@pytest.fixture(scope="session")
def simple_txt_email_pdf_file(sample_dir: Path) -> Path:
return sample_dir / "simple_text.eml.pdf"
@pytest.fixture(scope="session")
def simple_txt_email_thumbnail_file(sample_dir: Path) -> Path:
return sample_dir / "simple_text.eml.pdf.webp"
@pytest.fixture(scope="session")
def html_email_file(sample_dir: Path) -> Path:
return sample_dir / "html.eml"
@pytest.fixture(scope="session")
def html_email_pdf_file(sample_dir: Path) -> Path:
return sample_dir / "html.eml.pdf"
@pytest.fixture(scope="session")
def html_email_thumbnail_file(sample_dir: Path) -> Path:
return sample_dir / "html.eml.pdf.webp"
@pytest.fixture(scope="session")
def html_email_html_file(sample_dir: Path) -> Path:
return sample_dir / "html.eml.html"
@pytest.fixture(scope="session")
def merged_pdf_first(sample_dir: Path) -> Path:
return sample_dir / "first.pdf"
@pytest.fixture(scope="session")
def merged_pdf_second(sample_dir: Path) -> Path:
return sample_dir / "second.pdf"
@pytest.fixture()
def mail_parser() -> MailDocumentParser:
return MailDocumentParser(logging_group=None)
@pytest.fixture()
def live_mail_account() -> Generator[MailAccount, None, None]:
try:
account = MailAccount.objects.create(
name="test",
imap_server=os.environ["PAPERLESS_MAIL_TEST_HOST"],
username=os.environ["PAPERLESS_MAIL_TEST_USER"],
password=os.environ["PAPERLESS_MAIL_TEST_PASSWD"],
imap_port=993,
)
yield account
finally:
account.delete()
@pytest.fixture()
def mail_account_handler() -> MailAccountHandler:
return MailAccountHandler()

53
src/paperless_mail/tests/test_live_mail.py
View File

@@ -1,7 +1,7 @@
import os
import warnings
import pytest
from django.test import TestCase
from paperless_mail.mail import MailAccountHandler
from paperless_mail.mail import MailError
@@ -16,53 +16,46 @@ from paperless_mail.models import MailRule
or not len(os.environ["PAPERLESS_MAIL_TEST_HOST"]),
reason="Live server testing not enabled",
)
class TestMailLiveServer(TestCase):
def setUp(self) -> None:
self.mail_account_handler = MailAccountHandler()
self.account = MailAccount.objects.create(
name="test",
imap_server=os.environ["PAPERLESS_MAIL_TEST_HOST"],
username=os.environ["PAPERLESS_MAIL_TEST_USER"],
password=os.environ["PAPERLESS_MAIL_TEST_PASSWD"],
imap_port=993,
)
return super().setUp()
def tearDown(self) -> None:
self.account.delete()
return super().tearDown()
def test_process_non_gmail_server_flag(self):
@pytest.mark.django_db()
class TestMailLiveServer:
def test_process_non_gmail_server_flag(
self,
mail_account_handler: MailAccountHandler,
live_mail_account: MailAccount,
):
try:
rule1 = MailRule.objects.create(
name="testrule",
account=self.account,
account=live_mail_account,
action=MailRule.MailAction.FLAG,
)
self.mail_account_handler.handle_mail_account(self.account)
mail_account_handler.handle_mail_account(live_mail_account)
rule1.delete()
except MailError as e:
self.fail(f"Failure: {e}")
except Exception:
pass
pytest.fail(f"Failure: {e}")
except Exception as e:
warnings.warn(f"Unhandled exception: {e}")
def test_process_non_gmail_server_tag(self):
def test_process_non_gmail_server_tag(
self,
mail_account_handler: MailAccountHandler,
live_mail_account: MailAccount,
):
try:
rule2 = MailRule.objects.create(
name="testrule",
account=self.account,
account=live_mail_account,
action=MailRule.MailAction.TAG,
)
self.mail_account_handler.handle_mail_account(self.account)
mail_account_handler.handle_mail_account(live_mail_account)
rule2.delete()
except MailError as e:
self.fail(f"Failure: {e}")
except Exception:
pass
pytest.fail(f"Failure: {e}")
except Exception as e:
warnings.warn(f"Unhandled exception: {e}")

611
src/paperless_mail/tests/test_parsers.py
View File

@@ -1,39 +1,29 @@
import datetime
import logging
from pathlib import Path
from unittest import mock
import httpx
from django.test import TestCase
import pytest
from django.test.html import parse_html
from pytest_django.fixtures import SettingsWrapper
from pytest_httpx import HTTPXMock
from pytest_mock import MockerFixture
from documents.parsers import ParseError
from documents.tests.utils import FileSystemAssertsMixin
from paperless_mail.parsers import MailDocumentParser
from paperless_tika.tests.utils import HttpxMockMixin
class BaseMailParserTestCase(TestCase):
"""
Basic setup for the below test cases
"""
SAMPLE_DIR = Path(__file__).parent / "samples"
def setUp(self) -> None:
super().setUp()
self.parser = MailDocumentParser(logging_group=None)
def tearDown(self) -> None:
super().tearDown()
self.parser.cleanup()
class TestEmailFileParsing(FileSystemAssertsMixin, BaseMailParserTestCase):
class TestEmailFileParsing:
"""
Tests around reading a file and parsing it into a
MailMessage
"""
def test_parse_error_missing_file(self):
def test_parse_error_missing_file(
self,
mail_parser: MailDocumentParser,
sample_dir: Path,
):
"""
GIVEN:
- Fresh parser
@@ -43,17 +33,18 @@ class TestEmailFileParsing(FileSystemAssertsMixin, BaseMailParserTestCase):
- An Exception is thrown
"""
# Check if exception is raised when parsing fails.
test_file = self.SAMPLE_DIR / "doesntexist.eml"
test_file = sample_dir / "doesntexist.eml"
self.assertIsNotFile(test_file)
self.assertRaises(
ParseError,
self.parser.parse,
test_file,
"messages/rfc822",
)
assert not test_file.exists()
def test_parse_error_invalid_email(self):
with pytest.raises(ParseError):
mail_parser.parse(test_file, "messages/rfc822")
def test_parse_error_invalid_email(
self,
mail_parser: MailDocumentParser,
broken_email_file: Path,
):
"""
GIVEN:
- Fresh parser
@@ -63,14 +54,15 @@ class TestEmailFileParsing(FileSystemAssertsMixin, BaseMailParserTestCase):
- An Exception is thrown
"""
# Check if exception is raised when the mail is faulty.
self.assertRaises(
ParseError,
self.parser.parse,
self.SAMPLE_DIR / "broken.eml",
"messages/rfc822",
)
def test_parse_simple_text_email_file(self):
with pytest.raises(ParseError):
mail_parser.parse(broken_email_file, "messages/rfc822")
def test_parse_simple_text_email_file(
self,
mail_parser: MailDocumentParser,
simple_txt_email_file: Path,
):
"""
GIVEN:
- Fresh parser
@@ -80,29 +72,31 @@ class TestEmailFileParsing(FileSystemAssertsMixin, BaseMailParserTestCase):
- The content of the mail should be available in the parse result.
"""
# Parse Test file and check relevant content
parsed1 = self.parser.parse_file_to_message(
self.SAMPLE_DIR / "simple_text.eml",
)
parsed_msg = mail_parser.parse_file_to_message(simple_txt_email_file)
self.assertEqual(parsed1.date.year, 2022)
self.assertEqual(parsed1.date.month, 10)
self.assertEqual(parsed1.date.day, 12)
self.assertEqual(parsed1.date.hour, 21)
self.assertEqual(parsed1.date.minute, 40)
self.assertEqual(parsed1.date.second, 43)
self.assertEqual(parsed1.date.tzname(), "UTC+02:00")
self.assertEqual(parsed1.from_, "mail@someserver.de")
self.assertEqual(parsed1.subject, "Simple Text Mail")
self.assertEqual(parsed1.text, "This is just a simple Text Mail.\n")
self.assertEqual(parsed1.to, ("some@one.de",))
assert parsed_msg.date.year == 2022
assert parsed_msg.date.month == 10
assert parsed_msg.date.day == 12
assert parsed_msg.date.hour == 21
assert parsed_msg.date.minute == 40
assert parsed_msg.date.second == 43
assert parsed_msg.date.tzname() == "UTC+02:00"
assert parsed_msg.from_ == "mail@someserver.de"
assert parsed_msg.subject == "Simple Text Mail"
assert parsed_msg.text == "This is just a simple Text Mail.\n"
assert parsed_msg.to == ("some@one.de",)
class TestEmailMetadataExtraction(BaseMailParserTestCase):
class TestEmailMetadataExtraction:
"""
Tests extraction of metadata from an email
"""
def test_extract_metadata_fail(self):
def test_extract_metadata_fail(
self,
caplog: pytest.LogCaptureFixture,
mail_parser: MailDocumentParser,
):
"""
GIVEN:
- Fresh start
@@ -112,14 +106,20 @@ class TestEmailMetadataExtraction(BaseMailParserTestCase):
- A log warning should be generated
"""
# Validate if warning is logged when parsing fails
with self.assertLogs("paperless.parsing.mail", level="WARNING") as cm:
self.assertEqual([], self.parser.extract_metadata("na", "message/rfc822"))
self.assertIn(
"WARNING:paperless.parsing.mail:Error while fetching document metadata for na",
cm.output[0],
)
assert mail_parser.extract_metadata("na", "message/rfc822") == []
def test_extract_metadata(self):
assert len(caplog.records) == 1
record = caplog.records[0]
assert record.levelno == logging.WARNING
assert record.name == "paperless.parsing.mail"
assert "Error while fetching document metadata for na" in record.message
def test_extract_metadata(
self,
mail_parser: MailDocumentParser,
simple_txt_email_file: Path,
):
"""
GIVEN:
- Fresh start
@@ -129,149 +129,110 @@ class TestEmailMetadataExtraction(BaseMailParserTestCase):
- metadata is returned
"""
# Validate Metadata parsing returns the expected results
metadata = self.parser.extract_metadata(
self.SAMPLE_DIR / "simple_text.eml",
"message/rfc822",
)
metadata = mail_parser.extract_metadata(simple_txt_email_file, "message/rfc822")
self.assertIn(
{"namespace": "", "prefix": "", "key": "attachments", "value": ""},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "",
"key": "date",
"value": "2022-10-12 21:40:43 UTC+02:00",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "content-language",
"value": "en-US",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "content-type",
"value": "text/plain; charset=UTF-8; format=flowed",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "date",
"value": "Wed, 12 Oct 2022 21:40:43 +0200",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "delivered-to",
"value": "mail@someserver.de",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "from",
"value": "Some One <mail@someserver.de>",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "message-id",
"value": "<6e99e34d-e20a-80c4-ea61-d8234b612be9@someserver.de>",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "mime-version",
"value": "1.0",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "received",
"value": "from mail.someserver.org ([::1])\n\tby e1acdba3bd07 with LMTP\n\tid KBKZGD2YR2NTCgQAjubtDA\n\t(envelope-from <mail@someserver.de>)\n\tfor <mail@someserver.de>; Wed, 10 Oct 2022 11:40:46 +0200, from [127.0.0.1] (localhost [127.0.0.1]) by localhost (Mailerdaemon) with ESMTPSA id 2BC9064C1616\n\tfor <some@one.de>; Wed, 12 Oct 2022 21:40:46 +0200 (CEST)",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "return-path",
"value": "<mail@someserver.de>",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "subject",
"value": "Simple Text Mail",
},
metadata,
)
self.assertIn(
{"namespace": "", "prefix": "header", "key": "to", "value": "some@one.de"},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "user-agent",
"value": "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n Thunderbird/102.3.1",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "x-last-tls-session-version",
"value": "TLSv1.3",
},
metadata,
)
assert {
"namespace": "",
"prefix": "",
"key": "attachments",
"value": "",
} in metadata
assert {
"namespace": "",
"prefix": "",
"key": "date",
"value": "2022-10-12 21:40:43 UTC+02:00",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "content-language",
"value": "en-US",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "content-type",
"value": "text/plain; charset=UTF-8; format=flowed",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "date",
"value": "Wed, 12 Oct 2022 21:40:43 +0200",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "delivered-to",
"value": "mail@someserver.de",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "from",
"value": "Some One <mail@someserver.de>",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "message-id",
"value": "<6e99e34d-e20a-80c4-ea61-d8234b612be9@someserver.de>",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "mime-version",
"value": "1.0",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "received",
"value": "from mail.someserver.org ([::1])\n\tby e1acdba3bd07 with LMTP\n\tid KBKZGD2YR2NTCgQAjubtDA\n\t(envelope-from <mail@someserver.de>)\n\tfor <mail@someserver.de>; Wed, 10 Oct 2022 11:40:46 +0200, from [127.0.0.1] (localhost [127.0.0.1]) by localhost (Mailerdaemon) with ESMTPSA id 2BC9064C1616\n\tfor <some@one.de>; Wed, 12 Oct 2022 21:40:46 +0200 (CEST)",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "return-path",
"value": "<mail@someserver.de>",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "subject",
"value": "Simple Text Mail",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "to",
"value": "some@one.de",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "user-agent",
"value": "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n Thunderbird/102.3.1",
} in metadata
assert {
"namespace": "",
"prefix": "header",
"key": "x-last-tls-session-version",
"value": "TLSv1.3",
} in metadata
class TestEmailThumbnailGenerate(BaseMailParserTestCase):
class TestEmailThumbnailGenerate:
"""
Tests the correct generation of an thumbnail for an email
"""
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
@mock.patch("paperless_mail.parsers.make_thumbnail_from_pdf")
def test_get_thumbnail(
self,
mock_make_thumbnail_from_pdf: mock.MagicMock,
mock_generate_pdf: mock.MagicMock,
mocker: MockerFixture,
mail_parser: MailDocumentParser,
simple_txt_email_file: Path,
):
"""
GIVEN:
@@ -282,29 +243,34 @@ class TestEmailThumbnailGenerate(BaseMailParserTestCase):
- The parser should call the functions which generate the thumbnail
"""
mocked_return = "Passing the return value through.."
mock_make_thumbnail_from_pdf = mocker.patch(
"paperless_mail.parsers.make_thumbnail_from_pdf",
)
mock_make_thumbnail_from_pdf.return_value = mocked_return
mock_generate_pdf = mocker.patch(
"paperless_mail.parsers.MailDocumentParser.generate_pdf",
)
mock_generate_pdf.return_value = "Mocked return value.."
test_file = self.SAMPLE_DIR / "simple_text.eml"
thumb = self.parser.get_thumbnail(
test_file,
"message/rfc822",
)
thumb = mail_parser.get_thumbnail(simple_txt_email_file, "message/rfc822")
mock_generate_pdf.assert_called_once()
mock_make_thumbnail_from_pdf.assert_called_once_with(
"Mocked return value..",
self.parser.tempdir,
mail_parser.tempdir,
None,
)
self.assertEqual(mocked_return, thumb)
assert mocked_return == thumb
class TestTikaHtmlParse(HttpxMockMixin, BaseMailParserTestCase):
def test_tika_parse_unsuccessful(self):
class TestTikaHtmlParse:
def test_tika_parse_unsuccessful(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
):
"""
GIVEN:
- Fresh start
@@ -314,13 +280,13 @@ class TestTikaHtmlParse(HttpxMockMixin, BaseMailParserTestCase):
- the parser should return an empty string
"""
# Check unsuccessful parsing
self.httpx_mock.add_response(
httpx_mock.add_response(
json={"Content-Type": "text/html", "X-TIKA:Parsed-By": []},
)
parsed = self.parser.tika_parse("None")
self.assertEqual("", parsed)
parsed = mail_parser.tika_parse("None")
assert parsed == ""
def test_tika_parse(self):
def test_tika_parse(self, httpx_mock: HTTPXMock, mail_parser: MailDocumentParser):
"""
GIVEN:
- Fresh start
@@ -332,18 +298,22 @@ class TestTikaHtmlParse(HttpxMockMixin, BaseMailParserTestCase):
html = '<html><head><meta http-equiv="content-type" content="text/html; charset=UTF-8"></head><body><p>Some Text</p></body></html>'
expected_text = "Some Text"
self.httpx_mock.add_response(
httpx_mock.add_response(
json={
"Content-Type": "text/html",
"X-TIKA:Parsed-By": [],
"X-TIKA:content": expected_text,
},
)
parsed = self.parser.tika_parse(html)
self.assertEqual(expected_text, parsed.strip())
self.assertIn("http://localhost:9998", str(self.httpx_mock.get_request().url))
parsed = mail_parser.tika_parse(html)
assert expected_text == parsed.strip()
assert "http://localhost:9998" in str(httpx_mock.get_request().url)
def test_tika_parse_exception(self):
def test_tika_parse_exception(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
):
"""
GIVEN:
- Fresh start
@@ -354,11 +324,16 @@ class TestTikaHtmlParse(HttpxMockMixin, BaseMailParserTestCase):
"""
html = '<html><head><meta http-equiv="content-type" content="text/html; charset=UTF-8"></head><body><p>Some Text</p></body></html>'
self.httpx_mock.add_response(status_code=httpx.codes.INTERNAL_SERVER_ERROR)
httpx_mock.add_response(status_code=httpx.codes.INTERNAL_SERVER_ERROR)
self.assertRaises(ParseError, self.parser.tika_parse, html)
with pytest.raises(ParseError):
mail_parser.tika_parse(html)
def test_tika_parse_unreachable(self):
def test_tika_parse_unreachable(
self,
settings: SettingsWrapper,
mail_parser: MailDocumentParser,
):
"""
GIVEN:
- Fresh start
@@ -370,30 +345,18 @@ class TestTikaHtmlParse(HttpxMockMixin, BaseMailParserTestCase):
html = '<html><head><meta http-equiv="content-type" content="text/html; charset=UTF-8"></head><body><p>Some Text</p></body></html>'
# Check if exception is raised when Tika cannot be reached.
self.parser.tika_server = ""
self.assertRaises(ParseError, self.parser.tika_parse, html)
with pytest.raises(ParseError):
settings.TIKA_ENDPOINT = "http://does-not-exist:9998"
mail_parser.tika_parse(html)
class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase):
def test_parse_no_file(self):
"""
GIVEN:
- Fresh start
WHEN:
- parsing is attempted with nonexistent file
THEN:
- Exception is thrown
"""
# Check if exception is raised when parsing fails.
self.assertRaises(
ParseError,
self.parser.parse,
self.SAMPLE_DIR / "na.eml",
"message/rfc822",
)
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
def test_parse_eml_simple(self, mock_generate_pdf: mock.MagicMock):
class TestParser:
def test_parse_eml_simple(
self,
mocker: MockerFixture,
mail_parser: MailDocumentParser,
simple_txt_email_file: Path,
):
"""
GIVEN:
- Fresh start
@@ -403,11 +366,11 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
- parsed information is available
"""
# Validate parsing returns the expected results
self.parser.parse(
self.SAMPLE_DIR / "simple_text.eml",
"message/rfc822",
mock_generate_pdf = mocker.patch(
"paperless_mail.parsers.MailDocumentParser.generate_pdf",
)
mail_parser.parse(simple_txt_email_file, "message/rfc822")
text_expected = (
"Subject: Simple Text Mail\n\n"
"From: Some One <mail@someserver.de>\n\n"
@@ -416,8 +379,8 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
"BCC: fdf@fvf.de\n\n"
"\n\nThis is just a simple Text Mail."
)
self.assertEqual(text_expected, self.parser.text)
self.assertEqual(
assert text_expected == mail_parser.text
assert (
datetime.datetime(
2022,
10,
@@ -426,15 +389,20 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
40,
43,
tzinfo=datetime.timezone(datetime.timedelta(seconds=7200)),
),
self.parser.date,
)
== mail_parser.date
)
# Just check if tried to generate archive, the unittest for generate_pdf() goes deeper.
mock_generate_pdf.assert_called()
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
def test_parse_eml_html(self, mock_generate_pdf: mock.MagicMock):
def test_parse_eml_html(
self,
mocker: MockerFixture,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
html_email_file: Path,
):
"""
GIVEN:
- Fresh start
@@ -443,6 +411,11 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
THEN:
- Tika is called, parsed information from non html parts is available
"""
mock_generate_pdf = mocker.patch(
"paperless_mail.parsers.MailDocumentParser.generate_pdf",
)
# Validate parsing returns the expected results
text_expected = (
"Subject: HTML Message\n\n"
@@ -453,7 +426,7 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
"Some Text and an embedded image."
)
self.httpx_mock.add_response(
httpx_mock.add_response(
json={
"Content-Type": "text/html",
"X-TIKA:Parsed-By": [],
@@ -461,11 +434,11 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
},
)
self.parser.parse(self.SAMPLE_DIR / "html.eml", "message/rfc822")
mail_parser.parse(html_email_file, "message/rfc822")
mock_generate_pdf.assert_called_once()
self.assertEqual(text_expected, self.parser.text)
self.assertEqual(
assert text_expected == mail_parser.text
assert (
datetime.datetime(
2022,
10,
@@ -474,11 +447,16 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
23,
19,
tzinfo=datetime.timezone(datetime.timedelta(seconds=7200)),
),
self.parser.date,
)
== mail_parser.date
)
def test_generate_pdf_parse_error(self):
def test_generate_pdf_parse_error(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
simple_txt_email_file: Path,
):
"""
GIVEN:
- Fresh start
@@ -487,16 +465,18 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
THEN:
- a ParseError Exception is thrown
"""
self.httpx_mock.add_response(status_code=httpx.codes.INTERNAL_SERVER_ERROR)
httpx_mock.add_response(status_code=httpx.codes.INTERNAL_SERVER_ERROR)
self.assertRaises(
ParseError,
self.parser.parse,
self.SAMPLE_DIR / "simple_text.eml",
"message/rfc822",
)
with pytest.raises(ParseError):
mail_parser.parse(simple_txt_email_file, "message/rfc822")
def test_generate_pdf_simple_email(self):
def test_generate_pdf_simple_email(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
simple_txt_email_file: Path,
simple_txt_email_pdf_file: Path,
):
"""
GIVEN:
- Simple text email with no HTML content
@@ -507,17 +487,23 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
- Archive file is generated
"""
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:3000/forms/chromium/convert/html",
method="POST",
content=(self.SAMPLE_DIR / "simple_text.eml.pdf").read_bytes(),
content=simple_txt_email_pdf_file.read_bytes(),
)
self.parser.parse(self.SAMPLE_DIR / "simple_text.eml", "message/rfc822")
mail_parser.parse(simple_txt_email_file, "message/rfc822")
self.assertIsNotNone(self.parser.archive_path)
assert mail_parser.archive_path is not None
def test_generate_pdf_html_email(self):
def test_generate_pdf_html_email(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
html_email_file: Path,
html_email_pdf_file: Path,
):
"""
GIVEN:
- email with HTML content
@@ -528,7 +514,7 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
- Gotenberg is used to merge the two PDFs
- Archive file is generated
"""
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:9998/tika/text",
method="PUT",
json={
@@ -537,21 +523,27 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
"X-TIKA:content": "This is some Tika HTML text",
},
)
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:3000/forms/chromium/convert/html",
method="POST",
content=(self.SAMPLE_DIR / "html.eml.pdf").read_bytes(),
content=html_email_pdf_file.read_bytes(),
)
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:3000/forms/pdfengines/merge",
method="POST",
content=b"Pretend merged PDF content",
)
self.parser.parse(self.SAMPLE_DIR / "html.eml", "message/rfc822")
mail_parser.parse(html_email_file, "message/rfc822")
self.assertIsNotNone(self.parser.archive_path)
assert mail_parser.archive_path is not None
def test_generate_pdf_html_email_html_to_pdf_failure(self):
def test_generate_pdf_html_email_html_to_pdf_failure(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
html_email_file: Path,
html_email_pdf_file: Path,
):
"""
GIVEN:
- email with HTML content
@@ -561,7 +553,7 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
THEN:
- ParseError is raised
"""
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:9998/tika/text",
method="PUT",
json={
@@ -570,20 +562,26 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
"X-TIKA:content": "This is some Tika HTML text",
},
)
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:3000/forms/chromium/convert/html",
method="POST",
content=(self.SAMPLE_DIR / "html.eml.pdf").read_bytes(),
content=html_email_pdf_file.read_bytes(),
)
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:3000/forms/chromium/convert/html",
method="POST",
status_code=httpx.codes.INTERNAL_SERVER_ERROR,
)
with self.assertRaises(ParseError):
self.parser.parse(self.SAMPLE_DIR / "html.eml", "message/rfc822")
with pytest.raises(ParseError):
mail_parser.parse(html_email_file, "message/rfc822")
def test_generate_pdf_html_email_merge_failure(self):
def test_generate_pdf_html_email_merge_failure(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
html_email_file: Path,
html_email_pdf_file: Path,
):
"""
GIVEN:
- email with HTML content
@@ -593,7 +591,7 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
THEN:
- ParseError is raised
"""
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:9998/tika/text",
method="PUT",
json={
@@ -602,20 +600,25 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
"X-TIKA:content": "This is some Tika HTML text",
},
)
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:3000/forms/chromium/convert/html",
method="POST",
content=(self.SAMPLE_DIR / "html.eml.pdf").read_bytes(),
content=html_email_pdf_file.read_bytes(),
)
self.httpx_mock.add_response(
httpx_mock.add_response(
url="http://localhost:3000/forms/pdfengines/merge",
method="POST",
status_code=httpx.codes.INTERNAL_SERVER_ERROR,
)
with self.assertRaises(ParseError):
self.parser.parse(self.SAMPLE_DIR / "html.eml", "message/rfc822")
with pytest.raises(ParseError):
mail_parser.parse(html_email_file, "message/rfc822")
def test_mail_to_html(self):
def test_mail_to_html(
self,
mail_parser: MailDocumentParser,
html_email_file: Path,
html_email_html_file: Path,
):
"""
GIVEN:
- Email message with HTML content
@@ -624,14 +627,19 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
THEN:
- Resulting HTML is as expected
"""
mail = self.parser.parse_file_to_message(self.SAMPLE_DIR / "html.eml")
html_file = self.parser.mail_to_html(mail)
expected_html_file = self.SAMPLE_DIR / "html.eml.html"
mail = mail_parser.parse_file_to_message(html_email_file)
html_file = mail_parser.mail_to_html(mail)
self.assertHTMLEqual(expected_html_file.read_text(), html_file.read_text())
expected_html = parse_html(html_email_html_file.read_text())
actual_html = parse_html(html_file.read_text())
assert expected_html == actual_html
def test_generate_pdf_from_mail(
self,
httpx_mock: HTTPXMock,
mail_parser: MailDocumentParser,
html_email_file: Path,
):
"""
GIVEN:
@@ -642,16 +650,13 @@ class TestParser(FileSystemAssertsMixin, HttpxMockMixin, BaseMailParserTestCase)
- Gotenberg is used to convert HTML to PDF
"""
self.httpx_mock.add_response(content=b"Content")
httpx_mock.add_response(content=b"Content")
mail = self.parser.parse_file_to_message(self.SAMPLE_DIR / "html.eml")
mail = mail_parser.parse_file_to_message(html_email_file)
retval = self.parser.generate_pdf_from_mail(mail)
self.assertEqual(b"Content", retval.read_bytes())
retval = mail_parser.generate_pdf_from_mail(mail)
assert retval.read_bytes() == b"Content"
request = self.httpx_mock.get_request()
request = httpx_mock.get_request()
self.assertEqual(
str(request.url),
"http://localhost:3000/forms/chromium/convert/html",
)
assert str(request.url) == "http://localhost:3000/forms/chromium/convert/html"

116
src/paperless_mail/tests/test_parsers_live.py
View File

@@ -3,17 +3,15 @@ import shutil
import subprocess
import tempfile
from pathlib import Path
from unittest import mock
import httpx
import pytest
from django.test import TestCase
from imagehash import average_hash
from PIL import Image
from pytest_mock import MockerFixture
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import util_call_with_backoff
from paperless_mail.tests.test_parsers import BaseMailParserTestCase
from paperless_mail.parsers import MailDocumentParser
def extract_text(pdf_path: Path) -> str:
@@ -50,7 +48,7 @@ class MailAttachmentMock:
"PAPERLESS_CI_TEST" not in os.environ,
reason="No Gotenberg/Tika servers to test with",
)
class TestUrlCanary(TestCase):
class TestUrlCanary:
"""
Verify certain URLs are still available so testing is valid still
"""
@@ -69,13 +67,13 @@ class TestUrlCanary(TestCase):
whether this image stays online forever, so here we check if we can detect if is not
available anymore.
"""
with self.assertRaises(httpx.HTTPStatusError) as cm:
with pytest.raises(httpx.HTTPStatusError) as exec_info:
resp = httpx.get(
"https://upload.wikimedia.org/wikipedia/en/f/f7/nonexistent.png",
)
resp.raise_for_status()
self.assertEqual(cm.exception.response.status_code, httpx.codes.NOT_FOUND)
assert exec_info.value.response.status_code == httpx.codes.NOT_FOUND
def test_is_online_image_still_available(self):
"""
@@ -100,13 +98,19 @@ class TestUrlCanary(TestCase):
"PAPERLESS_CI_TEST" not in os.environ,
reason="No Gotenberg/Tika servers to test with",
)
class TestParserLive(FileSystemAssertsMixin, BaseMailParserTestCase):
class TestParserLive:
@staticmethod
def imagehash(file, hash_size=18):
return f"{average_hash(Image.open(file), hash_size)}"
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
def test_get_thumbnail(self, mock_generate_pdf: mock.MagicMock):
def test_get_thumbnail(
self,
mocker: MockerFixture,
mail_parser: MailDocumentParser,
simple_txt_email_file: Path,
simple_txt_email_pdf_file: Path,
simple_txt_email_thumbnail_file: Path,
):
"""
GIVEN:
- Fresh start
@@ -115,22 +119,21 @@ class TestParserLive(FileSystemAssertsMixin, BaseMailParserTestCase):
THEN:
- The returned thumbnail image file is as expected
"""
mock_generate_pdf.return_value = self.SAMPLE_DIR / "simple_text.eml.pdf"
thumb = self.parser.get_thumbnail(
self.SAMPLE_DIR / "simple_text.eml",
"message/rfc822",
mock_generate_pdf = mocker.patch(
"paperless_mail.parsers.MailDocumentParser.generate_pdf",
)
self.assertIsFile(thumb)
mock_generate_pdf.return_value = simple_txt_email_pdf_file
expected = self.SAMPLE_DIR / "simple_text.eml.pdf.webp"
thumb = mail_parser.get_thumbnail(simple_txt_email_file, "message/rfc822")
self.assertEqual(
self.imagehash(thumb),
self.imagehash(expected),
f"Created Thumbnail {thumb} differs from expected file {expected}",
)
assert thumb.exists()
assert thumb.is_file()
def test_tika_parse_successful(self):
assert (
self.imagehash(thumb) == self.imagehash(simple_txt_email_thumbnail_file)
), f"Created Thumbnail {thumb} differs from expected file {simple_txt_email_thumbnail_file}"
def test_tika_parse_successful(self, mail_parser: MailDocumentParser):
"""
GIVEN:
- Fresh start
@@ -143,15 +146,16 @@ class TestParserLive(FileSystemAssertsMixin, BaseMailParserTestCase):
expected_text = "Some Text"
# Check successful parsing
parsed = self.parser.tika_parse(html)
self.assertEqual(expected_text, parsed.strip())
parsed = mail_parser.tika_parse(html)
assert expected_text == parsed.strip()
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf_from_mail")
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf_from_html")
def test_generate_pdf_gotenberg_merging(
self,
mock_generate_pdf_from_html: mock.MagicMock,
mock_generate_pdf_from_mail: mock.MagicMock,
mocker: MockerFixture,
mail_parser: MailDocumentParser,
html_email_file: Path,
merged_pdf_first: Path,
merged_pdf_second: Path,
):
"""
GIVEN:
@@ -161,61 +165,67 @@ class TestParserLive(FileSystemAssertsMixin, BaseMailParserTestCase):
THEN:
- gotenberg is called to merge files and the resulting file is returned
"""
mock_generate_pdf_from_mail.return_value = self.SAMPLE_DIR / "first.pdf"
mock_generate_pdf_from_html.return_value = self.SAMPLE_DIR / "second.pdf"
msg = self.parser.parse_file_to_message(
self.SAMPLE_DIR / "html.eml",
mock_generate_pdf_from_html = mocker.patch(
"paperless_mail.parsers.MailDocumentParser.generate_pdf_from_html",
)
mock_generate_pdf_from_mail = mocker.patch(
"paperless_mail.parsers.MailDocumentParser.generate_pdf_from_mail",
)
mock_generate_pdf_from_mail.return_value = merged_pdf_first
mock_generate_pdf_from_html.return_value = merged_pdf_second
msg = mail_parser.parse_file_to_message(html_email_file)
_, pdf_path = util_call_with_backoff(
self.parser.generate_pdf,
mail_parser.generate_pdf,
[msg],
)
self.assertIsFile(pdf_path)
assert pdf_path.exists()
assert pdf_path.is_file()
extracted = extract_text(pdf_path)
expected = (
"first PDF to be merged.\n\x0csecond PDF to be merged.\n\x0c"
)
self.assertEqual(expected, extracted)
assert expected == extracted
def test_generate_pdf_from_mail(self):
def test_generate_pdf_from_mail(
self,
mail_parser: MailDocumentParser,
html_email_file: Path,
html_email_pdf_file: Path,
html_email_thumbnail_file: Path,
):
"""
GIVEN:
- Fresh start
WHEN:
- pdf generation from simple eml file is requested
THEN:
- gotenberg is called and the resulting file is returned and look as expected.
- Gotenberg is called and the resulting file is returned and look as expected.
"""
util_call_with_backoff(
self.parser.parse,
[self.SAMPLE_DIR / "html.eml", "message/rfc822"],
)
util_call_with_backoff(mail_parser.parse, [html_email_file, "message/rfc822"])
# Check the archive PDF
archive_path = self.parser.get_archive_path()
archive_path = mail_parser.get_archive_path()
archive_text = extract_text(archive_path)
expected_archive_text = extract_text(self.SAMPLE_DIR / "html.eml.pdf")
expected_archive_text = extract_text(html_email_pdf_file)
# Archive includes the HTML content, so use in
self.assertIn(expected_archive_text, archive_text)
assert expected_archive_text in archive_text
# Check the thumbnail
generated_thumbnail = self.parser.get_thumbnail(
self.SAMPLE_DIR / "html.eml",
generated_thumbnail = mail_parser.get_thumbnail(
html_email_file,
"message/rfc822",
)
generated_thumbnail_hash = self.imagehash(generated_thumbnail)
# The created pdf is not reproducible. But the converted image should always look the same.
expected_hash = self.imagehash(self.SAMPLE_DIR / "html.eml.pdf.webp")
expected_hash = self.imagehash(html_email_thumbnail_file)
self.assertEqual(
generated_thumbnail_hash,
expected_hash,
f"PDF looks different. Check if {generated_thumbnail} looks weird.",
)
assert (
generated_thumbnail_hash == expected_hash
), f"PDF looks different. Check if {generated_thumbnail} looks weird."