2022-11-20 23:26:20 +01:00

689 lines
23 KiB
Python

import datetime
import os
from unittest import mock
from django.test import TestCase
from documents.parsers import ParseError
from paperless_mail.parsers import MailDocumentParser
class TestParser(TestCase):
SAMPLE_FILES = os.path.join(os.path.dirname(__file__), "samples")
def setUp(self) -> None:
self.parser = MailDocumentParser(logging_group=None)
def tearDown(self) -> None:
self.parser.cleanup()
def test_get_parsed_missing_file(self):
"""
GIVEN:
- Fresh parser
WHEN:
- A nonexistent file should be parsed
THEN:
- An Exception is thrown
"""
# Check if exception is raised when parsing fails.
self.assertRaises(
ParseError,
self.parser.get_parsed,
os.path.join(self.SAMPLE_FILES, "na"),
)
def test_get_parsed_broken_file(self):
"""
GIVEN:
- Fresh parser
WHEN:
- A faulty file should be parsed
THEN:
- An Exception is thrown
"""
# Check if exception is raised when the mail is faulty.
self.assertRaises(
ParseError,
self.parser.get_parsed,
os.path.join(self.SAMPLE_FILES, "broken.eml"),
)
def test_get_parsed_simple_text_mail(self):
"""
GIVEN:
- Fresh parser
WHEN:
- A .eml file should be parsed
THEN:
- The content of the mail should be available in the parse result.
"""
# Parse Test file and check relevant content
parsed1 = self.parser.get_parsed(
os.path.join(self.SAMPLE_FILES, "simple_text.eml"),
)
self.assertEqual(parsed1.date.year, 2022)
self.assertEqual(parsed1.date.month, 10)
self.assertEqual(parsed1.date.day, 12)
self.assertEqual(parsed1.date.hour, 21)
self.assertEqual(parsed1.date.minute, 40)
self.assertEqual(parsed1.date.second, 43)
self.assertEqual(parsed1.date.tzname(), "UTC+02:00")
self.assertEqual(parsed1.from_, "mail@someserver.de")
self.assertEqual(parsed1.subject, "Simple Text Mail")
self.assertEqual(parsed1.text, "This is just a simple Text Mail.\n")
self.assertEqual(parsed1.to, ("some@one.de",))
def test_get_parsed_reparse(self):
"""
GIVEN:
- An E-Mail was parsed
WHEN:
- Another .eml file should be parsed
THEN:
- The parser should not retry to parse and return the old results
"""
# Parse Test file and check relevant content
parsed1 = self.parser.get_parsed(
os.path.join(self.SAMPLE_FILES, "simple_text.eml"),
)
# Check if same parsed object as before is returned, even if another file is given.
parsed2 = self.parser.get_parsed(
os.path.join(os.path.join(self.SAMPLE_FILES, "html.eml")),
)
self.assertEqual(parsed1, parsed2)
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
@mock.patch("paperless_mail.parsers.make_thumbnail_from_pdf")
def test_get_thumbnail(
self,
mock_make_thumbnail_from_pdf: mock.MagicMock,
mock_generate_pdf: mock.MagicMock,
):
"""
GIVEN:
- An E-Mail was parsed
WHEN:
- The Thumbnail is requested
THEN:
- The parser should call the functions which generate the thumbnail
"""
mocked_return = "Passing the return value through.."
mock_make_thumbnail_from_pdf.return_value = mocked_return
mock_generate_pdf.return_value = "Mocked return value.."
thumb = self.parser.get_thumbnail(
os.path.join(self.SAMPLE_FILES, "simple_text.eml"),
"message/rfc822",
)
self.assertEqual(
self.parser.archive_path,
mock_make_thumbnail_from_pdf.call_args_list[0].args[0],
)
self.assertEqual(
self.parser.tempdir,
mock_make_thumbnail_from_pdf.call_args_list[0].args[1],
)
self.assertEqual(mocked_return, thumb)
@mock.patch("documents.loggers.LoggingMixin.log")
def test_extract_metadata_fail(self, m: mock.MagicMock):
"""
GIVEN:
- Fresh start
WHEN:
- Metadata extraction is triggered for nonexistent file
THEN:
- A log warning should be generated
"""
# Validate if warning is logged when parsing fails
self.assertEqual([], self.parser.extract_metadata("na", "message/rfc822"))
self.assertEqual("warning", m.call_args[0][0])
def test_extract_metadata(self):
"""
GIVEN:
- Fresh start
WHEN:
- Metadata extraction is triggered
THEN:
- metadata is returned
"""
# Validate Metadata parsing returns the expected results
metadata = self.parser.extract_metadata(
os.path.join(self.SAMPLE_FILES, "simple_text.eml"),
"message/rfc822",
)
self.assertIn(
{"namespace": "", "prefix": "", "key": "attachments", "value": ""},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "",
"key": "date",
"value": "2022-10-12 21:40:43 UTC+02:00",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "content-language",
"value": "en-US",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "content-type",
"value": "text/plain; charset=UTF-8; format=flowed",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "date",
"value": "Wed, 12 Oct 2022 21:40:43 +0200",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "delivered-to",
"value": "mail@someserver.de",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "from",
"value": "Some One <mail@someserver.de>",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "message-id",
"value": "<6e99e34d-e20a-80c4-ea61-d8234b612be9@someserver.de>",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "mime-version",
"value": "1.0",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "received",
"value": "from mail.someserver.org ([::1])\n\tby e1acdba3bd07 with LMTP\n\tid KBKZGD2YR2NTCgQAjubtDA\n\t(envelope-from <mail@someserver.de>)\n\tfor <mail@someserver.de>; Wed, 10 Oct 2022 11:40:46 +0200, from [127.0.0.1] (localhost [127.0.0.1]) by localhost (Mailerdaemon) with ESMTPSA id 2BC9064C1616\n\tfor <some@one.de>; Wed, 12 Oct 2022 21:40:46 +0200 (CEST)",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "return-path",
"value": "<mail@someserver.de>",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "subject",
"value": "Simple Text Mail",
},
metadata,
)
self.assertIn(
{"namespace": "", "prefix": "header", "key": "to", "value": "some@one.de"},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "user-agent",
"value": "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101\n Thunderbird/102.3.1",
},
metadata,
)
self.assertIn(
{
"namespace": "",
"prefix": "header",
"key": "x-last-tls-session-version",
"value": "TLSv1.3",
},
metadata,
)
def test_parse_na(self):
"""
GIVEN:
- Fresh start
WHEN:
- parsing is attempted with nonexistent file
THEN:
- Exception is thrown
"""
# Check if exception is raised when parsing fails.
self.assertRaises(
ParseError,
self.parser.parse,
os.path.join(self.SAMPLE_FILES, "na"),
"message/rfc822",
)
@mock.patch("paperless_mail.parsers.MailDocumentParser.tika_parse")
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
def test_parse_html_eml(self, n, mock_tika_parse: mock.MagicMock):
"""
GIVEN:
- Fresh start
WHEN:
- parsing is done with html mail
THEN:
- Tika is called, parsed information from non html parts is available
"""
# Validate parsing returns the expected results
text_expected = "Subject: HTML Message\n\nFrom: Name <someone@example.de>\n\nTo: someone@example.de\n\nAttachments: IntM6gnXFm00FEV5.png (6.89 KiB), 600+kbfile.txt (600.24 KiB)\n\nHTML content: tika return\n\nSome Text and an embedded image."
mock_tika_parse.return_value = "tika return"
self.parser.parse(os.path.join(self.SAMPLE_FILES, "html.eml"), "message/rfc822")
self.assertEqual(text_expected, self.parser.text)
self.assertEqual(
datetime.datetime(
2022,
10,
15,
11,
23,
19,
tzinfo=datetime.timezone(datetime.timedelta(seconds=7200)),
),
self.parser.date,
)
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf")
def test_parse_simple_eml(self, n):
"""
GIVEN:
- Fresh start
WHEN:
- parsing is done with non html mail
THEN:
- parsed information is available
"""
# Validate parsing returns the expected results
self.parser.parse(
os.path.join(self.SAMPLE_FILES, "simple_text.eml"),
"message/rfc822",
)
text_expected = "Subject: Simple Text Mail\n\nFrom: Some One <mail@someserver.de>\n\nTo: some@one.de\n\nCC: asdasd@æsdasd.de, asdadasdasdasda.asdasd@æsdasd.de\n\nBCC: fdf@fvf.de\n\n\n\nThis is just a simple Text Mail."
self.assertEqual(text_expected, self.parser.text)
self.assertEqual(
datetime.datetime(
2022,
10,
12,
21,
40,
43,
tzinfo=datetime.timezone(datetime.timedelta(seconds=7200)),
),
self.parser.date,
)
# Just check if file exists, the unittest for generate_pdf() goes deeper.
self.assertTrue(os.path.isfile(self.parser.archive_path))
@mock.patch("paperless_mail.parsers.parser.from_buffer")
def test_tika_parse_unsuccessful(self, mock_from_buffer: mock.MagicMock):
"""
GIVEN:
- Fresh start
WHEN:
- tika parsing fails
THEN:
- the parser should return an empty string
"""
# Check unsuccessful parsing
mock_from_buffer.return_value = {"content": None}
parsed = self.parser.tika_parse(None)
self.assertEqual("", parsed)
@mock.patch("paperless_mail.parsers.parser.from_buffer")
def test_tika_parse(self, mock_from_buffer: mock.MagicMock):
"""
GIVEN:
- Fresh start
WHEN:
- tika parsing is called
THEN:
- a web request to tika shall be done and the reply es returned
"""
html = '<html><head><meta http-equiv="content-type" content="text/html; charset=UTF-8"></head><body><p>Some Text</p></body></html>'
expected_text = "Some Text"
# Check successful parsing
mock_from_buffer.return_value = {"content": expected_text}
parsed = self.parser.tika_parse(html)
self.assertEqual(expected_text, parsed.strip())
mock_from_buffer.assert_called_with(html, self.parser.tika_server)
@mock.patch("paperless_mail.parsers.parser.from_buffer")
def test_tika_parse_exception(self, mock_from_buffer: mock.MagicMock):
"""
GIVEN:
- Fresh start
WHEN:
- tika parsing is called and an exception is thrown on the request
THEN:
- a ParseError Exception is thrown
"""
html = '<html><head><meta http-equiv="content-type" content="text/html; charset=UTF-8"></head><body><p>Some Text</p></body></html>'
# Check ParseError
def my_side_effect():
raise Exception("Test")
mock_from_buffer.side_effect = my_side_effect
self.assertRaises(ParseError, self.parser.tika_parse, html)
def test_tika_parse_unreachable(self):
"""
GIVEN:
- Fresh start
WHEN:
- tika parsing is called but tika is not available
THEN:
- a ParseError Exception is thrown
"""
html = '<html><head><meta http-equiv="content-type" content="text/html; charset=UTF-8"></head><body><p>Some Text</p></body></html>'
# Check if exception is raised when Tika cannot be reached.
self.parser.tika_server = ""
self.assertRaises(ParseError, self.parser.tika_parse, html)
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf_from_mail")
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf_from_html")
def test_generate_pdf_parse_error(self, m: mock.MagicMock, n: mock.MagicMock):
"""
GIVEN:
- Fresh start
WHEN:
- pdf generation is requested but gotenberg can not be reached
THEN:
- a ParseError Exception is thrown
"""
m.return_value = b""
n.return_value = b""
# Check if exception is raised when the pdf can not be created.
self.parser.gotenberg_server = ""
self.assertRaises(
ParseError,
self.parser.generate_pdf,
os.path.join(self.SAMPLE_FILES, "html.eml"),
)
def test_generate_pdf_exception(self):
"""
GIVEN:
- Fresh start
WHEN:
- pdf generation is requested but parsing throws an exception
THEN:
- a ParseError Exception is thrown
"""
# Check if exception is raised when the mail can not be parsed.
self.assertRaises(
ParseError,
self.parser.generate_pdf,
os.path.join(self.SAMPLE_FILES, "broken.eml"),
)
@mock.patch("paperless_mail.parsers.requests.post")
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf_from_mail")
@mock.patch("paperless_mail.parsers.MailDocumentParser.generate_pdf_from_html")
def test_generate_pdf(
self,
mock_generate_pdf_from_html: mock.MagicMock,
mock_generate_pdf_from_mail: mock.MagicMock,
mock_post: mock.MagicMock,
):
"""
GIVEN:
- Fresh start
WHEN:
- pdf generation is requested
THEN:
- gotenberg is called and the resulting file is returned
"""
mock_generate_pdf_from_mail.return_value = b"Mail Return"
mock_generate_pdf_from_html.return_value = b"HTML Return"
mock_response = mock.MagicMock()
mock_response.content = b"Content"
mock_post.return_value = mock_response
pdf_path = self.parser.generate_pdf(os.path.join(self.SAMPLE_FILES, "html.eml"))
self.assertTrue(os.path.isfile(pdf_path))
mock_generate_pdf_from_mail.assert_called_once_with(
self.parser.get_parsed(None),
)
mock_generate_pdf_from_html.assert_called_once_with(
self.parser.get_parsed(None).html,
self.parser.get_parsed(None).attachments,
)
self.assertEqual(
self.parser.gotenberg_server + "/forms/pdfengines/merge",
mock_post.call_args.args[0],
)
self.assertEqual({}, mock_post.call_args.kwargs["headers"])
self.assertEqual(
b"Mail Return",
mock_post.call_args.kwargs["files"]["1_mail.pdf"][1].read(),
)
self.assertEqual(
b"HTML Return",
mock_post.call_args.kwargs["files"]["2_html.pdf"][1].read(),
)
mock_response.raise_for_status.assert_called_once()
with open(pdf_path, "rb") as file:
self.assertEqual(b"Content", file.read())
def test_mail_to_html(self):
"""
GIVEN:
- Fresh start
WHEN:
- conversion from eml to html is requested
THEN:
- html should be returned
"""
mail = self.parser.get_parsed(os.path.join(self.SAMPLE_FILES, "html.eml"))
html_handle = self.parser.mail_to_html(mail)
html_received = html_handle.read()
with open(
os.path.join(self.SAMPLE_FILES, "html.eml.html"),
) as html_expected_handle:
html_expected = html_expected_handle.read()
self.assertHTMLEqual(html_expected, html_received)
@mock.patch("paperless_mail.parsers.requests.post")
@mock.patch("paperless_mail.parsers.MailDocumentParser.mail_to_html")
def test_generate_pdf_from_mail(
self,
mock_mail_to_html: mock.MagicMock,
mock_post: mock.MagicMock,
):
"""
GIVEN:
- Fresh start
WHEN:
- conversion of PDF from .eml is requested
THEN:
- gotenberg should be called with valid intermediary html files, the resulting pdf is returned
"""
mock_response = mock.MagicMock()
mock_response.content = b"Content"
mock_post.return_value = mock_response
mock_mail_to_html.return_value = "Testresponse"
mail = self.parser.get_parsed(os.path.join(self.SAMPLE_FILES, "html.eml"))
retval = self.parser.generate_pdf_from_mail(mail)
self.assertEqual(b"Content", retval)
mock_mail_to_html.assert_called_once_with(mail)
self.assertEqual(
self.parser.gotenberg_server + "/forms/chromium/convert/html",
mock_post.call_args.args[0],
)
self.assertEqual({}, mock_post.call_args.kwargs["headers"])
self.assertEqual(
{
"marginTop": "0.1",
"marginBottom": "0.1",
"marginLeft": "0.1",
"marginRight": "0.1",
"paperWidth": "8.27",
"paperHeight": "11.7",
"scale": "1.0",
},
mock_post.call_args.kwargs["data"],
)
self.assertEqual(
"Testresponse",
mock_post.call_args.kwargs["files"]["html"][1],
)
self.assertEqual(
"output.css",
mock_post.call_args.kwargs["files"]["css"][0],
)
mock_response.raise_for_status.assert_called_once()
def test_transform_inline_html(self):
"""
GIVEN:
- Fresh start
WHEN:
- transforming of html content from an email with an inline image attachment is requested
THEN:
- html is returned and sanitized
"""
class MailAttachmentMock:
def __init__(self, payload, content_id):
self.payload = payload
self.content_id = content_id
result = None
with open(os.path.join(self.SAMPLE_FILES, "sample.html")) as html_file:
with open(os.path.join(self.SAMPLE_FILES, "sample.png"), "rb") as png_file:
html = html_file.read()
png = png_file.read()
attachments = [
MailAttachmentMock(png, "part1.pNdUSz0s.D3NqVtPg@example.de"),
]
result = self.parser.transform_inline_html(html, attachments)
resulting_html = result[-1][1].read()
self.assertTrue(result[-1][0] == "index.html")
self.assertIn(result[0][0], resulting_html)
self.assertNotIn("<script", resulting_html.lower())
@mock.patch("paperless_mail.parsers.requests.post")
def test_generate_pdf_from_html(self, mock_post: mock.MagicMock):
"""
GIVEN:
- Fresh start
WHEN:
- generating pdf from html with inline attachments is attempted
THEN:
- gotenberg is called with the correct parameters and the resulting pdf is returned
"""
class MailAttachmentMock:
def __init__(self, payload, content_id):
self.payload = payload
self.content_id = content_id
mock_response = mock.MagicMock()
mock_response.content = b"Content"
mock_post.return_value = mock_response
result = None
with open(os.path.join(self.SAMPLE_FILES, "sample.html")) as html_file:
with open(os.path.join(self.SAMPLE_FILES, "sample.png"), "rb") as png_file:
html = html_file.read()
png = png_file.read()
attachments = [
MailAttachmentMock(png, "part1.pNdUSz0s.D3NqVtPg@example.de"),
]
result = self.parser.generate_pdf_from_html(html, attachments)
self.assertEqual(
self.parser.gotenberg_server + "/forms/chromium/convert/html",
mock_post.call_args.args[0],
)
self.assertEqual({}, mock_post.call_args.kwargs["headers"])
self.assertEqual(
{
"marginTop": "0.1",
"marginBottom": "0.1",
"marginLeft": "0.1",
"marginRight": "0.1",
"paperWidth": "8.27",
"paperHeight": "11.7",
"scale": "1.0",
},
mock_post.call_args.kwargs["data"],
)
# read to assert it is a file like object.
mock_post.call_args.kwargs["files"]["cidpart1pNdUSz0sD3NqVtPgexamplede"][
1
].read()
mock_post.call_args.kwargs["files"]["index.html"][1].read()
mock_response.raise_for_status.assert_called_once()
self.assertEqual(b"Content", result)