mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-02 13:45:10 -05:00
add prototype archive pdf
This commit is contained in:
phail
parent
d8d2d53c59
commit
790bcf05ed
46
src/paperless_mail/mail_template/index.html
Normal file
46
src/paperless_mail/mail_template/index.html
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<link href="output.css" rel="stylesheet">
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body class="bg-white w-screen flex flex-col items-center">
|
||||||
|
<div class="container max-w-4xl">
|
||||||
|
<!-- Header -->
|
||||||
|
<div class="grid grid-rows-5 grid-cols-12 gap-x-2 bg-slate-200 p-4">
|
||||||
|
|
||||||
|
<div class="col-start-11 col-span-2 row-start-1 text-right">23.04.2022 18:18</div>
|
||||||
|
|
||||||
|
<div class="col-start-1 row-start-1 text-slate-400 text-right">From</div>
|
||||||
|
<div class="col-start-2 col-span-8 row-start 1">{{ from }}</div>
|
||||||
|
|
||||||
|
<div class="col-start-1 row-start-2 text-slate-400 text-right">Subject</div>
|
||||||
|
<div class=" col-start-2 col-span-10 row-start 2 font-bold">{{ Subject }}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="col-start-1 row-start-3 text-slate-400 text-right">To</div>
|
||||||
|
<div class="col-start-2 col-span-10 row-start 3">{{ To }}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="col-start-1 row-start-4 text-slate-400 text-right">CC</div>
|
||||||
|
<div class="col-start-2 col-span-10 row-start 4">{{ CC }}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="col-start-1 row-start-5 text-slate-400 text-right">BCC</div>
|
||||||
|
<div class="col-start-2 col-span-10 row-start 5">{{ BCC }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Separator-->
|
||||||
|
<div class="border-t border-solid border-b w-full h-[1px] box-content border-black mb-5 bg-slate-200"></div>
|
||||||
|
|
||||||
|
<!-- Content-->
|
||||||
|
<div class="w-full whitespace-pre-line">{{ content }}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
</html>
|
||||||
0
src/paperless_mail/mail_template/output.css
Normal file
0
src/paperless_mail/mail_template/output.css
Normal file
@ -8,9 +8,6 @@ from django.conf import settings
|
|||||||
from documents.parsers import DocumentParser
|
from documents.parsers import DocumentParser
|
||||||
from documents.parsers import make_thumbnail_from_pdf
|
from documents.parsers import make_thumbnail_from_pdf
|
||||||
from documents.parsers import ParseError
|
from documents.parsers import ParseError
|
||||||
from PIL import Image
|
|
||||||
from PIL import ImageDraw
|
|
||||||
from PIL import ImageFont
|
|
||||||
from tika import parser
|
from tika import parser
|
||||||
|
|
||||||
|
|
||||||
@ -112,22 +109,19 @@ class TikaDocumentParserEml(DocumentParser):
|
|||||||
logging_name = "paperless.parsing.tikaeml"
|
logging_name = "paperless.parsing.tikaeml"
|
||||||
|
|
||||||
def get_thumbnail(self, document_path, mime_type, file_name=None):
|
def get_thumbnail(self, document_path, mime_type, file_name=None):
|
||||||
|
if not self.archive_path:
|
||||||
|
self.archive_path = self.generate_pdf(document_path)
|
||||||
|
|
||||||
img = Image.new("RGB", (500, 700), color="white")
|
return make_thumbnail_from_pdf(
|
||||||
draw = ImageDraw.Draw(img)
|
self.archive_path,
|
||||||
font = ImageFont.truetype(
|
self.tempdir,
|
||||||
font=settings.THUMBNAIL_FONT_NAME,
|
self.logging_group,
|
||||||
size=20,
|
|
||||||
layout_engine=ImageFont.LAYOUT_BASIC,
|
|
||||||
)
|
)
|
||||||
draw.text((5, 5), self.text, font=font, fill="black")
|
|
||||||
|
|
||||||
out_path = os.path.join(self.tempdir, "thumb.png")
|
|
||||||
img.save(out_path)
|
|
||||||
|
|
||||||
return out_path
|
|
||||||
|
|
||||||
def extract_metadata(self, document_path, mime_type):
|
def extract_metadata(self, document_path, mime_type):
|
||||||
|
result = []
|
||||||
|
prefix_pattern = re.compile(r"(.*):(.*)")
|
||||||
|
|
||||||
tika_server = settings.PAPERLESS_TIKA_ENDPOINT
|
tika_server = settings.PAPERLESS_TIKA_ENDPOINT
|
||||||
try:
|
try:
|
||||||
parsed = parser.from_file(document_path, tika_server)
|
parsed = parser.from_file(document_path, tika_server)
|
||||||
@ -136,17 +130,38 @@ class TikaDocumentParserEml(DocumentParser):
|
|||||||
"warning",
|
"warning",
|
||||||
f"Error while fetching document metadata for " f"{document_path}: {e}",
|
f"Error while fetching document metadata for " f"{document_path}: {e}",
|
||||||
)
|
)
|
||||||
return []
|
return result
|
||||||
|
|
||||||
return [
|
for key, value in parsed["metadata"].items():
|
||||||
{
|
if isinstance(value, list):
|
||||||
"namespace": "",
|
value = ", ".join([str(e) for e in value])
|
||||||
"prefix": "",
|
value = str(value)
|
||||||
"key": key,
|
try:
|
||||||
"value": parsed["metadata"][key],
|
m = prefix_pattern.match(key)
|
||||||
}
|
result.append(
|
||||||
for key in parsed["metadata"]
|
{
|
||||||
]
|
"namespace": "",
|
||||||
|
"prefix": m.group(1),
|
||||||
|
"key": m.group(2),
|
||||||
|
"value": value,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except AttributeError:
|
||||||
|
result.append(
|
||||||
|
{
|
||||||
|
"namespace": "",
|
||||||
|
"prefix": "",
|
||||||
|
"key": key,
|
||||||
|
"value": value,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
self.log(
|
||||||
|
"warning",
|
||||||
|
f"Error while reading metadata {key}: {value}. Error: " f"{e}",
|
||||||
|
)
|
||||||
|
result.sort(key=lambda item: (item["prefix"], item["key"]))
|
||||||
|
return result
|
||||||
|
|
||||||
def parse(self, document_path, mime_type, file_name=None):
|
def parse(self, document_path, mime_type, file_name=None):
|
||||||
self.log("info", f"Sending {document_path} to Tika server")
|
self.log("info", f"Sending {document_path} to Tika server")
|
||||||
@ -160,57 +175,94 @@ class TikaDocumentParserEml(DocumentParser):
|
|||||||
f"{tika_server}: {err}",
|
f"{tika_server}: {err}",
|
||||||
)
|
)
|
||||||
|
|
||||||
text = re.sub(" +", " ", str(parsed))
|
metadata = parsed["metadata"].copy()
|
||||||
text = re.sub("\n+", "\n", text)
|
|
||||||
self.text = text
|
|
||||||
|
|
||||||
print(text)
|
subject = metadata.pop("dc:subject", "<no subject>")
|
||||||
|
content = parsed["content"].strip()
|
||||||
|
|
||||||
|
if content.startswith(subject):
|
||||||
|
content = content[len(subject) :].strip()
|
||||||
|
|
||||||
|
content = re.sub(" +", " ", content)
|
||||||
|
content = re.sub("\n+", "\n", content)
|
||||||
|
|
||||||
|
self.text = (
|
||||||
|
f"{content}\n"
|
||||||
|
f"______________________\n"
|
||||||
|
f"From: {metadata.pop('Message-From', '')}\n"
|
||||||
|
f"To: {metadata.pop('Message-To', '')}\n"
|
||||||
|
f"CC: {metadata.pop('Message-CC', '')}"
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.date = dateutil.parser.isoparse(parsed["metadata"]["Creation-Date"])
|
self.date = dateutil.parser.isoparse(parsed["metadata"]["dcterms:created"])
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log(
|
self.log(
|
||||||
"warning",
|
"warning",
|
||||||
f"Unable to extract date for document " f"{document_path}: {e}",
|
f"Unable to extract date for document " f"{document_path}: {e}",
|
||||||
)
|
)
|
||||||
|
|
||||||
md_path = self.convert_to_md(document_path, file_name)
|
self.archive_path = self.generate_pdf(document_path, parsed)
|
||||||
self.archive_path = self.convert_md_to_pdf(md_path)
|
|
||||||
|
def generate_pdf(self, document_path, parsed=None):
|
||||||
|
if not parsed:
|
||||||
|
self.log("info", f"Sending {document_path} to Tika server")
|
||||||
|
tika_server = settings.PAPERLESS_TIKA_ENDPOINT
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsed = parser.from_file(document_path, tika_server)
|
||||||
|
except Exception as err:
|
||||||
|
raise ParseError(
|
||||||
|
f"Could not parse {document_path} with tika server at "
|
||||||
|
f"{tika_server}: {err}",
|
||||||
|
)
|
||||||
|
|
||||||
|
def clean_html(text: str):
|
||||||
|
if isinstance(text, list):
|
||||||
|
text = ", ".join([str(e) for e in text])
|
||||||
|
if type(text) != str:
|
||||||
|
text = str(text)
|
||||||
|
text = text.replace("&", "&")
|
||||||
|
text = text.replace("<", "<")
|
||||||
|
text = text.replace(">", ">")
|
||||||
|
text = text.replace(" ", " ")
|
||||||
|
text = text.replace("'", "'")
|
||||||
|
text = text.replace('"', """)
|
||||||
|
return text
|
||||||
|
|
||||||
def convert_md_to_pdf(self, md_path):
|
|
||||||
pdf_path = os.path.join(self.tempdir, "convert.pdf")
|
pdf_path = os.path.join(self.tempdir, "convert.pdf")
|
||||||
gotenberg_server = settings.PAPERLESS_TIKA_GOTENBERG_ENDPOINT
|
gotenberg_server = settings.PAPERLESS_TIKA_GOTENBERG_ENDPOINT
|
||||||
url = gotenberg_server + "/forms/chromium/convert/markdown"
|
url = gotenberg_server + "/forms/chromium/convert/html"
|
||||||
|
|
||||||
|
self.log("info", f"Converting {document_path} to PDF as {pdf_path}")
|
||||||
|
|
||||||
|
subject = parsed["metadata"].pop("dc:subject", "<no subject>")
|
||||||
|
content = parsed.pop("content", "<no content>").strip()
|
||||||
|
|
||||||
|
if content.startswith(subject):
|
||||||
|
content = content[len(subject) :].strip()
|
||||||
|
|
||||||
self.log("info", f"Converting {md_path} to PDF as {pdf_path}")
|
|
||||||
html = StringIO(
|
html = StringIO(
|
||||||
"""
|
f"""
|
||||||
<!doctype html>
|
<!doctype html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8">
|
<meta charset="utf-8">
|
||||||
<title>My PDF</title>
|
<title>My PDF</title>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
{{ toHTML "convert.md" }}
|
<h1>{clean_html(subject)}</h1>
|
||||||
</body>
|
<p>From: {clean_html(parsed['metadata'].pop('Message-From', ''))}
|
||||||
</html>
|
<p>To: {clean_html(parsed['metadata'].pop('Message-To', ''))}
|
||||||
""",
|
<p>CC: {clean_html(parsed['metadata'].pop('Message-CC', ''))}
|
||||||
)
|
<p>Date: {clean_html(parsed['metadata'].pop('dcterms:created', ''))}
|
||||||
md = StringIO(
|
<pre>{clean_html(content)}</pre>
|
||||||
"""
|
</body>
|
||||||
# Subject
|
</html>
|
||||||
|
""",
|
||||||
blub \nblah
|
|
||||||
blib
|
|
||||||
""",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
files = {
|
files = {
|
||||||
"md": (
|
|
||||||
os.path.basename(md_path),
|
|
||||||
md,
|
|
||||||
),
|
|
||||||
"html": (
|
"html": (
|
||||||
"index.html",
|
"index.html",
|
||||||
html,
|
html,
|
||||||
@ -229,19 +281,3 @@ blib
|
|||||||
file.close()
|
file.close()
|
||||||
|
|
||||||
return pdf_path
|
return pdf_path
|
||||||
|
|
||||||
def convert_to_md(self, document_path, file_name):
|
|
||||||
md_path = os.path.join(self.tempdir, "convert.md")
|
|
||||||
|
|
||||||
self.log("info", f"Converting {document_path} to markdown as {md_path}")
|
|
||||||
|
|
||||||
with open(md_path, "w") as file:
|
|
||||||
md = [
|
|
||||||
"# Subject",
|
|
||||||
"\n\n",
|
|
||||||
"blah",
|
|
||||||
]
|
|
||||||
file.writelines(md)
|
|
||||||
file.close()
|
|
||||||
|
|
||||||
return md_path
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user