improve pdf generation

src/paperless_tika/mail_template/index.html

@@ -12,25 +12,25 @@
     <!-- Header -->
     <div class="grid grid-rows-5 grid-cols-12 gap-x-2 bg-slate-200 p-4">
 
-      <div class="col-start-11 col-span-2 row-start-1 text-right">23.04.2022 18:18</div>
+      <div class="col-start-11 col-span-2 row-start-1 text-right">{{ date }}</div>
 
       <div class="col-start-1 row-start-1 text-slate-400 text-right">From</div>
       <div class="col-start-2 col-span-8 row-start 1">{{ from }}</div>
 
       <div class="col-start-1 row-start-2 text-slate-400 text-right">Subject</div>
-      <div class=" col-start-2 col-span-10 row-start 2 font-bold">{{ Subject }}
+      <div class=" col-start-2 col-span-10 row-start 2 font-bold">{{ subject }}
       </div>
 
       <div class="col-start-1 row-start-3 text-slate-400 text-right">To</div>
-      <div class="col-start-2 col-span-10 row-start 3">{{ To }}
+      <div class="col-start-2 col-span-10 row-start 3">{{ to }}
       </div>
 
       <div class="col-start-1 row-start-4 text-slate-400 text-right">CC</div>
-      <div class="col-start-2 col-span-10 row-start 4">{{ CC }}
+      <div class="col-start-2 col-span-10 row-start 4">{{ cc }}
       </div>
 
       <div class="col-start-1 row-start-5 text-slate-400 text-right">BCC</div>
-      <div class="col-start-2 col-span-10 row-start 5">{{ BCC }}
+      <div class="col-start-2 col-span-10 row-start 5">{{ bcc }}
       </div>
     </div>
 

src/paperless_tika/parsers.py

@@ -107,6 +107,25 @@ class TikaDocumentParserEml(DocumentParser):
     """
 
     logging_name = "paperless.parsing.tikaeml"
+    _tika_parsed = None
+
+    def get_tika_result(self, document_path):
+        if not self._tika_parsed:
+            self.log("info", f"Sending {document_path} to Tika server")
+            tika_server = settings.PAPERLESS_TIKA_ENDPOINT
+
+            try:
+                self._tika_parsed = parser.from_file(
+                    document_path,
+                    tika_server,
+                )
+            except Exception as err:
+                raise ParseError(
+                    f"Could not parse {document_path} with tika server at "
+                    f"{tika_server}: {err}",
+                )
+
+        return self._tika_parsed
 
     def get_thumbnail(self, document_path, mime_type, file_name=None):
         if not self.archive_path:
@@ -122,10 +141,9 @@ class TikaDocumentParserEml(DocumentParser):
         result = []
         prefix_pattern = re.compile(r"(.*):(.*)")
 
-        tika_server = settings.PAPERLESS_TIKA_ENDPOINT
         try:
-            parsed = parser.from_file(document_path, tika_server)
+            parsed = self.get_tika_result(document_path)
-        except Exception as e:
+        except ParseError as e:
             self.log(
                 "warning",
                 f"Error while fetching document metadata for " f"{document_path}: {e}",
@@ -164,20 +182,9 @@ class TikaDocumentParserEml(DocumentParser):
         return result
 
     def parse(self, document_path, mime_type, file_name=None):
-        self.log("info", f"Sending {document_path} to Tika server")
+        parsed = self.get_tika_result(document_path)
-        tika_server = settings.PAPERLESS_TIKA_ENDPOINT
 
-        try:
+        subject = parsed["metadata"].get("dc:subject", "<no subject>")
-            parsed = parser.from_file(document_path, tika_server)
-        except Exception as err:
-            raise ParseError(
-                f"Could not parse {document_path} with tika server at "
-                f"{tika_server}: {err}",
-            )
-
-        metadata = parsed["metadata"].copy()
-
-        subject = metadata.pop("dc:subject", "<no subject>")
         content = parsed["content"].strip()
 
         if content.startswith(subject):
@@ -187,36 +194,25 @@ class TikaDocumentParserEml(DocumentParser):
         content = re.sub("\n+", "\n", content)
 
         self.text = (
-            f"{content}\n"
+            f"{content}\n\n"
-            f"______________________\n"
+            f"From: {parsed['metadata'].get('Message-From', '')}\n"
-            f"From: {metadata.pop('Message-From', '')}\n"
+            f"To: {parsed['metadata'].get('Message-To', '')}\n"
-            f"To: {metadata.pop('Message-To', '')}\n"
+            f"CC: {parsed['metadata'].get('Message-CC', '')}"
-            f"CC: {metadata.pop('Message-CC', '')}"
         )
 
         try:
-            self.date = dateutil.parser.isoparse(parsed["metadata"]["dcterms:created"])
+            self.date = dateutil.parser.isoparse(
+                parsed["metadata"]["dcterms:created"],
+            )
         except Exception as e:
             self.log(
                 "warning",
                 f"Unable to extract date for document " f"{document_path}: {e}",
             )
 
-        self.archive_path = self.generate_pdf(document_path, parsed)
+        self.archive_path = self.generate_pdf(document_path)
-
-    def generate_pdf(self, document_path, parsed=None):
-        if not parsed:
-            self.log("info", f"Sending {document_path} to Tika server")
-            tika_server = settings.PAPERLESS_TIKA_ENDPOINT
-
-            try:
-                parsed = parser.from_file(document_path, tika_server)
-            except Exception as err:
-                raise ParseError(
-                    f"Could not parse {document_path} with tika server at "
-                    f"{tika_server}: {err}",
-                )
 
+    def generate_pdf(self, document_path):
         def clean_html(text: str):
             if isinstance(text, list):
                 text = ", ".join([str(e) for e in text])
@@ -230,51 +226,59 @@ class TikaDocumentParserEml(DocumentParser):
             text = text.replace('"', "&quot;")
             return text
 
+        parsed = self.get_tika_result(document_path)
+
         pdf_path = os.path.join(self.tempdir, "convert.pdf")
         gotenberg_server = settings.PAPERLESS_TIKA_GOTENBERG_ENDPOINT
         url = gotenberg_server + "/forms/chromium/convert/html"
 
         self.log("info", f"Converting {document_path} to PDF as {pdf_path}")
 
-        subject = parsed["metadata"].pop("dc:subject", "<no subject>")
+        data = {}
-        content = parsed.pop("content", "<no content>").strip()
+        data["subject"] = clean_html(parsed["metadata"].get("dc:subject", ""))
+        data["from"] = clean_html(parsed["metadata"].get("Message-From", ""))
+        data["to"] = clean_html(parsed["metadata"].get("Message-To", ""))
+        data["cc"] = clean_html(parsed["metadata"].get("Message-CC", ""))
+        data["date"] = clean_html(parsed["metadata"].get("dcterms:created", ""))
 
-        if content.startswith(subject):
+        content = parsed.get("content", "").strip()
-            content = content[len(subject) :].strip()
+        if content.startswith(data["subject"]):
+            content = content[len(data["subject"]) :].strip()
+        data["content"] = clean_html(content)
 
-        html = StringIO(
+        html_file = os.path.join(os.path.dirname(__file__), "mail_template/index.html")
-            f"""
+        css_file = os.path.join(os.path.dirname(__file__), "mail_template/output.css")
-            <!doctype html>
+        placeholder_pattern = re.compile(r"{{(.+)}}")
-            <html lang="en">
+        html = StringIO()
-              <head>
-                <meta charset="utf-8">
-                <title>My PDF</title>
-              </head>
-              <body>
-                <h1>{clean_html(subject)}</h1>
-                <p>From: {clean_html(parsed['metadata'].pop('Message-From', ''))}
-                <p>To: {clean_html(parsed['metadata'].pop('Message-To', ''))}
-                <p>CC: {clean_html(parsed['metadata'].pop('Message-CC', ''))}
-                <p>Date: {clean_html(parsed['metadata'].pop('dcterms:created', ''))}
-                <pre>{clean_html(content)}</pre>
-              </body>
-            </html>
-            """,
-        )
 
-        files = {
+        with open(html_file, "r") as html_template_handle:
-            "html": (
+            with open(css_file, "rb") as css_handle:
-                "index.html",
+                for line in html_template_handle.readlines():
-                html,
+                    for placeholder in placeholder_pattern.findall(line):
-            ),
+                        line = re.sub(
-        }
+                            "{{" + placeholder + "}}",
-        headers = {}
+                            data.get(placeholder.strip(), ""),
+                            line,
+                        )
+                    html.write(line)
+                html.seek(0)
+                files = {
+                    "html": (
+                        "index.html",
+                        html,
+                    ),
+                    "css": (
+                        "output.css",
+                        css_handle,
+                    ),
+                }
+                headers = {}
 
-        try:
+                try:
-            response = requests.post(url, files=files, headers=headers)
+                    response = requests.post(url, files=files, headers=headers)
-            response.raise_for_status()  # ensure we notice bad responses
+                    response.raise_for_status()  # ensure we notice bad responses
-        except Exception as err:
+                except Exception as err:
-            raise ParseError(f"Error while converting document to PDF: {err}")
+                    raise ParseError(f"Error while converting document to PDF: {err}")
 
         with open(pdf_path, "wb") as file:
             file.write(response.content)