mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-07-30 18:27:45 -05:00
Moves the barcode related functionality out of tasks and into its own location. Splits up the testing based on that
This commit is contained in:
Trenton Holmes
@@ -1,15 +1,12 @@
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from typing import List # for type hinting. Can be removed, if only Python >3.8 is used
|
||||
|
||||
import magic
|
||||
import tqdm
|
||||
from asgiref.sync import async_to_sync
|
||||
from channels.layers import get_channel_layer
|
||||
from django.conf import settings
|
||||
from django.db.models.signals import post_save
|
||||
from documents import barcodes
|
||||
from documents import index
|
||||
from documents import sanity_checker
|
||||
from documents.classifier import DocumentClassifier
|
||||
@@ -22,11 +19,6 @@ from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.sanity_checker import SanityCheckFailedException
|
||||
from pdf2image import convert_from_path
|
||||
from pikepdf import Pdf
|
||||
from PIL import Image
|
||||
from PIL import ImageSequence
|
||||
from pyzbar import pyzbar
|
||||
from whoosh.writing import AsyncWriter
|
||||
|
||||
|
||||
@@ -77,161 +69,6 @@ def train_classifier():
|
||||
logger.warning("Classifier error: " + str(e))
|
||||
|
||||
|
||||
def barcode_reader(image) -> List[str]:
|
||||
"""
|
||||
Read any barcodes contained in image
|
||||
Returns a list containing all found barcodes
|
||||
"""
|
||||
barcodes = []
|
||||
# Decode the barcode image
|
||||
detected_barcodes = pyzbar.decode(image)
|
||||
|
||||
if detected_barcodes:
|
||||
# Traverse through all the detected barcodes in image
|
||||
for barcode in detected_barcodes:
|
||||
if barcode.data:
|
||||
decoded_barcode = barcode.data.decode("utf-8")
|
||||
barcodes.append(decoded_barcode)
|
||||
logger.debug(
|
||||
f"Barcode of type {str(barcode.type)} found: {decoded_barcode}",
|
||||
)
|
||||
return barcodes
|
||||
|
||||
|
||||
def get_file_type(path: str) -> str:
|
||||
"""
|
||||
Determines the file type, based on MIME type.
|
||||
|
||||
Returns the MIME type.
|
||||
"""
|
||||
mime_type = magic.from_file(path, mime=True)
|
||||
logger.debug(f"Detected mime type: {mime_type}")
|
||||
return mime_type
|
||||
|
||||
|
||||
def convert_from_tiff_to_pdf(filepath: str) -> str:
|
||||
"""
|
||||
converts a given TIFF image file to pdf into a temporary directory.
|
||||
|
||||
Returns the new pdf file.
|
||||
"""
|
||||
file_name = os.path.splitext(os.path.basename(filepath))[0]
|
||||
mime_type = get_file_type(filepath)
|
||||
tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR)
|
||||
# use old file name with pdf extension
|
||||
if mime_type == "image/tiff":
|
||||
newpath = os.path.join(tempdir, file_name + ".pdf")
|
||||
else:
|
||||
logger.warning(
|
||||
f"Cannot convert mime type {str(mime_type)} from {str(filepath)} to pdf.",
|
||||
)
|
||||
return None
|
||||
with Image.open(filepath) as image:
|
||||
images = []
|
||||
for i, page in enumerate(ImageSequence.Iterator(image)):
|
||||
page = page.convert("RGB")
|
||||
images.append(page)
|
||||
try:
|
||||
if len(images) == 1:
|
||||
images[0].save(newpath)
|
||||
else:
|
||||
images[0].save(newpath, save_all=True, append_images=images[1:])
|
||||
except OSError as e:
|
||||
logger.warning(
|
||||
f"Could not save the file as pdf. Error: {str(e)}",
|
||||
)
|
||||
return None
|
||||
return newpath
|
||||
|
||||
|
||||
def scan_file_for_separating_barcodes(filepath: str) -> List[int]:
|
||||
"""
|
||||
Scan the provided pdf file for page separating barcodes
|
||||
Returns a list of pagenumbers, which separate the file
|
||||
"""
|
||||
separator_page_numbers = []
|
||||
separator_barcode = str(settings.CONSUMER_BARCODE_STRING)
|
||||
# use a temporary directory in case the file os too big to handle in memory
|
||||
with tempfile.TemporaryDirectory() as path:
|
||||
pages_from_path = convert_from_path(filepath, output_folder=path)
|
||||
for current_page_number, page in enumerate(pages_from_path):
|
||||
current_barcodes = barcode_reader(page)
|
||||
if separator_barcode in current_barcodes:
|
||||
separator_page_numbers.append(current_page_number)
|
||||
return separator_page_numbers
|
||||
|
||||
|
||||
def separate_pages(filepath: str, pages_to_split_on: List[int]) -> List[str]:
|
||||
"""
|
||||
Separate the provided pdf file on the pages_to_split_on.
|
||||
The pages which are defined by page_numbers will be removed.
|
||||
Returns a list of (temporary) filepaths to consume.
|
||||
These will need to be deleted later.
|
||||
"""
|
||||
os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
|
||||
tempdir = tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR)
|
||||
fname = os.path.splitext(os.path.basename(filepath))[0]
|
||||
pdf = Pdf.open(filepath)
|
||||
document_paths = []
|
||||
logger.debug(f"Temp dir is {str(tempdir)}")
|
||||
if not pages_to_split_on:
|
||||
logger.warning("No pages to split on!")
|
||||
else:
|
||||
# go from the first page to the first separator page
|
||||
dst = Pdf.new()
|
||||
for n, page in enumerate(pdf.pages):
|
||||
if n < pages_to_split_on[0]:
|
||||
dst.pages.append(page)
|
||||
output_filename = f"{fname}_document_0.pdf"
|
||||
savepath = os.path.join(tempdir, output_filename)
|
||||
with open(savepath, "wb") as out:
|
||||
dst.save(out)
|
||||
document_paths = [savepath]
|
||||
|
||||
# iterate through the rest of the document
|
||||
for count, page_number in enumerate(pages_to_split_on):
|
||||
logger.debug(f"Count: {str(count)} page_number: {str(page_number)}")
|
||||
dst = Pdf.new()
|
||||
try:
|
||||
next_page = pages_to_split_on[count + 1]
|
||||
except IndexError:
|
||||
next_page = len(pdf.pages)
|
||||
# skip the first page_number. This contains the barcode page
|
||||
for page in range(page_number + 1, next_page):
|
||||
logger.debug(
|
||||
f"page_number: {str(page_number)} next_page: {str(next_page)}",
|
||||
)
|
||||
dst.pages.append(pdf.pages[page])
|
||||
output_filename = f"{fname}_document_{str(count + 1)}.pdf"
|
||||
logger.debug(f"pdf no:{str(count)} has {str(len(dst.pages))} pages")
|
||||
savepath = os.path.join(tempdir, output_filename)
|
||||
with open(savepath, "wb") as out:
|
||||
dst.save(out)
|
||||
document_paths.append(savepath)
|
||||
logger.debug(f"Temp files are {str(document_paths)}")
|
||||
return document_paths
|
||||
|
||||
|
||||
def save_to_dir(
|
||||
filepath: str,
|
||||
newname: str = None,
|
||||
target_dir: str = settings.CONSUMPTION_DIR,
|
||||
):
|
||||
"""
|
||||
Copies filepath to target_dir.
|
||||
Optionally rename the file.
|
||||
"""
|
||||
if os.path.isfile(filepath) and os.path.isdir(target_dir):
|
||||
dst = shutil.copy(filepath, target_dir)
|
||||
logging.debug(f"saved {str(filepath)} to {str(dst)}")
|
||||
if newname:
|
||||
dst_new = os.path.join(target_dir, newname)
|
||||
logger.debug(f"moving {str(dst)} to {str(dst_new)}")
|
||||
os.rename(dst, dst_new)
|
||||
else:
|
||||
logger.warning(f"{str(filepath)} or {str(target_dir)} don't exist.")
|
||||
|
||||
|
||||
def consume_file(
|
||||
path,
|
||||
override_filename=None,
|
||||
@@ -245,32 +82,31 @@ def consume_file(
|
||||
|
||||
# check for separators in current document
|
||||
if settings.CONSUMER_ENABLE_BARCODES:
|
||||
separators = []
|
||||
document_list = []
|
||||
converted_tiff = None
|
||||
if settings.CONSUMER_BARCODE_TIFF_SUPPORT:
|
||||
supported_mime = ["image/tiff", "application/pdf"]
|
||||
else:
|
||||
supported_mime = ["application/pdf"]
|
||||
mime_type = get_file_type(path)
|
||||
if mime_type not in supported_mime:
|
||||
|
||||
mime_type = barcodes.get_file_mime_type(path)
|
||||
|
||||
if not barcodes.supported_file_type(mime_type):
|
||||
# if not supported, skip this routine
|
||||
logger.warning(
|
||||
f"Unsupported file format for barcode reader: {str(mime_type)}",
|
||||
)
|
||||
else:
|
||||
separators = []
|
||||
document_list = []
|
||||
converted_tiff = None
|
||||
|
||||
if mime_type == "image/tiff":
|
||||
file_to_process = convert_from_tiff_to_pdf(path)
|
||||
file_to_process = barcodes.convert_from_tiff_to_pdf(path)
|
||||
else:
|
||||
file_to_process = path
|
||||
|
||||
separators = scan_file_for_separating_barcodes(file_to_process)
|
||||
separators = barcodes.scan_file_for_separating_barcodes(file_to_process)
|
||||
|
||||
if separators:
|
||||
logger.debug(
|
||||
f"Pages with separators found in: {str(path)}",
|
||||
)
|
||||
document_list = separate_pages(file_to_process, separators)
|
||||
document_list = barcodes.separate_pages(file_to_process, separators)
|
||||
|
||||
if document_list:
|
||||
for n, document in enumerate(document_list):
|
||||
@@ -280,10 +116,10 @@ def consume_file(
|
||||
newname = f"{str(n)}_" + override_filename
|
||||
else:
|
||||
newname = None
|
||||
save_to_dir(document, newname=newname)
|
||||
barcodes.save_to_dir(document, newname=newname)
|
||||
# if we got here, the document was successfully split
|
||||
# and can safely be deleted
|
||||
if converted_tiff:
|
||||
if converted_tiff is not None:
|
||||
logger.debug(f"Deleting file {file_to_process}")
|
||||
os.unlink(file_to_process)
|
||||
logger.debug(f"Deleting file {path}")
|
||||
|
||||
Reference in New Issue
Block a user