paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-04-09 09:58:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

Added test for duplicates

Browse Source
This commit is contained in:
Daniel Quinn 2016-04-03 18:44:00 +01:00
parent 2853545b9d
commit 64b72d4337
1 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/documents/consumer.py
View File

@ -1,4 +1,5 @@
import datetime import datetime
import hashlib
import logging import logging
import tempfile import tempfile
import uuid import uuid
@ -101,6 +102,14 @@ class Consumer(object):
if self._is_ready(doc): if self._is_ready(doc):
continue continue
if self._is_duplicate(doc):
self.log(
"info",
"Skipping {} as it appears to be a duplicate".format(doc)
)
self._ignore.append(doc)
continue
self.logging_group = uuid.uuid4() self.logging_group = uuid.uuid4()
self.log("info", "Consuming {}".format(doc)) self.log("info", "Consuming {}".format(doc))
@ -340,6 +349,12 @@ class Consumer(object):
return False return False
@staticmethod
def _is_duplicate(doc):
with open(doc, "rb") as f:
checksum = hashlib.md5(f.read()).hexdigest()
return Document.objects.filter(checksum=checksum).exists()
def image_to_string(args): def image_to_string(args):
img, lang = args img, lang = args