From 2fe9b0cbc129929127482771f5c4053a6ae09cc6 Mon Sep 17 00:00:00 2001 From: Daniel Quinn Date: Sat, 27 Feb 2016 20:18:50 +0000 Subject: [PATCH] New logging appears to work --- docs/changelog.rst | 2 + src/documents/admin.py | 43 +++--- src/documents/consumer.py | 109 ++++++++------- src/documents/loggers.py | 30 +++++ src/documents/mail.py | 48 ++++--- .../migrations/0010_log.py} | 18 +-- src/documents/models.py | 32 +++++ src/documents/tests/test_logger.py | 124 ++++++++++++++++++ src/logger/__init__.py | 0 src/logger/admin.py | 12 -- src/logger/apps.py | 5 - src/logger/migrations/__init__.py | 0 src/logger/models.py | 53 -------- src/logger/tests.py | 3 - src/logger/views.py | 3 - src/paperless/settings.py | 52 +++++--- 16 files changed, 346 insertions(+), 188 deletions(-) create mode 100644 src/documents/loggers.py rename src/{logger/migrations/0001_initial.py => documents/migrations/0010_log.py} (57%) create mode 100644 src/documents/tests/test_logger.py delete mode 100644 src/logger/__init__.py delete mode 100644 src/logger/admin.py delete mode 100644 src/logger/apps.py delete mode 100644 src/logger/migrations/__init__.py delete mode 100644 src/logger/models.py delete mode 100644 src/logger/tests.py delete mode 100644 src/logger/views.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 86f365653..cdb720926 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,7 @@ Changelog * 0.1.1 (master) + * `#60`_: Setup logging to actually use the Python native logging framework. * `#53`_: Fixed an annoying bug that caused ``.jpeg`` and ``.JPG`` images to be imported but made unavailable. @@ -73,3 +74,4 @@ Changelog .. _#53: https://github.com/danielquinn/paperless/issues/53 .. _#54: https://github.com/danielquinn/paperless/issues/54 .. _#57: https://github.com/danielquinn/paperless/issues/57 +.. _#60: https://github.com/danielquinn/paperless/issues/60 diff --git a/src/documents/admin.py b/src/documents/admin.py index 42c3fc968..118a295eb 100644 --- a/src/documents/admin.py +++ b/src/documents/admin.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User, Group from django.core.urlresolvers import reverse from django.templatetags.static import static -from .models import Sender, Tag, Document +from .models import Sender, Tag, Document, Log class MonthListFilter(admin.SimpleListFilter): @@ -57,7 +57,7 @@ class DocumentAdmin(admin.ModelAdmin): r = "" for tag in obj.tags.all(): colour = tag.get_colour_display() - r += html_tag( + r += self._html_tag( "a", tag.slug, **{ @@ -73,9 +73,9 @@ class DocumentAdmin(admin.ModelAdmin): tags_.allow_tags = True def document(self, obj): - return html_tag( + return self._html_tag( "a", - html_tag( + self._html_tag( "img", src=static("documents/img/{}.png".format(obj.file_type)), width=22, @@ -87,23 +87,32 @@ class DocumentAdmin(admin.ModelAdmin): ) document.allow_tags = True + @staticmethod + def _html_tag(kind, inside=None, **kwargs): + + attributes = [] + for lft, rgt in kwargs.items(): + attributes.append('{}="{}"'.format(lft, rgt)) + + if inside is not None: + return "<{kind} {attributes}>{inside}".format( + kind=kind, attributes=" ".join(attributes), inside=inside) + + return "<{} {}/>".format(kind, " ".join(attributes)) + + +class LogAdmin(admin.ModelAdmin): + + list_display = ("message", "level", "component") + list_filter = ("level", "component",) + + admin.site.register(Sender) admin.site.register(Tag, TagAdmin) admin.site.register(Document, DocumentAdmin) +admin.site.register(Log, LogAdmin) + # Unless we implement multi-user, these default registrations don't make sense. admin.site.unregister(Group) admin.site.unregister(User) - - -def html_tag(kind, inside=None, **kwargs): - - attributes = [] - for lft, rgt in kwargs.items(): - attributes.append('{}="{}"'.format(lft, rgt)) - - if inside is not None: - return "<{kind} {attributes}>{inside}".format( - kind=kind, attributes=" ".join(attributes), inside=inside) - - return "<{} {}/>".format(kind, " ".join(attributes)) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index ddbe474a7..37b348495 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -1,5 +1,8 @@ import datetime +import logging import tempfile +import uuid + from multiprocessing.pool import Pool import itertools @@ -19,10 +22,9 @@ from django.utils import timezone from django.template.defaultfilters import slugify from pyocr.tesseract import TesseractError -from logger.models import Log from paperless.db import GnuPG -from .models import Sender, Tag, Document +from .models import Sender, Tag, Document, Log from .languages import ISO639 @@ -67,6 +69,8 @@ class Consumer(object): def __init__(self, verbosity=1): self.verbosity = verbosity + self.logger = logging.getLogger(__name__) + self.logging_group = None try: os.makedirs(self.SCRATCH) @@ -86,6 +90,12 @@ class Consumer(object): raise ConsumerError( "Consumption directory {} does not exist".format(self.CONSUME)) + def log(self, level, message): + getattr(self.logger, level)(message, extra={ + "group": self.logging_group, + "component": Log.COMPONENT_CONSUMER + }) + def consume(self): for doc in os.listdir(self.CONSUME): @@ -104,7 +114,9 @@ class Consumer(object): if self._is_ready(doc): continue - Log.info("Consuming {}".format(doc), Log.COMPONENT_CONSUMER) + self.logging_group = uuid.uuid4() + + self.log("info", "Consuming {}".format(doc)) tempdir = tempfile.mkdtemp(prefix="paperless", dir=self.SCRATCH) pngs = self._get_greyscale(tempdir, doc) @@ -114,8 +126,7 @@ class Consumer(object): self._store(text, doc) except OCRError: self._ignore.append(doc) - Log.error( - "OCR FAILURE: {}".format(doc), Log.COMPONENT_CONSUMER) + self.log("error", "OCR FAILURE: {}".format(doc)) self._cleanup_tempdir(tempdir) continue else: @@ -124,10 +135,7 @@ class Consumer(object): def _get_greyscale(self, tempdir, doc): - Log.debug( - "Generating greyscale image from {}".format(doc), - Log.COMPONENT_CONSUMER - ) + self.log("info", "Generating greyscale image from {}".format(doc)) png = os.path.join(tempdir, "convert-%04d.jpg") @@ -143,18 +151,13 @@ class Consumer(object): return sorted(filter(lambda __: os.path.isfile(__), pngs)) - @staticmethod - def _guess_language(text): + def _guess_language(self, text): try: guess = langdetect.detect(text) - Log.debug( - "Language detected: {}".format(guess), - Log.COMPONENT_CONSUMER - ) + self.log("debug", "Language detected: {}".format(guess)) return guess except Exception as e: - Log.warning( - "Language detection error: {}".format(e), Log.COMPONENT_MAIL) + self.log("warning", "Language detection error: {}".format(e)) def _get_ocr(self, pngs): """ @@ -165,7 +168,7 @@ class Consumer(object): if not pngs: raise OCRError - Log.debug("OCRing the document", Log.COMPONENT_CONSUMER) + self.log("info", "OCRing the document") # Since the division gets rounded down by int, this calculation works # for every edge-case, i.e. 1 @@ -175,12 +178,12 @@ class Consumer(object): guessed_language = self._guess_language(raw_text) if not guessed_language or guessed_language not in ISO639: - Log.warning("Language detection failed!", Log.COMPONENT_CONSUMER) + self.log("warning", "Language detection failed!") if settings.FORGIVING_OCR: - Log.warning( + self.log( + "warning", "As FORGIVING_OCR is enabled, we're going to make the " - "best with what we have.", - Log.COMPONENT_CONSUMER + "best with what we have." ) raw_text = self._assemble_ocr_sections(pngs, middle, raw_text) return raw_text @@ -194,12 +197,12 @@ class Consumer(object): return self._ocr(pngs, ISO639[guessed_language]) except pyocr.pyocr.tesseract.TesseractError: if settings.FORGIVING_OCR: - Log.warning( + self.log( + "warning", "OCR for {} failed, but we're going to stick with what " "we've got since FORGIVING_OCR is enabled.".format( guessed_language - ), - Log.COMPONENT_CONSUMER + ) ) raw_text = self._assemble_ocr_sections(pngs, middle, raw_text) return raw_text @@ -222,28 +225,15 @@ class Consumer(object): if not pngs: return "" - Log.debug("Parsing for {}".format(lang), Log.COMPONENT_CONSUMER) + self.log("info", "Parsing for {}".format(lang)) with Pool(processes=self.THREADS) as pool: - r = pool.map( - self.image_to_string, itertools.product(pngs, [lang])) + r = pool.map(image_to_string, itertools.product(pngs, [lang])) r = " ".join(r) # Strip out excess white space to allow matching to go smoother return re.sub(r"\s+", " ", r) - def image_to_string(self, args): - png, lang = args - ocr = pyocr.get_available_tools()[0] - with Image.open(os.path.join(self.SCRATCH, png)) as f: - if ocr.can_detect_orientation(): - try: - orientation = ocr.detect_orientation(f, lang=lang) - f = f.rotate(orientation["angle"], expand=1) - except TesseractError: - pass - return ocr.image_to_string(f, lang=lang) - def _guess_attributes_from_name(self, parseable): """ We use a crude naming convention to make handling the sender, title, @@ -301,7 +291,7 @@ class Consumer(object): stats = os.stat(doc) - Log.debug("Saving record to database", Log.COMPONENT_CONSUMER) + self.log("debug", "Saving record to database") document = Document.objects.create( sender=sender, @@ -316,23 +306,22 @@ class Consumer(object): if relevant_tags: tag_names = ", ".join([t.slug for t in relevant_tags]) - Log.debug( - "Tagging with {}".format(tag_names), Log.COMPONENT_CONSUMER) + self.log("debug", "Tagging with {}".format(tag_names)) document.tags.add(*relevant_tags) with open(doc, "rb") as unencrypted: with open(document.source_path, "wb") as encrypted: - Log.debug("Encrypting", Log.COMPONENT_CONSUMER) + self.log("debug", "Encrypting") encrypted.write(GnuPG.encrypted(unencrypted)) - @staticmethod - def _cleanup_tempdir(d): - Log.debug("Deleting directory {}".format(d), Log.COMPONENT_CONSUMER) + self.log("info", "Completed") + + def _cleanup_tempdir(self, d): + self.log("debug", "Deleting directory {}".format(d)) shutil.rmtree(d) - @staticmethod - def _cleanup_doc(doc): - Log.debug("Deleting document {}".format(doc), Log.COMPONENT_CONSUMER) + def _cleanup_doc(self, doc): + self.log("debug", "Deleting document {}".format(doc)) os.unlink(doc) def _is_ready(self, doc): @@ -350,3 +339,23 @@ class Consumer(object): self.stats[doc] = t return False + + +def image_to_string(args): + """ + I have no idea why, but if this function were a method of Consumer, it + would explode with: + + `TypeError: cannot serialize '_io.TextIOWrapper' object`. + """ + + png, lang = args + ocr = pyocr.get_available_tools()[0] + with Image.open(os.path.join(Consumer.SCRATCH, png)) as f: + if ocr.can_detect_orientation(): + try: + orientation = ocr.detect_orientation(f, lang=lang) + f = f.rotate(orientation["angle"], expand=1) + except TesseractError: + pass + return ocr.image_to_string(f, lang=lang) diff --git a/src/documents/loggers.py b/src/documents/loggers.py new file mode 100644 index 000000000..3464478cc --- /dev/null +++ b/src/documents/loggers.py @@ -0,0 +1,30 @@ +import logging + + +class PaperlessLogger(logging.StreamHandler): + """ + A logger smart enough to know to log some kinds of messages to the database + for later retrieval in a pretty interface. + """ + + def emit(self, record): + + logging.StreamHandler.emit(self, record) + + if not hasattr(record, "component"): + return + + # We have to do the import here or Django will barf when it tries to + # load this because the apps aren't loaded at that point + from .models import Log + + kwargs = { + "message": record.msg, + "component": record.component, + "level": record.levelno, + } + + if hasattr(record, "group"): + kwargs["group"] = record.group + + Log.objects.create(**kwargs) diff --git a/src/documents/mail.py b/src/documents/mail.py index 384567e60..cc987bf64 100644 --- a/src/documents/mail.py +++ b/src/documents/mail.py @@ -1,8 +1,10 @@ import datetime import imaplib +import logging import os import re import time +import uuid from base64 import b64decode from email import policy @@ -11,10 +13,8 @@ from dateutil import parser from django.conf import settings -from logger.models import Log - from .consumer import Consumer -from .models import Sender +from .models import Sender, Log class MailFetcherError(Exception): @@ -25,7 +25,20 @@ class InvalidMessageError(Exception): pass -class Message(object): +class Loggable(object): + + def __init__(self, group=None): + self.logger = logging.getLogger(__name__) + self.logging_group = group or uuid.uuid4() + + def log(self, level, message): + getattr(self.logger, level)(message, extra={ + "group": self.logging_group, + "component": Log.COMPONENT_MAIL + }) + + +class Message(Loggable): """ A crude, but simple email message class. We assume that there's a subject and n attachments, and that we don't care about the message body. @@ -33,13 +46,13 @@ class Message(object): SECRET = settings.UPLOAD_SHARED_SECRET - def __init__(self, data, verbosity=1): + def __init__(self, data, group=None): """ Cribbed heavily from https://www.ianlewis.org/en/parsing-email-attachments-python """ - self.verbosity = verbosity + Loggable.__init__(self, group=group) self.subject = None self.time = None @@ -54,8 +67,7 @@ class Message(object): self._set_time(message) - Log.info( - 'Importing email: "{}"'.format(self.subject), Log.COMPONENT_MAIL) + self.log("info", 'Importing email: "{}"'.format(self.subject)) attachments = [] for part in message.walk(): @@ -134,9 +146,11 @@ class Attachment(object): return self.data -class MailFetcher(object): +class MailFetcher(Loggable): - def __init__(self, verbosity=1): + def __init__(self): + + Loggable.__init__(self) self._connection = None self._host = settings.MAIL_CONSUMPTION["HOST"] @@ -148,7 +162,6 @@ class MailFetcher(object): self._enabled = bool(self._host) self.last_checked = datetime.datetime.now() - self.verbosity = verbosity def pull(self): """ @@ -159,14 +172,11 @@ class MailFetcher(object): if self._enabled: - Log.info("Checking mail", Log.COMPONENT_MAIL) + self.log("info", "Checking mail") for message in self._get_messages(): - Log.debug( - 'Storing email: "{}"'.format(message.subject), - Log.COMPONENT_MAIL - ) + self.log("info", 'Storing email: "{}"'.format(message.subject)) t = int(time.mktime(message.time.timetuple())) file_name = os.path.join(Consumer.CONSUME, message.file_name) @@ -193,7 +203,7 @@ class MailFetcher(object): self._connection.logout() except Exception as e: - Log.error(e, Log.COMPONENT_MAIL) + self.log("error", str(e)) return r @@ -218,9 +228,9 @@ class MailFetcher(object): message = None try: - message = Message(data[0][1], self.verbosity) + message = Message(data[0][1], self.logging_group) except InvalidMessageError as e: - Log.error(e, Log.COMPONENT_MAIL) + self.log("error", str(e)) else: self._connection.store(num, "+FLAGS", "\\Deleted") diff --git a/src/logger/migrations/0001_initial.py b/src/documents/migrations/0010_log.py similarity index 57% rename from src/logger/migrations/0001_initial.py rename to src/documents/migrations/0010_log.py index 029fe43c2..57cf804b7 100644 --- a/src/logger/migrations/0001_initial.py +++ b/src/documents/migrations/0010_log.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.9 on 2016-02-14 16:08 +# Generated by Django 1.9 on 2016-02-27 17:54 from __future__ import unicode_literals from django.db import migrations, models @@ -7,9 +7,8 @@ from django.db import migrations, models class Migration(migrations.Migration): - initial = True - dependencies = [ + ('documents', '0009_auto_20160214_0040'), ] operations = [ @@ -17,14 +16,15 @@ class Migration(migrations.Migration): name='Log', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('time', models.DateTimeField(auto_now_add=True)), + ('group', models.UUIDField(blank=True)), ('message', models.TextField()), - ('level', models.PositiveIntegerField(choices=[(1, 'Error'), (2, 'Warning'), (3, 'Informational'), (4, 'Debugging')], default=3)), + ('level', models.PositiveIntegerField(choices=[(10, 'Debugging'), (20, 'Informational'), (30, 'Warning'), (40, 'Error'), (50, 'Critical')], default=20)), ('component', models.PositiveIntegerField(choices=[(1, 'Consumer'), (2, 'Mail Fetcher')])), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), ], - ), - migrations.AlterModelOptions( - name='log', - options={'ordering': ('-time',)}, + options={ + 'ordering': ('-modified',), + }, ), ] diff --git a/src/documents/models.py b/src/documents/models.py index 267bebffe..91dd458ea 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -1,3 +1,4 @@ +import logging import os import re @@ -187,3 +188,34 @@ class Document(models.Model): @property def download_url(self): return reverse("fetch", kwargs={"pk": self.pk}) + + +class Log(models.Model): + + LEVELS = ( + (logging.DEBUG, "Debugging"), + (logging.INFO, "Informational"), + (logging.WARNING, "Warning"), + (logging.ERROR, "Error"), + (logging.CRITICAL, "Critical"), + ) + + COMPONENT_CONSUMER = 1 + COMPONENT_MAIL = 2 + COMPONENTS = ( + (COMPONENT_CONSUMER, "Consumer"), + (COMPONENT_MAIL, "Mail Fetcher") + ) + + group = models.UUIDField(blank=True) + message = models.TextField() + level = models.PositiveIntegerField(choices=LEVELS, default=logging.INFO) + component = models.PositiveIntegerField(choices=COMPONENTS) + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + class Meta(object): + ordering = ("-modified",) + + def __str__(self): + return self.message diff --git a/src/documents/tests/test_logger.py b/src/documents/tests/test_logger.py new file mode 100644 index 000000000..d5527d7c6 --- /dev/null +++ b/src/documents/tests/test_logger.py @@ -0,0 +1,124 @@ +import logging +import uuid + +from unittest import mock + +from django.test import TestCase + +from ..models import Log + + +class TestPaperlessLog(TestCase): + + def __init__(self, *args, **kwargs): + TestCase.__init__(self, *args, **kwargs) + self.logger = logging.getLogger( + "documents.management.commands.document_consumer") + + def test_ignored(self): + with mock.patch("logging.StreamHandler.emit") as __: + self.assertEqual(Log.objects.all().count(), 0) + self.logger.info("This is an informational message") + self.logger.warning("This is an informational message") + self.logger.error("This is an informational message") + self.logger.critical("This is an informational message") + self.assertEqual(Log.objects.all().count(), 0) + + def test_that_it_saves_at_all(self): + + kw = { + "group": uuid.uuid4(), + "component": Log.COMPONENT_MAIL + } + + self.assertEqual(Log.objects.all().count(), 0) + + with mock.patch("logging.StreamHandler.emit") as __: + + # Debug messages are ignored by default + self.logger.debug("This is a debugging message", extra=kw) + self.assertEqual(Log.objects.all().count(), 0) + + self.logger.info("This is an informational message", extra=kw) + self.assertEqual(Log.objects.all().count(), 1) + + self.logger.warning("This is an warning message", extra=kw) + self.assertEqual(Log.objects.all().count(), 2) + + self.logger.error("This is an error message", extra=kw) + self.assertEqual(Log.objects.all().count(), 3) + + self.logger.critical("This is a critical message", extra=kw) + self.assertEqual(Log.objects.all().count(), 4) + + def test_groups(self): + + kw1 = { + "group": uuid.uuid4(), + "component": Log.COMPONENT_MAIL + } + kw2 = { + "group": uuid.uuid4(), + "component": Log.COMPONENT_MAIL + } + + self.assertEqual(Log.objects.all().count(), 0) + + with mock.patch("logging.StreamHandler.emit") as __: + + # Debug messages are ignored by default + self.logger.debug("This is a debugging message", extra=kw1) + self.assertEqual(Log.objects.all().count(), 0) + + self.logger.info("This is an informational message", extra=kw2) + self.assertEqual(Log.objects.all().count(), 1) + self.assertEqual(Log.objects.filter(group=kw2["group"]).count(), 1) + + self.logger.warning("This is an warning message", extra=kw1) + self.assertEqual(Log.objects.all().count(), 2) + self.assertEqual(Log.objects.filter(group=kw1["group"]).count(), 1) + + self.logger.error("This is an error message", extra=kw2) + self.assertEqual(Log.objects.all().count(), 3) + self.assertEqual(Log.objects.filter(group=kw2["group"]).count(), 2) + + self.logger.critical("This is a critical message", extra=kw1) + self.assertEqual(Log.objects.all().count(), 4) + self.assertEqual(Log.objects.filter(group=kw1["group"]).count(), 2) + + def test_components(self): + + c1 = Log.COMPONENT_CONSUMER + c2 = Log.COMPONENT_MAIL + kw1 = { + "group": uuid.uuid4(), + "component": c1 + } + kw2 = { + "group": kw1["group"], + "component": c2 + } + + self.assertEqual(Log.objects.all().count(), 0) + + with mock.patch("logging.StreamHandler.emit") as __: + + # Debug messages are ignored by default + self.logger.debug("This is a debugging message", extra=kw1) + self.assertEqual(Log.objects.all().count(), 0) + + self.logger.info("This is an informational message", extra=kw2) + self.assertEqual(Log.objects.all().count(), 1) + self.assertEqual(Log.objects.filter(component=c2).count(), 1) + + self.logger.warning("This is an warning message", extra=kw1) + self.assertEqual(Log.objects.all().count(), 2) + self.assertEqual(Log.objects.filter(component=c1).count(), 1) + + self.logger.error("This is an error message", extra=kw2) + self.assertEqual(Log.objects.all().count(), 3) + self.assertEqual(Log.objects.filter(component=c2).count(), 2) + + self.logger.critical("This is a critical message", extra=kw1) + self.assertEqual(Log.objects.all().count(), 4) + self.assertEqual(Log.objects.filter(component=c1).count(), 2) diff --git a/src/logger/__init__.py b/src/logger/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/logger/admin.py b/src/logger/admin.py deleted file mode 100644 index dc9446821..000000000 --- a/src/logger/admin.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.contrib import admin - -from .models import Log - - -class LogAdmin(admin.ModelAdmin): - - list_display = ("message", "level", "component") - list_filter = ("level", "component",) - - -admin.site.register(Log, LogAdmin) diff --git a/src/logger/apps.py b/src/logger/apps.py deleted file mode 100644 index 2c1a7d735..000000000 --- a/src/logger/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class LoggerConfig(AppConfig): - name = 'logger' diff --git a/src/logger/migrations/__init__.py b/src/logger/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/logger/models.py b/src/logger/models.py deleted file mode 100644 index f7f2c421a..000000000 --- a/src/logger/models.py +++ /dev/null @@ -1,53 +0,0 @@ -from django.db import models - - -class Log(models.Model): - - LEVEL_ERROR = 1 - LEVEL_WARNING = 2 - LEVEL_INFO = 3 - LEVEL_DEBUG = 4 - LEVELS = ( - (LEVEL_ERROR, "Error"), - (LEVEL_WARNING, "Warning"), - (LEVEL_INFO, "Informational"), - (LEVEL_DEBUG, "Debugging"), - ) - - COMPONENT_CONSUMER = 1 - COMPONENT_MAIL = 2 - COMPONENTS = ( - (COMPONENT_CONSUMER, "Consumer"), - (COMPONENT_MAIL, "Mail Fetcher") - ) - - time = models.DateTimeField(auto_now_add=True) - message = models.TextField() - level = models.PositiveIntegerField(choices=LEVELS, default=LEVEL_INFO) - component = models.PositiveIntegerField(choices=COMPONENTS) - - class Meta(object): - ordering = ("-time",) - - def __str__(self): - return self.message - - @classmethod - def error(cls, message, component): - cls.objects.create( - message=message, level=cls.LEVEL_ERROR, component=component) - - @classmethod - def warning(cls, message, component): - cls.objects.create( - message=message, level=cls.LEVEL_WARNING, component=component) - - @classmethod - def info(cls, message, component): - cls.objects.create( - message=message, level=cls.LEVEL_INFO, component=component) - - @classmethod - def debug(cls, message, component): - cls.objects.create( - message=message, level=cls.LEVEL_DEBUG, component=component) diff --git a/src/logger/tests.py b/src/logger/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/src/logger/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/src/logger/views.py b/src/logger/views.py deleted file mode 100644 index 91ea44a21..000000000 --- a/src/logger/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 5d7cc3b2f..1f7bb6d0a 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -42,7 +42,6 @@ INSTALLED_APPS = [ "django_extensions", "documents", - "logger", "rest_framework", @@ -89,12 +88,12 @@ DATABASES = { "NAME": os.path.join(BASE_DIR, "..", "data", "db.sqlite3"), } } -if os.environ.get("PAPERLESS_DBUSER") and os.environ.get("PAPERLESS_DBPASS"): +if os.getenv("PAPERLESS_DBUSER") and os.getenv("PAPERLESS_DBPASS"): DATABASES["default"] = { "ENGINE": "django.db.backends.postgresql_psycopg2", - "NAME": os.environ.get("PAPERLESS_DBNAME", "paperless"), - "USER": os.environ.get("PAPERLESS_DBUSER"), - "PASSWORD": os.environ.get("PAPERLESS_DBPASS") + "NAME": os.getenv("PAPERLESS_DBNAME", "paperless"), + "USER": os.getenv("PAPERLESS_DBUSER"), + "PASSWORD": os.getenv("PAPERLESS_DBPASS") } @@ -141,6 +140,25 @@ STATIC_URL = '/static/' MEDIA_URL = "/media/" +# Logging + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "consumer": { + "class": "documents.loggers.PaperlessLogger", + } + }, + "loggers": { + "documents": { + "handlers": ["consumer"], + "level": os.getenv("PAPERLESS_CONSUMER_LOG_LEVEL", "INFO"), + }, + }, +} + + # Paperless-specific stuffs # Change these paths if yours are different # ---------------------------------------------------------------------------- @@ -150,15 +168,15 @@ MEDIA_URL = "/media/" OCR_LANGUAGE = "eng" # The amount of threads to use for OCR -OCR_THREADS = os.environ.get("PAPERLESS_OCR_THREADS") +OCR_THREADS = os.getenv("PAPERLESS_OCR_THREADS") -# If this is true, any failed attempts to OCR a PDF will result in the PDF being -# indexed anyway, with whatever we could get. If it's False, the file will -# simply be left in the CONSUMPTION_DIR. +# If this is true, any failed attempts to OCR a PDF will result in the PDF +# being indexed anyway, with whatever we could get. If it's False, the file +# will simply be left in the CONSUMPTION_DIR. FORGIVING_OCR = True # GNUPG needs a home directory for some reason -GNUPG_HOME = os.environ.get("HOME", "/tmp") +GNUPG_HOME = os.getenv("HOME", "/tmp") # Convert is part of the Imagemagick package CONVERT_BINARY = "/usr/bin/convert" @@ -167,16 +185,16 @@ CONVERT_BINARY = "/usr/bin/convert" SCRATCH_DIR = "/tmp/paperless" # This is where Paperless will look for PDFs to index -CONSUMPTION_DIR = os.environ.get("PAPERLESS_CONSUME") +CONSUMPTION_DIR = os.getenv("PAPERLESS_CONSUME") # If you want to use IMAP mail consumption, populate this with useful values. # If you leave HOST set to None, we assume you're not going to use this # feature. MAIL_CONSUMPTION = { - "HOST": os.environ.get("PAPERLESS_CONSUME_MAIL_HOST"), - "PORT": os.environ.get("PAPERLESS_CONSUME_MAIL_PORT"), - "USERNAME": os.environ.get("PAPERLESS_CONSUME_MAIL_USER"), - "PASSWORD": os.environ.get("PAPERLESS_CONSUME_MAIL_PASS"), + "HOST": os.getenv("PAPERLESS_CONSUME_MAIL_HOST"), + "PORT": os.getenv("PAPERLESS_CONSUME_MAIL_PORT"), + "USERNAME": os.getenv("PAPERLESS_CONSUME_MAIL_USER"), + "PASSWORD": os.getenv("PAPERLESS_CONSUME_MAIL_PASS"), "USE_SSL": True, # If True, use SSL/TLS to connect "INBOX": "INBOX" # The name of the inbox on the server } @@ -188,9 +206,9 @@ MAIL_CONSUMPTION = { # DON'T FORGET TO SET THIS as leaving it blank may cause some strange things # with GPG, including an interesting case where it may "encrypt" zero-byte # files. -PASSPHRASE = os.environ.get("PAPERLESS_PASSPHRASE") +PASSPHRASE = os.getenv("PAPERLESS_PASSPHRASE") # If you intend to use the "API" to push files into the consumer, you'll need # to provide a shared secret here. Leaving this as the default will disable # the API. -UPLOAD_SHARED_SECRET = os.environ.get("PAPERLESS_SECRET", "") +UPLOAD_SHARED_SECRET = os.getenv("PAPERLESS_SECRET", "")