Merge data models

src/documents/barcodes.py

@@ -14,13 +14,13 @@ from pikepdf import PasswordError
 from pikepdf import Pdf
 
 from documents.converters import convert_from_tiff_to_pdf
-from documents.data_models import ConsumableDocument
 from documents.plugins.base import ConsumeTaskPlugin
 from documents.plugins.base import StopConsumeTaskError
 from documents.plugins.helpers import ProgressStatusOptions
 from documents.utils import copy_basic_file_stats
 from documents.utils import copy_file_with_basic_stats
 from documents.utils import maybe_override_pixel_limit
+from paperless.data_models import ConsumableDocument
 from paperless.models import Tag
 
 if TYPE_CHECKING:

src/documents/bulk_edit.py

@@ -16,14 +16,14 @@ from django.conf import settings
 from django.db.models import Q
 from django.utils import timezone
 
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
-from documents.data_models import DocumentSource
 from documents.permissions import set_permissions_for_object
 from documents.plugins.helpers import DocumentsStatusManager
 from documents.tasks import bulk_update_documents
 from documents.tasks import consume_file
 from documents.tasks import update_document_content_maybe_archive_file
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
+from paperless.data_models import DocumentSource
 from paperless.models import Correspondent
 from paperless.models import CustomField
 from paperless.models import CustomFieldInstance

src/documents/consumer.py

@@ -16,8 +16,6 @@ from filelock import FileLock
 from rest_framework.reverse import reverse
 
 from documents.classifier import load_classifier
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
 from documents.file_handling import create_source_path_directory
 from documents.file_handling import generate_unique_filename
 from documents.loggers import LoggingMixin
@@ -39,6 +37,8 @@ from documents.templating.workflows import parse_w_workflow_placeholders
 from documents.utils import copy_basic_file_stats
 from documents.utils import copy_file_with_basic_stats
 from documents.utils import run_subprocess
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
 from paperless.models import Correspondent
 from paperless.models import CustomField
 from paperless.models import CustomFieldInstance

src/documents/data_models.py

@@ -1,174 +0,0 @@
-import dataclasses
-import datetime
-from enum import IntEnum
-from pathlib import Path
-
-import magic
-from guardian.shortcuts import get_groups_with_perms
-from guardian.shortcuts import get_users_with_perms
-
-
-@dataclasses.dataclass
-class DocumentMetadataOverrides:
-    """
-    Manages overrides for document fields which normally would
-    be set from content or matching.  All fields default to None,
-    meaning no override is happening
-    """
-
-    filename: str | None = None
-    title: str | None = None
-    correspondent_id: int | None = None
-    document_type_id: int | None = None
-    tag_ids: list[int] | None = None
-    storage_path_id: int | None = None
-    created: datetime.datetime | None = None
-    asn: int | None = None
-    owner_id: int | None = None
-    view_users: list[int] | None = None
-    view_groups: list[int] | None = None
-    change_users: list[int] | None = None
-    change_groups: list[int] | None = None
-    custom_fields: dict | None = None
-
-    def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
-        """
-        Merges two DocumentMetadataOverrides objects such that object B's overrides
-        are applied to object A or merged if multiple are accepted.
-
-        The update is an in-place modification of self
-        """
-        # only if empty
-        if other.title is not None:
-            self.title = other.title
-        if other.correspondent_id is not None:
-            self.correspondent_id = other.correspondent_id
-        if other.document_type_id is not None:
-            self.document_type_id = other.document_type_id
-        if other.storage_path_id is not None:
-            self.storage_path_id = other.storage_path_id
-        if other.owner_id is not None:
-            self.owner_id = other.owner_id
-
-        # merge
-        if self.tag_ids is None:
-            self.tag_ids = other.tag_ids
-        elif other.tag_ids is not None:
-            self.tag_ids.extend(other.tag_ids)
-            self.tag_ids = list(set(self.tag_ids))
-
-        if self.view_users is None:
-            self.view_users = other.view_users
-        elif other.view_users is not None:
-            self.view_users.extend(other.view_users)
-            self.view_users = list(set(self.view_users))
-
-        if self.view_groups is None:
-            self.view_groups = other.view_groups
-        elif other.view_groups is not None:
-            self.view_groups.extend(other.view_groups)
-            self.view_groups = list(set(self.view_groups))
-
-        if self.change_users is None:
-            self.change_users = other.change_users
-        elif other.change_users is not None:
-            self.change_users.extend(other.change_users)
-            self.change_users = list(set(self.change_users))
-
-        if self.change_groups is None:
-            self.change_groups = other.change_groups
-        elif other.change_groups is not None:
-            self.change_groups.extend(other.change_groups)
-            self.change_groups = list(set(self.change_groups))
-
-        if self.custom_fields is None:
-            self.custom_fields = other.custom_fields
-        elif other.custom_fields is not None:
-            self.custom_fields.update(other.custom_fields)
-
-        return self
-
-    @staticmethod
-    def from_document(doc) -> "DocumentMetadataOverrides":
-        """
-        Fills in the overrides from a document object
-        """
-        overrides = DocumentMetadataOverrides()
-        overrides.title = doc.title
-        overrides.correspondent_id = doc.correspondent.id if doc.correspondent else None
-        overrides.document_type_id = doc.document_type.id if doc.document_type else None
-        overrides.storage_path_id = doc.storage_path.id if doc.storage_path else None
-        overrides.owner_id = doc.owner.id if doc.owner else None
-        overrides.tag_ids = list(doc.tags.values_list("id", flat=True))
-
-        overrides.view_users = list(
-            get_users_with_perms(
-                doc,
-                only_with_perms_in=["view_document"],
-            ).values_list("id", flat=True),
-        )
-        overrides.change_users = list(
-            get_users_with_perms(
-                doc,
-                only_with_perms_in=["change_document"],
-            ).values_list("id", flat=True),
-        )
-        overrides.custom_fields = {
-            custom_field.id: custom_field.value
-            for custom_field in doc.custom_fields.all()
-        }
-
-        groups_with_perms = get_groups_with_perms(
-            doc,
-            attach_perms=True,
-        )
-        overrides.view_groups = [
-            group.id
-            for group in groups_with_perms
-            if "view_document" in groups_with_perms[group]
-        ]
-        overrides.change_groups = [
-            group.id
-            for group in groups_with_perms
-            if "change_document" in groups_with_perms[group]
-        ]
-
-        return overrides
-
-
-class DocumentSource(IntEnum):
-    """
-    The source of an incoming document.  May have other uses in the future
-    """
-
-    ConsumeFolder = 1
-    ApiUpload = 2
-    MailFetch = 3
-    WebUI = 4
-
-
-@dataclasses.dataclass
-class ConsumableDocument:
-    """
-    Encapsulates an incoming document, either from consume folder, API upload
-    or mail fetching and certain useful operations on it.
-    """
-
-    source: DocumentSource
-    original_file: Path
-    mailrule_id: int | None = None
-    mime_type: str = dataclasses.field(init=False, default=None)
-
-    def __post_init__(self):
-        """
-        After a dataclass is initialized, this is called to finalize some data
-        1. Make sure the original path is an absolute, fully qualified path
-        2. Get the mime type of the file
-        """
-        # Always fully qualify the path first thing
-        # Just in case, convert to a path if it's a str
-        self.original_file = Path(self.original_file).resolve()
-
-        # Get the file type once at init
-        # Note this function isn't called when the object is unpickled
-        self.mime_type = magic.from_file(self.original_file, mime=True)

src/documents/management/commands/document_consumer.py

@@ -16,11 +16,11 @@ from django.core.management.base import CommandError
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers.polling import PollingObserver
 
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
-from documents.data_models import DocumentSource
 from documents.parsers import is_file_ext_supported
 from documents.tasks import consume_file
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
+from paperless.data_models import DocumentSource
 from paperless.models import Tag
 
 try:

src/documents/plugins/base.py

@@ -2,9 +2,9 @@ import abc
 from pathlib import Path
 from typing import Final
 
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
 from documents.plugins.helpers import ProgressManager
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
 
 
 class StopConsumeTaskError(Exception):

src/documents/serialisers.py

@@ -36,7 +36,6 @@ if settings.AUDIT_LOG_ENABLED:
 
 
 from documents import bulk_edit
-from documents.data_models import DocumentSource
 from documents.parsers import is_mime_type_supported
 from documents.permissions import get_groups_with_only_permission
 from documents.permissions import set_permissions_for_object
@@ -44,6 +43,7 @@ from documents.templating.filepath import validate_filepath_template_and_render
 from documents.templating.utils import convert_format_str_to_template_format
 from documents.validators import uri_validator
 from documents.validators import url_validator
+from paperless.data_models import DocumentSource
 from paperless.models import Correspondent
 from paperless.models import CustomField
 from paperless.models import CustomFieldInstance

src/documents/signals/handlers.py

@@ -50,8 +50,8 @@ if TYPE_CHECKING:
     from pathlib import Path
 
     from documents.classifier import DocumentClassifier
-    from documents.data_models import ConsumableDocument
-    from documents.data_models import DocumentMetadataOverrides
+    from paperless.data_models import ConsumableDocument
+    from paperless.data_models import DocumentMetadataOverrides
 
 logger = logging.getLogger("paperless.handlers")
 

src/documents/tasks.py

@@ -27,8 +27,6 @@ from documents.classifier import DocumentClassifier
 from documents.classifier import load_classifier
 from documents.consumer import ConsumerPlugin
 from documents.consumer import WorkflowTriggerPlugin
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
 from documents.double_sided import CollatePlugin
 from documents.file_handling import create_source_path_directory
 from documents.file_handling import generate_unique_filename
@@ -42,6 +40,8 @@ from documents.sanity_checker import SanityCheckFailedException
 from documents.signals import document_updated
 from documents.signals.handlers import cleanup_document_deletion
 from documents.signals.handlers import run_workflows
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
 from paperless.models import Correspondent
 from paperless.models import CustomFieldInstance
 from paperless.models import Document

src/documents/tests/test_api_documents.py

@@ -28,10 +28,10 @@ from documents.caching import CACHE_50_MINUTES
 from documents.caching import CLASSIFIER_HASH_KEY
 from documents.caching import CLASSIFIER_MODIFIED_KEY
 from documents.caching import CLASSIFIER_VERSION_KEY
-from documents.data_models import DocumentSource
 from documents.signals.handlers import run_workflows
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import DocumentConsumeDelayMixin
+from paperless.data_models import DocumentSource
 from paperless.models import Correspondent
 from paperless.models import CustomField
 from paperless.models import CustomFieldInstance

src/documents/tests/test_api_workflows.py

@@ -5,8 +5,8 @@ from django.contrib.auth.models import User
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from documents.data_models import DocumentSource
 from documents.tests.utils import DirectoriesMixin
+from paperless.data_models import DocumentSource
 from paperless.models import Correspondent
 from paperless.models import CustomField
 from paperless.models import DocumentType

src/documents/tests/test_barcodes.py

@@ -11,15 +11,15 @@ from django.test import override_settings
 
 from documents import tasks
 from documents.barcodes import BarcodePlugin
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
-from documents.data_models import DocumentSource
 from documents.plugins.base import StopConsumeTaskError
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import DocumentConsumeDelayMixin
 from documents.tests.utils import DummyProgressManager
 from documents.tests.utils import FileSystemAssertsMixin
 from documents.tests.utils import SampleDirMixin
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
+from paperless.data_models import DocumentSource
 from paperless.models import Document
 from paperless.models import Tag
 

src/documents/tests/test_consumer.py

@@ -18,8 +18,6 @@ from django.utils import timezone
 from guardian.core import ObjectPermissionChecker
 
 from documents.consumer import ConsumerError
-from documents.data_models import DocumentMetadataOverrides
-from documents.data_models import DocumentSource
 from documents.parsers import DocumentParser
 from documents.parsers import ParseError
 from documents.plugins.helpers import ProgressStatusOptions
@@ -27,6 +25,8 @@ from documents.tasks import sanity_check
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import FileSystemAssertsMixin
 from documents.tests.utils import GetConsumerMixin
+from paperless.data_models import DocumentMetadataOverrides
+from paperless.data_models import DocumentSource
 from paperless.models import Correspondent
 from paperless.models import CustomField
 from paperless.models import Document

src/documents/tests/test_double_sided.py

@@ -11,13 +11,13 @@ from pikepdf import Pdf
 
 from documents import tasks
 from documents.consumer import ConsumerError
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentSource
 from documents.double_sided import STAGING_FILE_NAME
 from documents.double_sided import TIMEOUT_MINUTES
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import DummyProgressManager
 from documents.tests.utils import FileSystemAssertsMixin
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentSource
 
 
 @override_settings(

src/documents/tests/test_management_consumer.py

@@ -13,10 +13,10 @@ from django.test import TransactionTestCase
 from django.test import override_settings
 
 from documents.consumer import ConsumerError
-from documents.data_models import ConsumableDocument
 from documents.management.commands import document_consumer
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import DocumentConsumeDelayMixin
+from paperless.data_models import ConsumableDocument
 from paperless.models import Tag
 
 

src/documents/tests/test_migration_workflows.py

@@ -1,5 +1,5 @@
-from documents.data_models import DocumentSource
 from documents.tests.utils import TestMigrations
+from paperless.data_models import DocumentSource
 
 
 class TestMigrateWorkflow(TestMigrations):

src/documents/tests/test_task_signals.py

@@ -4,15 +4,15 @@ from unittest import mock
 import celery
 from django.test import TestCase
 
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
-from documents.data_models import DocumentSource
 from documents.signals.handlers import before_task_publish_handler
 from documents.signals.handlers import task_failure_handler
 from documents.signals.handlers import task_postrun_handler
 from documents.signals.handlers import task_prerun_handler
 from documents.tests.test_consumer import fake_magic_from_file
 from documents.tests.utils import DirectoriesMixin
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
+from paperless.data_models import DocumentSource
 from paperless.models import PaperlessTask
 
 

src/documents/tests/test_workflows.py

@@ -22,13 +22,13 @@ if TYPE_CHECKING:
 
 
 from documents import tasks
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentSource
 from documents.signals import document_consumption_finished
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import DummyProgressManager
 from documents.tests.utils import FileSystemAssertsMixin
 from documents.tests.utils import SampleDirMixin
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentSource
 from paperless.matching import document_matches_workflow
 from paperless.models import Correspondent
 from paperless.models import CustomField

src/documents/tests/utils.py

@@ -21,11 +21,11 @@ from django.test import TransactionTestCase
 from django.test import override_settings
 
 from documents.consumer import ConsumerPlugin
-from documents.data_models import ConsumableDocument
-from documents.data_models import DocumentMetadataOverrides
-from documents.data_models import DocumentSource
 from documents.parsers import ParseError
 from documents.plugins.helpers import ProgressStatusOptions
+from paperless.data_models import ConsumableDocument
+from paperless.data_models import DocumentMetadataOverrides
+from paperless.data_models import DocumentSource
 
 
 def setup_directories():