paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Implement custom fields for documents (#4502)

Browse Source 
Adds custom fields of certain data types, attachable to documents and searchable

Co-Authored-By: Trenton H <797416+stumpylog@users.noreply.github.com>
This commit is contained in:
shamoon
2023-11-05 17:26:51 -08:00
parent 800f54f263
commit 10729f0362
67 changed files with 3199 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/documents/admin.py
View File

@@ -3,6 +3,8 @@ from django.contrib import admin
from guardian.admin import GuardedModelAdmin
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
@@ -144,6 +146,20 @@ class ShareLinksAdmin(GuardedModelAdmin):
list_display_links = ("created",)
class CustomFieldsAdmin(GuardedModelAdmin):
fields = ("name", "created", "data_type")
readonly_fields = ("created", "data_type")
list_display = ("name", "created", "data_type")
list_filter = ("created", "data_type")
class CustomFieldInstancesAdmin(GuardedModelAdmin):
fields = ("field", "document", "created", "value")
readonly_fields = ("field", "document", "created", "value")
list_display = ("field", "document", "value", "created")
list_filter = ("document", "created")
admin.site.register(Correspondent, CorrespondentAdmin)
admin.site.register(Tag, TagAdmin)
admin.site.register(DocumentType, DocumentTypeAdmin)
@@ -153,6 +169,8 @@ admin.site.register(StoragePath, StoragePathAdmin)
admin.site.register(PaperlessTask, TaskAdmin)
admin.site.register(Note, NotesAdmin)
admin.site.register(ShareLink, ShareLinksAdmin)
admin.site.register(CustomField, CustomFieldsAdmin)
admin.site.register(CustomFieldInstance, CustomFieldInstancesAdmin)
if settings.AUDIT_LOG_ENABLED:

18
src/documents/filters.py
View File

@@ -82,6 +82,21 @@ class TitleContentFilter(Filter):
return qs
class CustomFieldsFilter(Filter):
def filter(self, qs, value):
if value:
return (
qs.filter(custom_fields__field__name__icontains=value)
| qs.filter(custom_fields__value_text__icontains=value)
| qs.filter(custom_fields__value_bool__icontains=value)
| qs.filter(custom_fields__value_int__icontains=value)
| qs.filter(custom_fields__value_date__icontains=value)
| qs.filter(custom_fields__value_url__icontains=value)
)
else:
return qs
class DocumentFilterSet(FilterSet):
is_tagged = BooleanFilter(
label="Is tagged",
@@ -108,6 +123,8 @@ class DocumentFilterSet(FilterSet):
owner__id__none = ObjectFilter(field_name="owner", exclude=True)
custom_fields__icontains = CustomFieldsFilter()
class Meta:
model = Document
fields = {
@@ -132,6 +149,7 @@ class DocumentFilterSet(FilterSet):
"storage_path__name": CHAR_KWARGS,
"owner": ["isnull"],
"owner__id": ID_KWARGS,
"custom_fields": ["icontains"],
}

34
src/documents/index.py
View File

@@ -30,6 +30,8 @@ from whoosh.searching import ResultsPage
from whoosh.searching import Searcher
from whoosh.writing import AsyncWriter
# from documents.models import CustomMetadata
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
from documents.models import User
@@ -60,6 +62,8 @@ def get_schema():
has_path=BOOLEAN(),
notes=TEXT(),
num_notes=NUMERIC(sortable=True, signed=False),
custom_fields=TEXT(),
custom_field_count=NUMERIC(sortable=True, signed=False),
owner=TEXT(),
owner_id=NUMERIC(),
has_owner=BOOLEAN(),
@@ -69,7 +73,7 @@ def get_schema():
)
def open_index(recreate=False):
def open_index(recreate=False) -> FileIndex:
try:
if exists_in(settings.INDEX_DIR) and not recreate:
return open_dir(settings.INDEX_DIR, schema=get_schema())
@@ -82,7 +86,7 @@ def open_index(recreate=False):
@contextmanager
def open_index_writer(optimize=False):
def open_index_writer(optimize=False) -> AsyncWriter:
writer = AsyncWriter(open_index())
try:
@@ -95,7 +99,7 @@ def open_index_writer(optimize=False):
@contextmanager
def open_index_searcher():
def open_index_searcher() -> Searcher:
searcher = open_index().searcher()
try:
@@ -108,6 +112,9 @@ def update_document(writer: AsyncWriter, doc: Document):
tags = ",".join([t.name for t in doc.tags.all()])
tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)])
custom_fields = ",".join(
[str(c) for c in CustomFieldInstance.objects.filter(document=doc)],
)
asn = doc.archive_serial_number
if asn is not None and (
asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
@@ -147,6 +154,8 @@ def update_document(writer: AsyncWriter, doc: Document):
has_path=doc.storage_path is not None,
notes=notes,
num_notes=len(notes),
custom_fields=custom_fields,
custom_field_count=len(doc.custom_fields.all()),
owner=doc.owner.username if doc.owner else None,
owner_id=doc.owner.id if doc.owner else None,
has_owner=doc.owner is not None,
@@ -156,20 +165,20 @@ def update_document(writer: AsyncWriter, doc: Document):
)
def remove_document(writer, doc):
def remove_document(writer: AsyncWriter, doc: Document):
remove_document_by_id(writer, doc.pk)
def remove_document_by_id(writer, doc_id):
def remove_document_by_id(writer: AsyncWriter, doc_id):
writer.delete_by_term("id", doc_id)
def add_or_update_document(document):
def add_or_update_document(document: Document):
with open_index_writer() as writer:
update_document(writer, document)
def remove_document_from_index(document):
def remove_document_from_index(document: Document):
with open_index_writer() as writer:
remove_document(writer, document)
@@ -185,6 +194,7 @@ class DelayedQuery:
"created": ("created", ["date__lt", "date__gt"]),
"checksum": ("checksum", ["icontains", "istartswith"]),
"original_filename": ("original_filename", ["icontains", "istartswith"]),
"custom_fields": ("custom_fields", ["icontains", "istartswith"]),
}
def _get_query(self):
@@ -350,7 +360,15 @@ class DelayedFullTextQuery(DelayedQuery):
def _get_query(self):
q_str = self.query_params["query"]
qp = MultifieldParser(
["content", "title", "correspondent", "tag", "type", "notes"],
[
"content",
"title",
"correspondent",
"tag",
"type",
"notes",
"custom_fields",
],
self.searcher.ixreader.schema,
)
qp.add_plugin(DateParserPlugin(basedate=timezone.now()))

131
src/documents/migrations/1040_customfield_customfieldinstance_and_more.py Normal file
View File

@@ -0,0 +1,131 @@
# Generated by Django 4.2.6 on 2023-11-02 17:38
import django.db.models.deletion
import django.utils.timezone
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1039_consumptiontemplate"),
]
operations = [
migrations.CreateModel(
name="CustomField",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
models.DateTimeField(
db_index=True,
default=django.utils.timezone.now,
editable=False,
verbose_name="created",
),
),
("name", models.CharField(max_length=128)),
(
"data_type",
models.CharField(
choices=[
("string", "String"),
("url", "URL"),
("date", "Date"),
("boolean", "Boolean"),
("integer", "Integer"),
("float", "Float"),
("monetary", "Monetary"),
],
editable=False,
max_length=50,
verbose_name="data type",
),
),
],
options={
"verbose_name": "custom field",
"verbose_name_plural": "custom fields",
"ordering": ("created",),
},
),
migrations.CreateModel(
name="CustomFieldInstance",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
models.DateTimeField(
db_index=True,
default=django.utils.timezone.now,
editable=False,
verbose_name="created",
),
),
("value_text", models.CharField(max_length=128, null=True)),
("value_bool", models.BooleanField(null=True)),
("value_url", models.URLField(null=True)),
("value_date", models.DateField(null=True)),
("value_int", models.IntegerField(null=True)),
("value_float", models.FloatField(null=True)),
(
"value_monetary",
models.DecimalField(decimal_places=2, max_digits=12, null=True),
),
(
"document",
models.ForeignKey(
editable=False,
on_delete=django.db.models.deletion.CASCADE,
related_name="custom_fields",
to="documents.document",
),
),
(
"field",
models.ForeignKey(
editable=False,
on_delete=django.db.models.deletion.CASCADE,
related_name="fields",
to="documents.customfield",
),
),
],
options={
"verbose_name": "custom field instance",
"verbose_name_plural": "custom field instances",
"ordering": ("created",),
},
),
migrations.AddConstraint(
model_name="customfield",
constraint=models.UniqueConstraint(
fields=("name",),
name="documents_customfield_unique_name",
),
),
migrations.AddConstraint(
model_name="customfieldinstance",
constraint=models.UniqueConstraint(
fields=("document", "field"),
name="documents_customfieldinstance_unique_document_field",
),
),
]

130
src/documents/models.py
View File

@@ -877,9 +877,139 @@ class ConsumptionTemplate(models.Model):
return f"{self.name}"
class CustomField(models.Model):
"""
Defines the name and type of a custom field
"""
class FieldDataType(models.TextChoices):
STRING = ("string", _("String"))
URL = ("url", _("URL"))
DATE = ("date", _("Date"))
BOOL = ("boolean"), _("Boolean")
INT = ("integer", _("Integer"))
FLOAT = ("float", _("Float"))
MONETARY = ("monetary", _("Monetary"))
created = models.DateTimeField(
_("created"),
default=timezone.now,
db_index=True,
editable=False,
)
name = models.CharField(max_length=128)
data_type = models.CharField(
_("data type"),
max_length=50,
choices=FieldDataType.choices,
editable=False,
)
class Meta:
ordering = ("created",)
verbose_name = _("custom field")
verbose_name_plural = _("custom fields")
constraints = [
models.UniqueConstraint(
fields=["name"],
name="%(app_label)s_%(class)s_unique_name",
),
]
def __str__(self) -> str:
return f"{self.name} : {self.data_type}"
class CustomFieldInstance(models.Model):
"""
A single instance of a field, attached to a CustomField for the name and type
and attached to a single Document to be metadata for it
"""
created = models.DateTimeField(
_("created"),
default=timezone.now,
db_index=True,
editable=False,
)
document = models.ForeignKey(
Document,
blank=False,
null=False,
on_delete=models.CASCADE,
related_name="custom_fields",
editable=False,
)
field = models.ForeignKey(
CustomField,
blank=False,
null=False,
on_delete=models.CASCADE,
related_name="fields",
editable=False,
)
# Actual data storage
value_text = models.CharField(max_length=128, null=True)
value_bool = models.BooleanField(null=True)
value_url = models.URLField(null=True)
value_date = models.DateField(null=True)
value_int = models.IntegerField(null=True)
value_float = models.FloatField(null=True)
value_monetary = models.DecimalField(null=True, decimal_places=2, max_digits=12)
class Meta:
ordering = ("created",)
verbose_name = _("custom field instance")
verbose_name_plural = _("custom field instances")
constraints = [
models.UniqueConstraint(
fields=["document", "field"],
name="%(app_label)s_%(class)s_unique_document_field",
),
]
def __str__(self) -> str:
return str(self.field.name) + f" : {self.value}"
@property
def value(self):
"""
Based on the data type, access the actual value the instance stores
A little shorthand/quick way to get what is actually here
"""
if self.field.data_type == CustomField.FieldDataType.STRING:
return self.value_text
elif self.field.data_type == CustomField.FieldDataType.URL:
return self.value_url
elif self.field.data_type == CustomField.FieldDataType.DATE:
return self.value_date
elif self.field.data_type == CustomField.FieldDataType.BOOL:
return self.value_bool
elif self.field.data_type == CustomField.FieldDataType.INT:
return self.value_int
elif self.field.data_type == CustomField.FieldDataType.FLOAT:
return self.value_float
elif self.field.data_type == CustomField.FieldDataType.MONETARY:
return self.value_monetary
raise NotImplementedError(self.field.data_type)
if settings.AUDIT_LOG_ENABLED:
auditlog.register(Document, m2m_fields={"tags"})
auditlog.register(Correspondent)
auditlog.register(Tag)
auditlog.register(DocumentType)
auditlog.register(Note)
auditlog.register(CustomField)
auditlog.register(CustomFieldInstance)

96
src/documents/serialisers.py
View File

@@ -8,9 +8,11 @@ from celery import states
from django.conf import settings
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.core.validators import URLValidator
from django.utils.crypto import get_random_string
from django.utils.text import slugify
from django.utils.translation import gettext as _
from drf_writable_nested.serializers import NestedUpdateMixin
from guardian.core import ObjectPermissionChecker
from guardian.shortcuts import get_users_with_perms
from rest_framework import fields
@@ -21,6 +23,8 @@ from documents import bulk_edit
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
@@ -394,7 +398,92 @@ class StoragePathField(serializers.PrimaryKeyRelatedField):
return StoragePath.objects.all()
class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer):
class CustomFieldSerializer(serializers.ModelSerializer):
data_type = serializers.ChoiceField(
choices=CustomField.FieldDataType,
read_only=False,
)
class Meta:
model = CustomField
fields = [
"id",
"name",
"data_type",
]
class ReadWriteSerializerMethodField(serializers.SerializerMethodField):
"""
Based on https://stackoverflow.com/a/62579804
"""
def __init__(self, method_name=None, *args, **kwargs):
self.method_name = method_name
kwargs["source"] = "*"
super(serializers.SerializerMethodField, self).__init__(*args, **kwargs)
def to_internal_value(self, data):
return {self.field_name: data}
class CustomFieldInstanceSerializer(serializers.ModelSerializer):
field = serializers.PrimaryKeyRelatedField(queryset=CustomField.objects.all())
value = ReadWriteSerializerMethodField()
def create(self, validated_data):
type_to_data_store_name_map = {
CustomField.FieldDataType.STRING: "value_text",
CustomField.FieldDataType.URL: "value_url",
CustomField.FieldDataType.DATE: "value_date",
CustomField.FieldDataType.BOOL: "value_bool",
CustomField.FieldDataType.INT: "value_int",
CustomField.FieldDataType.FLOAT: "value_float",
CustomField.FieldDataType.MONETARY: "value_monetary",
}
# An instance is attached to a document
document: Document = validated_data["document"]
# And to a CustomField
custom_field: CustomField = validated_data["field"]
# This key must exist, as it is validated
data_store_name = type_to_data_store_name_map[custom_field.data_type]
# Actually update or create the instance, providing the value
# to fill in the correct attribute based on the type
instance, _ = CustomFieldInstance.objects.update_or_create(
document=document,
field=custom_field,
defaults={data_store_name: validated_data["value"]},
)
return instance
def get_value(self, obj: CustomFieldInstance):
return obj.value
def validate(self, data):
"""
For some reason, URLField validation is not run against the value
automatically. Force it to run against the value
"""
data = super().validate(data)
field: CustomField = data["field"]
if field.data_type == CustomField.FieldDataType.URL:
URLValidator()(data["value"])
return data
class Meta:
model = CustomFieldInstance
fields = [
"value",
"field",
]
class DocumentSerializer(
OwnedObjectSerializer,
NestedUpdateMixin,
DynamicFieldsModelSerializer,
):
correspondent = CorrespondentField(allow_null=True)
tags = TagsField(many=True)
document_type = DocumentTypeField(allow_null=True)
@@ -404,6 +493,8 @@ class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer):
archived_file_name = SerializerMethodField()
created_date = serializers.DateField(required=False)
custom_fields = CustomFieldInstanceSerializer(many=True, allow_null=True)
owner = serializers.PrimaryKeyRelatedField(
queryset=User.objects.all(),
required=False,
@@ -425,7 +516,7 @@ class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer):
doc["content"] = doc.get("content")[0:550]
return doc
def update(self, instance, validated_data):
def update(self, instance: Document, validated_data):
if "created_date" in validated_data and "created" not in validated_data:
new_datetime = datetime.datetime.combine(
validated_data.get("created_date"),
@@ -466,6 +557,7 @@ class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer):
"user_can_change",
"set_permissions",
"notes",
"custom_fields",
)

106
src/documents/tests/test_api.py
View File

@@ -35,6 +35,8 @@ from documents import index
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
@@ -347,11 +349,36 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
tag_2 = Tag.objects.create(name="t2")
tag_3 = Tag.objects.create(name="t3")
cf1 = CustomField.objects.create(
name="stringfield",
data_type=CustomField.FieldDataType.STRING,
)
cf2 = CustomField.objects.create(
name="numberfield",
data_type=CustomField.FieldDataType.INT,
)
doc1.tags.add(tag_inbox)
doc2.tags.add(tag_2)
doc3.tags.add(tag_2)
doc3.tags.add(tag_3)
cf1_d1 = CustomFieldInstance.objects.create(
document=doc1,
field=cf1,
value_text="foobard1",
)
CustomFieldInstance.objects.create(
document=doc1,
field=cf2,
value_int=999,
)
cf1_d3 = CustomFieldInstance.objects.create(
document=doc3,
field=cf1,
value_text="foobard3",
)
response = self.client.get("/api/documents/?is_in_inbox=true")
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
@@ -423,6 +450,31 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
results = response.data["results"]
self.assertEqual(len(results), 0)
# custom field name
response = self.client.get(
f"/api/documents/?custom_fields__icontains={cf1.name}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
self.assertEqual(len(results), 2)
# custom field value
response = self.client.get(
f"/api/documents/?custom_fields__icontains={cf1_d1.value}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], doc1.id)
response = self.client.get(
f"/api/documents/?custom_fields__icontains={cf1_d3.value}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], doc3.id)
def test_document_checksum_filter(self):
Document.objects.create(
title="none1",
@@ -1146,6 +1198,14 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
dt2 = DocumentType.objects.create(name="type2")
sp = StoragePath.objects.create(name="path")
sp2 = StoragePath.objects.create(name="path2")
cf1 = CustomField.objects.create(
name="string field",
data_type=CustomField.FieldDataType.STRING,
)
cf2 = CustomField.objects.create(
name="number field",
data_type=CustomField.FieldDataType.INT,
)
d1 = Document.objects.create(checksum="1", correspondent=c, content="test")
d2 = Document.objects.create(checksum="2", document_type=dt, content="test")
@@ -1176,6 +1236,22 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
content="test",
)
cf1_d1 = CustomFieldInstance.objects.create(
document=d1,
field=cf1,
value_text="foobard1",
)
cf2_d1 = CustomFieldInstance.objects.create(
document=d1,
field=cf2,
value_int=999,
)
cf1_d4 = CustomFieldInstance.objects.create(
document=d4,
field=cf1,
value_text="foobard4",
)
with AsyncWriter(index.open_index()) as writer:
for doc in Document.objects.all():
index.update_document(writer, doc)
@@ -1304,6 +1380,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
+ datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"),
),
)
self.assertIn(
d5.id,
search_query(
@@ -1322,6 +1399,27 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
[d4.id, d5.id],
)
self.assertIn(
d1.id,
search_query(
"&custom_fields__icontains=" + cf1_d1.value,
),
)
self.assertIn(
d1.id,
search_query(
"&custom_fields__icontains=" + str(cf2_d1.value),
),
)
self.assertIn(
d4.id,
search_query(
"&custom_fields__icontains=" + cf1_d4.value,
),
)
def test_search_filtering_respect_owner(self):
"""
GIVEN:
@@ -2421,7 +2519,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
f"/api/documents/{doc.pk}/notes/",
format="json",
)
self.assertEqual(resp.content, b"Insufficient permissions to view")
self.assertEqual(resp.content, b"Insufficient permissions to view notes")
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
assign_perm("view_document", user1, doc)
@@ -2430,7 +2528,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
f"/api/documents/{doc.pk}/notes/",
data={"note": "this is a posted note"},
)
self.assertEqual(resp.content, b"Insufficient permissions to create")
self.assertEqual(resp.content, b"Insufficient permissions to create notes")
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
note = Note.objects.create(
@@ -2444,7 +2542,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
format="json",
)
self.assertEqual(response.content, b"Insufficient permissions to delete")
self.assertEqual(response.content, b"Insufficient permissions to delete notes")
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_delete_note(self):
@@ -2694,7 +2792,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
f"/api/documents/{doc.pk}/share_links/",
format="json",
)
self.assertEqual(resp.content, b"Insufficient permissions")
self.assertEqual(resp.content, b"Insufficient permissions to add share link")
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
assign_perm("change_document", user1, doc)

384
src/documents/tests/test_api_custom_fields.py Normal file
View File

@@ -0,0 +1,384 @@
from datetime import date
from django.contrib.auth.models import User
from rest_framework import status
from rest_framework.test import APITestCase
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.tests.utils import DirectoriesMixin
class TestCustomField(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/custom_fields/"
def setUp(self):
self.user = User.objects.create_superuser(username="temp_admin")
self.client.force_authenticate(user=self.user)
return super().setUp()
def test_create_custom_field(self):
"""
GIVEN:
- Each of the supported data types is created
WHEN:
- API request to create custom metadata is made
THEN:
- the field is created
- the field returns the correct fields
"""
for field_type, name in [
("string", "Custom Text"),
("url", "Wikipedia Link"),
("date", "Invoiced Date"),
("integer", "Invoice #"),
("boolean", "Is Active"),
("float", "Total Paid"),
]:
resp = self.client.post(
self.ENDPOINT,
data={
"data_type": field_type,
"name": name,
},
)
self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
data = resp.json()
self.assertEqual(len(data), 3)
self.assertEqual(data["name"], name)
self.assertEqual(data["data_type"], field_type)
def test_create_custom_field_instance(self):
"""
GIVEN:
- Field of each data type is created
WHEN:
- API request to create custom metadata instance with each data type
THEN:
- the field instance is created
- the field returns the correct fields and values
- the field is attached to the given document
"""
doc = Document.objects.create(
title="WOW",
content="the content",
checksum="123",
mime_type="application/pdf",
)
custom_field_string = CustomField.objects.create(
name="Test Custom Field String",
data_type=CustomField.FieldDataType.STRING,
)
custom_field_date = CustomField.objects.create(
name="Test Custom Field Date",
data_type=CustomField.FieldDataType.DATE,
)
custom_field_int = CustomField.objects.create(
name="Test Custom Field Int",
data_type=CustomField.FieldDataType.INT,
)
custom_field_boolean = CustomField.objects.create(
name="Test Custom Field Boolean",
data_type=CustomField.FieldDataType.BOOL,
)
custom_field_url = CustomField.objects.create(
name="Test Custom Field Url",
data_type=CustomField.FieldDataType.URL,
)
custom_field_float = CustomField.objects.create(
name="Test Custom Field Float",
data_type=CustomField.FieldDataType.FLOAT,
)
custom_field_monetary = CustomField.objects.create(
name="Test Custom Field Monetary",
data_type=CustomField.FieldDataType.MONETARY,
)
date_value = date.today()
resp = self.client.patch(
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_string.id,
"value": "test value",
},
{
"field": custom_field_date.id,
"value": date_value.isoformat(),
},
{
"field": custom_field_int.id,
"value": 3,
},
{
"field": custom_field_boolean.id,
"value": True,
},
{
"field": custom_field_url.id,
"value": "https://example.com",
},
{
"field": custom_field_float.id,
"value": 12.3456,
},
{
"field": custom_field_monetary.id,
"value": 11.10,
},
],
},
format="json",
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
resp_data = resp.json()["custom_fields"]
self.assertCountEqual(
resp_data,
[
{"field": custom_field_string.id, "value": "test value"},
{"field": custom_field_date.id, "value": date_value.isoformat()},
{"field": custom_field_int.id, "value": 3},
{"field": custom_field_boolean.id, "value": True},
{"field": custom_field_url.id, "value": "https://example.com"},
{"field": custom_field_float.id, "value": 12.3456},
{"field": custom_field_monetary.id, "value": 11.10},
],
)
doc.refresh_from_db()
self.assertEqual(len(doc.custom_fields.all()), 7)
def test_change_custom_field_instance_value(self):
"""
GIVEN:
- Custom field instance is created and attached to document
WHEN:
- API request to create change the value of the custom field
THEN:
- the field instance is updated
- the field returns the correct fields and values
"""
doc = Document.objects.create(
title="WOW",
content="the content",
checksum="123",
mime_type="application/pdf",
)
custom_field_string = CustomField.objects.create(
name="Test Custom Field String",
data_type=CustomField.FieldDataType.STRING,
)
self.assertEqual(CustomFieldInstance.objects.count(), 0)
# Create
resp = self.client.patch(
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_string.id,
"value": "test value",
},
],
},
format="json",
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(CustomFieldInstance.objects.count(), 1)
self.assertEqual(doc.custom_fields.first().value, "test value")
# Update
resp = self.client.patch(
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_string.id,
"value": "a new test value",
},
],
},
format="json",
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(CustomFieldInstance.objects.count(), 1)
self.assertEqual(doc.custom_fields.first().value, "a new test value")
def test_delete_custom_field_instance(self):
"""
GIVEN:
- Multiple custom field instances are created and attached to document
WHEN:
- API request to remove a field
THEN:
- the field instance is removed
- the other field remains unchanged
- the field returns the correct fields and values
"""
doc = Document.objects.create(
title="WOW",
content="the content",
checksum="123",
mime_type="application/pdf",
)
custom_field_string = CustomField.objects.create(
name="Test Custom Field String",
data_type=CustomField.FieldDataType.STRING,
)
custom_field_date = CustomField.objects.create(
name="Test Custom Field Date",
data_type=CustomField.FieldDataType.DATE,
)
date_value = date.today()
resp = self.client.patch(
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_string.id,
"value": "a new test value",
},
{
"field": custom_field_date.id,
"value": date_value.isoformat(),
},
],
},
format="json",
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(CustomFieldInstance.objects.count(), 2)
self.assertEqual(len(doc.custom_fields.all()), 2)
resp = self.client.patch(
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_date.id,
"value": date_value.isoformat(),
},
],
},
format="json",
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(CustomFieldInstance.objects.count(), 1)
self.assertEqual(Document.objects.count(), 1)
self.assertEqual(len(doc.custom_fields.all()), 1)
self.assertEqual(doc.custom_fields.first().value, date_value)
def test_custom_field_validation(self):
"""
GIVEN:
- Document exists with no fields
WHEN:
- API request to remove a field
- API request is not valid
THEN:
- HTTP 400 is returned
- No field created
- No field attached to the document
"""
doc = Document.objects.create(
title="WOW",
content="the content",
checksum="123",
mime_type="application/pdf",
)
custom_field_string = CustomField.objects.create(
name="Test Custom Field String",
data_type=CustomField.FieldDataType.STRING,
)
resp = self.client.patch(
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_string.id,
# Whoops, spelling
"valeu": "a new test value",
},
],
},
format="json",
)
from pprint import pprint
pprint(resp.json())
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(CustomFieldInstance.objects.count(), 0)
self.assertEqual(len(doc.custom_fields.all()), 0)
def test_custom_field_value_validation(self):
"""
GIVEN:
- Document & custom field exist
WHEN:
- API request to set a field value
THEN:
- HTTP 400 is returned
- No field instance is created or attached to the document
"""
doc = Document.objects.create(
title="WOW",
content="the content",
checksum="123",
mime_type="application/pdf",
)
custom_field_url = CustomField.objects.create(
name="Test Custom Field URL",
data_type=CustomField.FieldDataType.URL,
)
custom_field_int = CustomField.objects.create(
name="Test Custom Field INT",
data_type=CustomField.FieldDataType.INT,
)
resp = self.client.patch(
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_url.id,
"value": "not a url",
},
],
},
format="json",
)
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(CustomFieldInstance.objects.count(), 0)
self.assertEqual(len(doc.custom_fields.all()), 0)
self.assertRaises(
Exception,
self.client.patch,
f"/api/documents/{doc.id}/",
data={
"custom_fields": [
{
"field": custom_field_int.id,
"value": "not an int",
},
],
},
format="json",
)
self.assertEqual(CustomFieldInstance.objects.count(), 0)
self.assertEqual(len(doc.custom_fields.all()), 0)

16
src/documents/tests/test_management_exporter.py
View File

@@ -153,7 +153,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
manifest = self._do_export(use_filename_format=use_filename_format)
self.assertEqual(len(manifest), 159)
self.assertEqual(len(manifest), 169)
# dont include consumer or AnonymousUser users
self.assertEqual(
@@ -247,7 +247,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec")
self.assertEqual(GroupObjectPermission.objects.count(), 1)
self.assertEqual(UserObjectPermission.objects.count(), 1)
self.assertEqual(Permission.objects.count(), 116)
self.assertEqual(Permission.objects.count(), 124)
messages = check_sanity()
# everything is alright after the test
self.assertEqual(len(messages), 0)
@@ -676,15 +676,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
os.path.join(self.dirs.media_dir, "documents"),
)
self.assertEqual(ContentType.objects.count(), 29)
self.assertEqual(Permission.objects.count(), 116)
self.assertEqual(ContentType.objects.count(), 31)
self.assertEqual(Permission.objects.count(), 124)
manifest = self._do_export()
with paperless_environment():
self.assertEqual(
len(list(filter(lambda e: e["model"] == "auth.permission", manifest))),
116,
124,
)
# add 1 more to db to show objects are not re-created by import
Permission.objects.create(
@@ -692,7 +692,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
codename="test_perm",
content_type_id=1,
)
self.assertEqual(Permission.objects.count(), 117)
self.assertEqual(Permission.objects.count(), 125)
# will cause an import error
self.user.delete()
@@ -701,5 +701,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
with self.assertRaises(IntegrityError):
call_command("document_importer", "--no-progress-bar", self.target)
self.assertEqual(ContentType.objects.count(), 29)
self.assertEqual(Permission.objects.count(), 117)
self.assertEqual(ContentType.objects.count(), 31)
self.assertEqual(Permission.objects.count(), 125)

111
src/documents/views.py
View File

@@ -78,6 +78,7 @@ from documents.matching import match_storage_paths
from documents.matching import match_tags
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
@@ -99,6 +100,7 @@ from documents.serialisers import BulkEditObjectPermissionsSerializer
from documents.serialisers import BulkEditSerializer
from documents.serialisers import ConsumptionTemplateSerializer
from documents.serialisers import CorrespondentSerializer
from documents.serialisers import CustomFieldSerializer
from documents.serialisers import DocumentListSerializer
from documents.serialisers import DocumentSerializer
from documents.serialisers import DocumentTypeSerializer
@@ -497,7 +499,7 @@ class DocumentViewSet(
"view_document",
doc,
):
return HttpResponseForbidden("Insufficient permissions to view")
return HttpResponseForbidden("Insufficient permissions to view notes")
except Document.DoesNotExist:
raise Http404
@@ -507,7 +509,7 @@ class DocumentViewSet(
except Exception as e:
logger.warning(f"An error occurred retrieving notes: {e!s}")
return Response(
{"error": "Error retreiving notes, check logs for more detail."},
{"error": "Error retrieving notes, check logs for more detail."},
)
elif request.method == "POST":
try:
@@ -516,7 +518,9 @@ class DocumentViewSet(
"change_document",
doc,
):
return HttpResponseForbidden("Insufficient permissions to create")
return HttpResponseForbidden(
"Insufficient permissions to create notes",
)
c = Note.objects.create(
document=doc,
@@ -558,7 +562,7 @@ class DocumentViewSet(
"change_document",
doc,
):
return HttpResponseForbidden("Insufficient permissions to delete")
return HttpResponseForbidden("Insufficient permissions to delete notes")
note = Note.objects.get(id=int(request.GET.get("id")))
if settings.AUDIT_LOG_ENABLED:
@@ -599,7 +603,9 @@ class DocumentViewSet(
"change_document",
doc,
):
return HttpResponseForbidden("Insufficient permissions")
return HttpResponseForbidden(
"Insufficient permissions to add share link",
)
except Document.DoesNotExist:
raise Http404
@@ -1071,47 +1077,6 @@ class BulkDownloadView(GenericAPIView):
return response
class RemoteVersionView(GenericAPIView):
def get(self, request, format=None):
remote_version = "0.0.0"
is_greater_than_current = False
current_version = packaging_version.parse(version.__full_version_str__)
try:
req = urllib.request.Request(
"https://api.github.com/repos/paperless-ngx/"
"paperless-ngx/releases/latest",
)
# Ensure a JSON response
req.add_header("Accept", "application/json")
with urllib.request.urlopen(req) as response:
remote = response.read().decode("utf-8")
try:
remote_json = json.loads(remote)
remote_version = remote_json["tag_name"]
# Basically PEP 616 but that only went in 3.9
if remote_version.startswith("ngx-"):
remote_version = remote_version[len("ngx-") :]
except ValueError:
logger.debug("An error occurred parsing remote version json")
except urllib.error.URLError:
logger.debug("An error occurred checking for available updates")
is_greater_than_current = (
packaging_version.parse(
remote_version,
)
> current_version
)
return Response(
{
"version": remote_version,
"update_available": is_greater_than_current,
},
)
class StoragePathViewSet(ModelViewSet, PassUserMixin):
model = StoragePath
@@ -1186,6 +1151,47 @@ class UiSettingsView(GenericAPIView):
)
class RemoteVersionView(GenericAPIView):
def get(self, request, format=None):
remote_version = "0.0.0"
is_greater_than_current = False
current_version = packaging_version.parse(version.__full_version_str__)
try:
req = urllib.request.Request(
"https://api.github.com/repos/paperlessngx/"
"paperlessngx/releases/latest",
)
# Ensure a JSON response
req.add_header("Accept", "application/json")
with urllib.request.urlopen(req) as response:
remote = response.read().decode("utf8")
try:
remote_json = json.loads(remote)
remote_version = remote_json["tag_name"]
# Basically PEP 616 but that only went in 3.9
if remote_version.startswith("ngx-"):
remote_version = remote_version[len("ngx-") :]
except ValueError:
logger.debug("An error occurred parsing remote version json")
except urllib.error.URLError:
logger.debug("An error occurred checking for available updates")
is_greater_than_current = (
packaging_version.parse(
remote_version,
)
> current_version
)
return Response(
{
"version": remote_version,
"update_available": is_greater_than_current,
},
)
class TasksViewSet(ReadOnlyModelViewSet):
permission_classes = (IsAuthenticated,)
serializer_class = TasksViewSerializer
@@ -1341,4 +1347,15 @@ class ConsumptionTemplateViewSet(ModelViewSet):
model = ConsumptionTemplate
queryset = ConsumptionTemplate.objects.all().order_by("order")
queryset = ConsumptionTemplate.objects.all().order_by("name")
class CustomFieldViewSet(ModelViewSet):
permission_classes = (IsAuthenticated, PaperlessObjectPermissions)
serializer_class = CustomFieldSerializer
pagination_class = StandardPagination
model = CustomField
queryset = CustomField.objects.all().order_by("-created")