paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-28 18:24:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Format Python code with black

Browse Source
This commit is contained in:
kpj
2022-02-27 15:26:41 +01:00
parent f0ffc69010
commit c56cb25b5f
136 changed files with 6142 additions and 3811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/documents/migrations/0001_initial.py
View File

@@ -10,19 +10,33 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
]
dependencies = []
operations = [
migrations.CreateModel(
name='Document',
name="Document",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('sender', models.CharField(blank=True, db_index=True, max_length=128)),
('title', models.CharField(blank=True, db_index=True, max_length=128)),
('content', models.TextField(db_index=("mysql" not in settings.DATABASES["default"]["ENGINE"]))),
('created', models.DateTimeField(auto_now_add=True)),
('modified', models.DateTimeField(auto_now=True)),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("sender", models.CharField(blank=True, db_index=True, max_length=128)),
("title", models.CharField(blank=True, db_index=True, max_length=128)),
(
"content",
models.TextField(
db_index=(
"mysql" not in settings.DATABASES["default"]["ENGINE"]
)
),
),
("created", models.DateTimeField(auto_now_add=True)),
("modified", models.DateTimeField(auto_now=True)),
],
),
]

14
src/documents/migrations/0002_auto_20151226_1316.py
View File

@@ -9,17 +9,19 @@ import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
('documents', '0001_initial'),
("documents", "0001_initial"),
]
operations = [
migrations.AlterModelOptions(
name='document',
options={'ordering': ('sender', 'title')},
name="document",
options={"ordering": ("sender", "title")},
),
migrations.AlterField(
model_name='document',
name='created',
field=models.DateTimeField(default=django.utils.timezone.now, editable=False),
model_name="document",
name="created",
field=models.DateTimeField(
default=django.utils.timezone.now, editable=False
),
),
]

40
src/documents/migrations/0003_sender.py
View File

@@ -19,9 +19,11 @@ def move_sender_strings_to_sender_model(apps, schema_editor):
# Create the sender and log the relationship with the document
for document in document_model.objects.all():
if document.sender:
DOCUMENT_SENDER_MAP[document.pk], created = sender_model.objects.get_or_create(
name=document.sender,
defaults={"slug": slugify(document.sender)}
(
DOCUMENT_SENDER_MAP[document.pk],
created,
) = sender_model.objects.get_or_create(
name=document.sender, defaults={"slug": slugify(document.sender)}
)
@@ -33,27 +35,39 @@ def realign_senders(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '0002_auto_20151226_1316'),
("documents", "0002_auto_20151226_1316"),
]
operations = [
migrations.CreateModel(
name='Sender',
name="Sender",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128, unique=True)),
('slug', models.SlugField()),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=128, unique=True)),
("slug", models.SlugField()),
],
),
migrations.RunPython(move_sender_strings_to_sender_model),
migrations.RemoveField(
model_name='document',
name='sender',
model_name="document",
name="sender",
),
migrations.AddField(
model_name='document',
name='sender',
field=models.ForeignKey(blank=True, on_delete=django.db.models.deletion.CASCADE, to='documents.Sender'),
model_name="document",
name="sender",
field=models.ForeignKey(
blank=True,
on_delete=django.db.models.deletion.CASCADE,
to="documents.Sender",
),
),
migrations.RunPython(realign_senders),
]

14
src/documents/migrations/0004_auto_20160114_1844.py
View File

@@ -9,13 +9,19 @@ import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('documents', '0003_sender'),
("documents", "0003_sender"),
]
operations = [
migrations.AlterField(
model_name='document',
name='sender',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='documents', to='documents.Sender'),
model_name="document",
name="sender",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="documents",
to="documents.Sender",
),
),
]

6
src/documents/migrations/0005_auto_20160123_0313.py
View File

@@ -8,12 +8,12 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('documents', '0004_auto_20160114_1844'),
("documents", "0004_auto_20160114_1844"),
]
operations = [
migrations.AlterModelOptions(
name='sender',
options={'ordering': ('name',)},
name="sender",
options={"ordering": ("name",)},
),
]

53
src/documents/migrations/0006_auto_20160123_0430.py
View File

@@ -8,30 +8,59 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0005_auto_20160123_0313'),
("documents", "0005_auto_20160123_0313"),
]
operations = [
migrations.CreateModel(
name='Tag',
name="Tag",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128, unique=True)),
('slug', models.SlugField(blank=True)),
('colour', models.PositiveIntegerField(choices=[(1, '#a6cee3'), (2, '#1f78b4'), (3, '#b2df8a'), (4, '#33a02c'), (5, '#fb9a99'), (6, '#e31a1c'), (7, '#fdbf6f'), (8, '#ff7f00'), (9, '#cab2d6'), (10, '#6a3d9a'), (11, '#ffff99'), (12, '#b15928'), (13, '#000000'), (14, '#cccccc')], default=1)),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=128, unique=True)),
("slug", models.SlugField(blank=True)),
(
"colour",
models.PositiveIntegerField(
choices=[
(1, "#a6cee3"),
(2, "#1f78b4"),
(3, "#b2df8a"),
(4, "#33a02c"),
(5, "#fb9a99"),
(6, "#e31a1c"),
(7, "#fdbf6f"),
(8, "#ff7f00"),
(9, "#cab2d6"),
(10, "#6a3d9a"),
(11, "#ffff99"),
(12, "#b15928"),
(13, "#000000"),
(14, "#cccccc"),
],
default=1,
),
),
],
options={
'abstract': False,
"abstract": False,
},
),
migrations.AlterField(
model_name='sender',
name='slug',
model_name="sender",
name="slug",
field=models.SlugField(blank=True),
),
migrations.AddField(
model_name='document',
name='tags',
field=models.ManyToManyField(related_name='documents', to='documents.Tag'),
model_name="document",
name="tags",
field=models.ManyToManyField(related_name="documents", to="documents.Tag"),
),
]

45
src/documents/migrations/0007_auto_20160126_2114.py
View File

@@ -8,23 +8,50 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0006_auto_20160123_0430'),
("documents", "0006_auto_20160123_0430"),
]
operations = [
migrations.AddField(
model_name='tag',
name='match',
model_name="tag",
name="match",
field=models.CharField(blank=True, max_length=256),
),
migrations.AddField(
model_name='tag',
name='matching_algorithm',
field=models.PositiveIntegerField(blank=True, choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression')], help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. If you don\'t know what a regex is, you probably don\'t want this option.', null=True),
model_name="tag",
name="matching_algorithm",
field=models.PositiveIntegerField(
blank=True,
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
],
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. If you don\'t know what a regex is, you probably don\'t want this option.',
null=True,
),
),
migrations.AlterField(
model_name='tag',
name='colour',
field=models.PositiveIntegerField(choices=[(1, '#a6cee3'), (2, '#1f78b4'), (3, '#b2df8a'), (4, '#33a02c'), (5, '#fb9a99'), (6, '#e31a1c'), (7, '#fdbf6f'), (8, '#ff7f00'), (9, '#cab2d6'), (10, '#6a3d9a'), (11, '#b15928'), (12, '#000000'), (13, '#cccccc')], default=1),
model_name="tag",
name="colour",
field=models.PositiveIntegerField(
choices=[
(1, "#a6cee3"),
(2, "#1f78b4"),
(3, "#b2df8a"),
(4, "#33a02c"),
(5, "#fb9a99"),
(6, "#e31a1c"),
(7, "#fdbf6f"),
(8, "#ff7f00"),
(9, "#cab2d6"),
(10, "#6a3d9a"),
(11, "#b15928"),
(12, "#000000"),
(13, "#cccccc"),
],
default=1,
),
),
]

28
src/documents/migrations/0008_document_file_type.py
View File

@@ -8,20 +8,32 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0007_auto_20160126_2114'),
("documents", "0007_auto_20160126_2114"),
]
operations = [
migrations.AddField(
model_name='document',
name='file_type',
field=models.CharField(choices=[('pdf', 'PDF'), ('png', 'PNG'), ('jpg', 'JPG'), ('gif', 'GIF'), ('tiff', 'TIFF')], default='pdf', editable=False, max_length=4),
model_name="document",
name="file_type",
field=models.CharField(
choices=[
("pdf", "PDF"),
("png", "PNG"),
("jpg", "JPG"),
("gif", "GIF"),
("tiff", "TIFF"),
],
default="pdf",
editable=False,
max_length=4,
),
preserve_default=False,
),
migrations.AlterField(
model_name='document',
name='tags',
field=models.ManyToManyField(blank=True, related_name='documents', to='documents.Tag'),
model_name="document",
name="tags",
field=models.ManyToManyField(
blank=True, related_name="documents", to="documents.Tag"
),
),
]

17
src/documents/migrations/0009_auto_20160214_0040.py
View File

@@ -8,13 +8,22 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0008_document_file_type'),
("documents", "0008_document_file_type"),
]
operations = [
migrations.AlterField(
model_name='tag',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression')], default=1, help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. If you don\'t know what a regex is, you probably don\'t want this option.'),
model_name="tag",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
],
default=1,
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. If you don\'t know what a regex is, you probably don\'t want this option.',
),
),
]

45
src/documents/migrations/0010_log.py
View File

@@ -8,23 +8,48 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0009_auto_20160214_0040'),
("documents", "0009_auto_20160214_0040"),
]
operations = [
migrations.CreateModel(
name='Log',
name="Log",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('group', models.UUIDField(blank=True)),
('message', models.TextField()),
('level', models.PositiveIntegerField(choices=[(10, 'Debugging'), (20, 'Informational'), (30, 'Warning'), (40, 'Error'), (50, 'Critical')], default=20)),
('component', models.PositiveIntegerField(choices=[(1, 'Consumer'), (2, 'Mail Fetcher')])),
('created', models.DateTimeField(auto_now_add=True)),
('modified', models.DateTimeField(auto_now=True)),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("group", models.UUIDField(blank=True)),
("message", models.TextField()),
(
"level",
models.PositiveIntegerField(
choices=[
(10, "Debugging"),
(20, "Informational"),
(30, "Warning"),
(40, "Error"),
(50, "Critical"),
],
default=20,
),
),
(
"component",
models.PositiveIntegerField(
choices=[(1, "Consumer"), (2, "Mail Fetcher")]
),
),
("created", models.DateTimeField(auto_now_add=True)),
("modified", models.DateTimeField(auto_now=True)),
],
options={
'ordering': ('-modified',),
"ordering": ("-modified",),
},
),
]

16
src/documents/migrations/0011_auto_20160303_1929.py
View File

@@ -8,21 +8,21 @@ from django.db import migrations
class Migration(migrations.Migration):
atomic = False
dependencies = [
('documents', '0010_log'),
("documents", "0010_log"),
]
operations = [
migrations.RenameModel(
old_name='Sender',
new_name='Correspondent',
old_name="Sender",
new_name="Correspondent",
),
migrations.AlterModelOptions(
name='document',
options={'ordering': ('correspondent', 'title')},
name="document",
options={"ordering": ("correspondent", "title")},
),
migrations.RenameField(
model_name='document',
old_name='sender',
new_name='correspondent',
model_name="document",
old_name="sender",
new_name="correspondent",
),
]

75
src/documents/migrations/0012_auto_20160305_0040.py
View File

@@ -23,37 +23,40 @@ class GnuPG(object):
@classmethod
def decrypted(cls, file_handle):
return cls.gpg.decrypt_file(
file_handle, passphrase=settings.PASSPHRASE).data
return cls.gpg.decrypt_file(file_handle, passphrase=settings.PASSPHRASE).data
@classmethod
def encrypted(cls, file_handle):
return cls.gpg.encrypt_file(
file_handle,
recipients=None,
passphrase=settings.PASSPHRASE,
symmetric=True
file_handle, recipients=None, passphrase=settings.PASSPHRASE, symmetric=True
).data
def move_documents_and_create_thumbnails(apps, schema_editor):
os.makedirs(os.path.join(settings.MEDIA_ROOT, "documents", "originals"), exist_ok=True)
os.makedirs(os.path.join(settings.MEDIA_ROOT, "documents", "thumbnails"), exist_ok=True)
os.makedirs(
os.path.join(settings.MEDIA_ROOT, "documents", "originals"), exist_ok=True
)
os.makedirs(
os.path.join(settings.MEDIA_ROOT, "documents", "thumbnails"), exist_ok=True
)
documents = os.listdir(os.path.join(settings.MEDIA_ROOT, "documents"))
if set(documents) == {"originals", "thumbnails"}:
return
print(colourise(
"\n\n"
" This is a one-time only migration to generate thumbnails for all of your\n"
" documents so that future UIs will have something to work with. If you have\n"
" a lot of documents though, this may take a while, so a coffee break may be\n"
" in order."
"\n", opts=("bold",)
))
print(
colourise(
"\n\n"
" This is a one-time only migration to generate thumbnails for all of your\n"
" documents so that future UIs will have something to work with. If you have\n"
" a lot of documents though, this may take a while, so a coffee break may be\n"
" in order."
"\n",
opts=("bold",),
)
)
try:
os.makedirs(settings.SCRATCH_DIR)
@@ -65,16 +68,16 @@ def move_documents_and_create_thumbnails(apps, schema_editor):
if not f.endswith("gpg"):
continue
print(" {} {} {}".format(
colourise("*", fg="green"),
colourise("Generating a thumbnail for", fg="white"),
colourise(f, fg="cyan")
))
print(
" {} {} {}".format(
colourise("*", fg="green"),
colourise("Generating a thumbnail for", fg="white"),
colourise(f, fg="cyan"),
)
)
thumb_temp = tempfile.mkdtemp(
prefix="paperless", dir=settings.SCRATCH_DIR)
orig_temp = tempfile.mkdtemp(
prefix="paperless", dir=settings.SCRATCH_DIR)
thumb_temp = tempfile.mkdtemp(prefix="paperless", dir=settings.SCRATCH_DIR)
orig_temp = tempfile.mkdtemp(prefix="paperless", dir=settings.SCRATCH_DIR)
orig_source = os.path.join(settings.MEDIA_ROOT, "documents", f)
orig_target = os.path.join(orig_temp, f.replace(".gpg", ""))
@@ -83,20 +86,24 @@ def move_documents_and_create_thumbnails(apps, schema_editor):
with open(orig_target, "wb") as unencrypted:
unencrypted.write(GnuPG.decrypted(encrypted))
subprocess.Popen((
settings.CONVERT_BINARY,
"-scale", "500x5000",
"-alpha", "remove",
orig_target,
os.path.join(thumb_temp, "convert-%04d.png")
)).wait()
subprocess.Popen(
(
settings.CONVERT_BINARY,
"-scale",
"500x5000",
"-alpha",
"remove",
orig_target,
os.path.join(thumb_temp, "convert-%04d.png"),
)
).wait()
thumb_source = os.path.join(thumb_temp, "convert-0000.png")
thumb_target = os.path.join(
settings.MEDIA_ROOT,
"documents",
"thumbnails",
re.sub(r"(\d+)\.\w+(\.gpg)", "\\1.png\\2", f)
re.sub(r"(\d+)\.\w+(\.gpg)", "\\1.png\\2", f),
)
with open(thumb_source, "rb") as unencrypted:
with open(thumb_target, "wb") as encrypted:
@@ -113,7 +120,7 @@ def move_documents_and_create_thumbnails(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '0011_auto_20160303_1929'),
("documents", "0011_auto_20160303_1929"),
]
operations = [

29
src/documents/migrations/0013_auto_20160325_2111.py
View File

@@ -9,27 +9,36 @@ import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
('documents', '0012_auto_20160305_0040'),
("documents", "0012_auto_20160305_0040"),
]
operations = [
migrations.AddField(
model_name='correspondent',
name='match',
model_name="correspondent",
name="match",
field=models.CharField(blank=True, max_length=256),
),
migrations.AddField(
model_name='correspondent',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression')], default=1, help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. If you don\'t know what a regex is, you probably don\'t want this option.'),
model_name="correspondent",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
],
default=1,
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. If you don\'t know what a regex is, you probably don\'t want this option.',
),
),
migrations.AlterField(
model_name='document',
name='created',
model_name="document",
name="created",
field=models.DateTimeField(default=django.utils.timezone.now),
),
migrations.RemoveField(
model_name='log',
name='component',
model_name="log",
name="component",
),
]

103
src/documents/migrations/0014_document_checksum.py
View File

@@ -22,16 +22,12 @@ class GnuPG(object):
@classmethod
def decrypted(cls, file_handle):
return cls.gpg.decrypt_file(
file_handle, passphrase=settings.PASSPHRASE).data
return cls.gpg.decrypt_file(file_handle, passphrase=settings.PASSPHRASE).data
@classmethod
def encrypted(cls, file_handle):
return cls.gpg.encrypt_file(
file_handle,
recipients=None,
passphrase=settings.PASSPHRASE,
symmetric=True
file_handle, recipients=None, passphrase=settings.PASSPHRASE, symmetric=True
).data
@@ -53,8 +49,7 @@ class Document(object):
def __str__(self):
created = self.created.strftime("%Y%m%d%H%M%S")
if self.correspondent and self.title:
return "{}: {} - {}".format(
created, self.correspondent, self.title)
return "{}: {} - {}".format(created, self.correspondent, self.title)
if self.correspondent or self.title:
return "{}: {}".format(created, self.correspondent or self.title)
return str(created)
@@ -65,7 +60,7 @@ class Document(object):
settings.MEDIA_ROOT,
"documents",
"originals",
"{:07}.{}.gpg".format(self.pk, self.file_type)
"{:07}.{}.gpg".format(self.pk, self.file_type),
)
@property
@@ -84,38 +79,62 @@ def set_checksums(apps, schema_editor):
if not document_model.objects.all().exists():
return
print(colourise(
"\n\n"
" This is a one-time only migration to generate checksums for all\n"
" of your existing documents. If you have a lot of documents\n"
" though, this may take a while, so a coffee break may be in\n"
" order."
"\n", opts=("bold",)
))
print(
colourise(
"\n\n"
" This is a one-time only migration to generate checksums for all\n"
" of your existing documents. If you have a lot of documents\n"
" though, this may take a while, so a coffee break may be in\n"
" order."
"\n",
opts=("bold",),
)
)
sums = {}
for d in document_model.objects.all():
document = Document(d)
print(" {} {} {}".format(
colourise("*", fg="green"),
colourise("Generating a checksum for", fg="white"),
colourise(document.file_name, fg="cyan")
))
print(
" {} {} {}".format(
colourise("*", fg="green"),
colourise("Generating a checksum for", fg="white"),
colourise(document.file_name, fg="cyan"),
)
)
with document.source_file as encrypted:
checksum = hashlib.md5(GnuPG.decrypted(encrypted)).hexdigest()
if checksum in sums:
error = "\n{line}{p1}\n\n{doc1}\n{doc2}\n\n{p2}\n\n{code}\n\n{p3}{line}".format(
p1=colourise("It appears that you have two identical documents in your collection and \nPaperless no longer supports this (see issue #97). The documents in question\nare:", fg="yellow"),
p2=colourise("To fix this problem, you'll have to remove one of them from the database, a task\nmost easily done by running the following command in the same\ndirectory as manage.py:", fg="yellow"),
p3=colourise("When that's finished, re-run the migrate, and provided that there aren't any\nother duplicates, you should be good to go.", fg="yellow"),
doc1=colourise(" * {} (id: {})".format(sums[checksum][1], sums[checksum][0]), fg="red"),
doc2=colourise(" * {} (id: {})".format(document.file_name, document.pk), fg="red"),
code=colourise(" $ echo 'DELETE FROM documents_document WHERE id = {pk};' | ./manage.py dbshell".format(pk=document.pk), fg="green"),
line=colourise("\n{}\n".format("=" * 80), fg="white", opts=("bold",))
p1=colourise(
"It appears that you have two identical documents in your collection and \nPaperless no longer supports this (see issue #97). The documents in question\nare:",
fg="yellow",
),
p2=colourise(
"To fix this problem, you'll have to remove one of them from the database, a task\nmost easily done by running the following command in the same\ndirectory as manage.py:",
fg="yellow",
),
p3=colourise(
"When that's finished, re-run the migrate, and provided that there aren't any\nother duplicates, you should be good to go.",
fg="yellow",
),
doc1=colourise(
" * {} (id: {})".format(sums[checksum][1], sums[checksum][0]),
fg="red",
),
doc2=colourise(
" * {} (id: {})".format(document.file_name, document.pk), fg="red"
),
code=colourise(
" $ echo 'DELETE FROM documents_document WHERE id = {pk};' | ./manage.py dbshell".format(
pk=document.pk
),
fg="green",
),
line=colourise("\n{}\n".format("=" * 80), fg="white", opts=("bold",)),
)
raise RuntimeError(error)
sums[checksum] = (document.pk, document.file_name)
@@ -129,33 +148,35 @@ def do_nothing(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '0013_auto_20160325_2111'),
("documents", "0013_auto_20160325_2111"),
]
operations = [
migrations.AddField(
model_name='document',
name='checksum',
model_name="document",
name="checksum",
field=models.CharField(
default='-',
default="-",
db_index=True,
editable=False,
max_length=32,
help_text='The checksum of the original document (before it '
'was encrypted). We use this to prevent duplicate '
'document imports.',
help_text="The checksum of the original document (before it "
"was encrypted). We use this to prevent duplicate "
"document imports.",
),
preserve_default=False,
),
migrations.RunPython(set_checksums, do_nothing),
migrations.AlterField(
model_name='document',
name='created',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
model_name="document",
name="created",
field=models.DateTimeField(
db_index=True, default=django.utils.timezone.now
),
),
migrations.AlterField(
model_name='document',
name='modified',
model_name="document",
name="modified",
field=models.DateTimeField(auto_now=True, db_index=True),
),
]

21
src/documents/migrations/0015_add_insensitive_to_match.py
View File

@@ -8,23 +8,28 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0014_document_checksum'),
("documents", "0014_document_checksum"),
]
operations = [
migrations.AlterField(
model_name='document',
name='checksum',
field=models.CharField(editable=False, help_text='The checksum of the original document (before it was encrypted). We use this to prevent duplicate document imports.', max_length=32, unique=True),
model_name="document",
name="checksum",
field=models.CharField(
editable=False,
help_text="The checksum of the original document (before it was encrypted). We use this to prevent duplicate document imports.",
max_length=32,
unique=True,
),
),
migrations.AddField(
model_name='correspondent',
name='is_insensitive',
model_name="correspondent",
name="is_insensitive",
field=models.BooleanField(default=True),
),
migrations.AddField(
model_name='tag',
name='is_insensitive',
model_name="tag",
name="is_insensitive",
field=models.BooleanField(default=True),
),
]

12
src/documents/migrations/0016_auto_20170325_1558.py
View File

@@ -9,13 +9,17 @@ from django.conf import settings
class Migration(migrations.Migration):
dependencies = [
('documents', '0015_add_insensitive_to_match'),
("documents", "0015_add_insensitive_to_match"),
]
operations = [
migrations.AlterField(
model_name='document',
name='content',
field=models.TextField(blank=True, db_index=("mysql" not in settings.DATABASES["default"]["ENGINE"]), help_text='The raw, text-only data of the document. This field is primarily used for searching.'),
model_name="document",
name="content",
field=models.TextField(
blank=True,
db_index=("mysql" not in settings.DATABASES["default"]["ENGINE"]),
help_text="The raw, text-only data of the document. This field is primarily used for searching.",
),
),
]

34
src/documents/migrations/0017_auto_20170512_0507.py
View File

@@ -8,18 +8,38 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0016_auto_20170325_1558'),
("documents", "0016_auto_20170325_1558"),
]
operations = [
migrations.AlterField(
model_name='correspondent',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression'), (5, 'Fuzzy Match')], default=1, help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.'),
model_name="correspondent",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
(5, "Fuzzy Match"),
],
default=1,
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.',
),
),
migrations.AlterField(
model_name='tag',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression'), (5, 'Fuzzy Match')], default=1, help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.'),
model_name="tag",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
(5, "Fuzzy Match"),
],
default=1,
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.',
),
),
]

14
src/documents/migrations/0018_auto_20170715_1712.py
View File

@@ -9,13 +9,19 @@ import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('documents', '0017_auto_20170512_0507'),
("documents", "0017_auto_20170512_0507"),
]
operations = [
migrations.AlterField(
model_name='document',
name='correspondent',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='documents.Correspondent'),
model_name="document",
name="correspondent",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="documents",
to="documents.Correspondent",
),
),
]

2
src/documents/migrations/0019_add_consumer_user.py
View File

@@ -16,7 +16,7 @@ def reverse_func(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '0018_auto_20170715_1712'),
("documents", "0018_auto_20170715_1712"),
]
operations = [

12
src/documents/migrations/0020_document_added.py
View File

@@ -14,14 +14,16 @@ def set_added_time_to_created_time(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '0019_add_consumer_user'),
("documents", "0019_add_consumer_user"),
]
operations = [
migrations.AddField(
model_name='document',
name='added',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, editable=False),
model_name="document",
name="added",
field=models.DateTimeField(
db_index=True, default=django.utils.timezone.now, editable=False
),
),
migrations.RunPython(set_added_time_to_created_time)
migrations.RunPython(set_added_time_to_created_time),
]

33
src/documents/migrations/0021_document_storage_type.py
View File

@@ -8,23 +8,36 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '0020_document_added'),
("documents", "0020_document_added"),
]
operations = [
# Add the field with the default GPG-encrypted value
migrations.AddField(
model_name='document',
name='storage_type',
field=models.CharField(choices=[('unencrypted', 'Unencrypted'), ('gpg', 'Encrypted with GNU Privacy Guard')], default='gpg', editable=False, max_length=11),
model_name="document",
name="storage_type",
field=models.CharField(
choices=[
("unencrypted", "Unencrypted"),
("gpg", "Encrypted with GNU Privacy Guard"),
],
default="gpg",
editable=False,
max_length=11,
),
),
# Now that the field is added, change the default to unencrypted
migrations.AlterField(
model_name='document',
name='storage_type',
field=models.CharField(choices=[('unencrypted', 'Unencrypted'), ('gpg', 'Encrypted with GNU Privacy Guard')], default='unencrypted', editable=False, max_length=11),
model_name="document",
name="storage_type",
field=models.CharField(
choices=[
("unencrypted", "Unencrypted"),
("gpg", "Encrypted with GNU Privacy Guard"),
],
default="unencrypted",
editable=False,
max_length=11,
),
),
]

41
src/documents/migrations/0022_auto_20181007_1420.py
View File

@@ -15,38 +15,47 @@ def re_slug_all_the_things(apps, schema_editor):
for klass in (Tag, Correspondent):
for instance in klass.objects.all():
klass.objects.filter(
pk=instance.pk
).update(
slug=slugify(instance.slug)
)
klass.objects.filter(pk=instance.pk).update(slug=slugify(instance.slug))
class Migration(migrations.Migration):
dependencies = [
('documents', '0021_document_storage_type'),
("documents", "0021_document_storage_type"),
]
operations = [
migrations.AlterModelOptions(
name='tag',
options={'ordering': ('name',)},
name="tag",
options={"ordering": ("name",)},
),
migrations.AlterField(
model_name='correspondent',
name='slug',
model_name="correspondent",
name="slug",
field=models.SlugField(blank=True, editable=False),
),
migrations.AlterField(
model_name='document',
name='file_type',
field=models.CharField(choices=[('pdf', 'PDF'), ('png', 'PNG'), ('jpg', 'JPG'), ('gif', 'GIF'), ('tiff', 'TIFF'), ('txt', 'TXT'), ('csv', 'CSV'), ('md', 'MD')], editable=False, max_length=4),
model_name="document",
name="file_type",
field=models.CharField(
choices=[
("pdf", "PDF"),
("png", "PNG"),
("jpg", "JPG"),
("gif", "GIF"),
("tiff", "TIFF"),
("txt", "TXT"),
("csv", "CSV"),
("md", "MD"),
],
editable=False,
max_length=4,
),
),
migrations.AlterField(
model_name='tag',
name='slug',
model_name="tag",
name="slug",
field=models.SlugField(blank=True, editable=False),
),
migrations.RunPython(re_slug_all_the_things, migrations.RunPython.noop)
migrations.RunPython(re_slug_all_the_things, migrations.RunPython.noop),
]

20
src/documents/migrations/0023_document_current_filename.py
View File

@@ -20,18 +20,20 @@ def set_filename(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '0022_auto_20181007_1420'),
("documents", "0022_auto_20181007_1420"),
]
operations = [
migrations.AddField(
model_name='document',
name='filename',
field=models.FilePathField(default=None,
null=True,
editable=False,
help_text='Current filename in storage',
max_length=256),
model_name="document",
name="filename",
field=models.FilePathField(
default=None,
null=True,
editable=False,
help_text="Current filename in storage",
max_length=256,
),
),
migrations.RunPython(set_filename)
migrations.RunPython(set_filename),
]

134
src/documents/migrations/1000_update_paperless_all.py
View File

@@ -6,7 +6,7 @@ import django.db.models.deletion
def logs_set_default_group(apps, schema_editor):
Log = apps.get_model('documents', 'Log')
Log = apps.get_model("documents", "Log")
for log in Log.objects.all():
if log.group is None:
log.group = uuid.uuid4()
@@ -16,70 +16,132 @@ def logs_set_default_group(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '0023_document_current_filename'),
("documents", "0023_document_current_filename"),
]
operations = [
migrations.AddField(
model_name='document',
name='archive_serial_number',
field=models.IntegerField(blank=True, db_index=True, help_text='The position of this document in your physical document archive.', null=True, unique=True),
model_name="document",
name="archive_serial_number",
field=models.IntegerField(
blank=True,
db_index=True,
help_text="The position of this document in your physical document archive.",
null=True,
unique=True,
),
),
migrations.AddField(
model_name='tag',
name='is_inbox_tag',
field=models.BooleanField(default=False, help_text='Marks this tag as an inbox tag: All newly consumed documents will be tagged with inbox tags.'),
model_name="tag",
name="is_inbox_tag",
field=models.BooleanField(
default=False,
help_text="Marks this tag as an inbox tag: All newly consumed documents will be tagged with inbox tags.",
),
),
migrations.CreateModel(
name='DocumentType',
name="DocumentType",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128, unique=True)),
('slug', models.SlugField(blank=True, editable=False)),
('match', models.CharField(blank=True, max_length=256)),
('matching_algorithm', models.PositiveIntegerField(choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression'), (5, 'Fuzzy Match'), (6, 'Automatic Classification')], default=1, help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.')),
('is_insensitive', models.BooleanField(default=True)),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=128, unique=True)),
("slug", models.SlugField(blank=True, editable=False)),
("match", models.CharField(blank=True, max_length=256)),
(
"matching_algorithm",
models.PositiveIntegerField(
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
(5, "Fuzzy Match"),
(6, "Automatic Classification"),
],
default=1,
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.',
),
),
("is_insensitive", models.BooleanField(default=True)),
],
options={
'abstract': False,
'ordering': ('name',),
"abstract": False,
"ordering": ("name",),
},
),
migrations.AddField(
model_name='document',
name='document_type',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='documents.documenttype'),
model_name="document",
name="document_type",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="documents",
to="documents.documenttype",
),
),
migrations.AlterField(
model_name='correspondent',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression'), (5, 'Fuzzy Match'), (6, 'Automatic Classification')], default=1, help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.'),
model_name="correspondent",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
(5, "Fuzzy Match"),
(6, "Automatic Classification"),
],
default=1,
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.',
),
),
migrations.AlterField(
model_name='tag',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any'), (2, 'All'), (3, 'Literal'), (4, 'Regular Expression'), (5, 'Fuzzy Match'), (6, 'Automatic Classification')], default=1, help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.'),
model_name="tag",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any"),
(2, "All"),
(3, "Literal"),
(4, "Regular Expression"),
(5, "Fuzzy Match"),
(6, "Automatic Classification"),
],
default=1,
help_text='Which algorithm you want to use when matching text to the OCR\'d PDF. Here, "any" looks for any occurrence of any word provided in the PDF, while "all" requires that every word provided appear in the PDF, albeit not in the order provided. A "literal" match means that the text you enter must appear in the PDF exactly as you\'ve entered it, and "regular expression" uses a regex to match the PDF. (If you don\'t know what a regex is, you probably don\'t want this option.) Finally, a "fuzzy match" looks for words or phrases that are mostly—but not exactly—the same, which can be useful for matching against documents containg imperfections that foil accurate OCR.',
),
),
migrations.AlterField(
model_name='document',
name='content',
field=models.TextField(blank=True, help_text='The raw, text-only data of the document. This field is primarily used for searching.'),
model_name="document",
name="content",
field=models.TextField(
blank=True,
help_text="The raw, text-only data of the document. This field is primarily used for searching.",
),
),
migrations.AlterModelOptions(
name='log',
options={'ordering': ('-created',)},
name="log",
options={"ordering": ("-created",)},
),
migrations.RemoveField(
model_name='log',
name='modified',
model_name="log",
name="modified",
),
migrations.AlterField(
model_name='log',
name='group',
model_name="log",
name="group",
field=models.UUIDField(blank=True, null=True),
),
migrations.RunPython(
code=django.db.migrations.operations.special.RunPython.noop,
reverse_code=logs_set_default_group
reverse_code=logs_set_default_group,
),
]

24
src/documents/migrations/1001_auto_20201109_1636.py
View File

@@ -7,22 +7,28 @@ from django_q.tasks import schedule
def add_schedules(apps, schema_editor):
schedule('documents.tasks.train_classifier', name="Train the classifier", schedule_type=Schedule.HOURLY)
schedule('documents.tasks.index_optimize', name="Optimize the index", schedule_type=Schedule.DAILY)
schedule(
"documents.tasks.train_classifier",
name="Train the classifier",
schedule_type=Schedule.HOURLY,
)
schedule(
"documents.tasks.index_optimize",
name="Optimize the index",
schedule_type=Schedule.DAILY,
)
def remove_schedules(apps, schema_editor):
Schedule.objects.filter(func='documents.tasks.train_classifier').delete()
Schedule.objects.filter(func='documents.tasks.index_optimize').delete()
Schedule.objects.filter(func="documents.tasks.train_classifier").delete()
Schedule.objects.filter(func="documents.tasks.index_optimize").delete()
class Migration(migrations.Migration):
dependencies = [
('documents', '1000_update_paperless_all'),
('django_q', '0013_task_attempt_count'),
("documents", "1000_update_paperless_all"),
("django_q", "0013_task_attempt_count"),
]
operations = [
RunPython(add_schedules, remove_schedules)
]
operations = [RunPython(add_schedules, remove_schedules)]

14
src/documents/migrations/1002_auto_20201111_1105.py
View File

@@ -6,13 +6,19 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '1001_auto_20201109_1636'),
("documents", "1001_auto_20201109_1636"),
]
operations = [
migrations.AlterField(
model_name='document',
name='filename',
field=models.FilePathField(default=None, editable=False, help_text='Current filename in storage', max_length=1024, null=True),
model_name="document",
name="filename",
field=models.FilePathField(
default=None,
editable=False,
help_text="Current filename in storage",
max_length=1024,
null=True,
),
),
]

35
src/documents/migrations/1003_mime_types.py
View File

@@ -20,10 +20,7 @@ def source_path(self):
if self.storage_type == STORAGE_TYPE_GPG:
fname += ".gpg"
return os.path.join(
settings.ORIGINALS_DIR,
fname
)
return os.path.join(settings.ORIGINALS_DIR, fname)
def add_mime_types(apps, schema_editor):
@@ -49,43 +46,51 @@ def add_file_extensions(apps, schema_editor):
documents = Document.objects.all()
for d in documents:
d.file_type = os.path.splitext(d.filename)[1].strip('.')
d.file_type = os.path.splitext(d.filename)[1].strip(".")
d.save()
class Migration(migrations.Migration):
dependencies = [
('documents', '1002_auto_20201111_1105'),
("documents", "1002_auto_20201111_1105"),
]
operations = [
migrations.AddField(
model_name='document',
name='mime_type',
model_name="document",
name="mime_type",
field=models.CharField(default="-", editable=False, max_length=256),
preserve_default=False,
),
migrations.RunPython(add_mime_types, migrations.RunPython.noop),
# This operation is here so that we can revert the entire migration:
# By allowing this field to be blank and null, we can revert the
# remove operation further down and the database won't complain about
# NOT NULL violations.
migrations.AlterField(
model_name='document',
name='file_type',
model_name="document",
name="file_type",
field=models.CharField(
choices=[('pdf', 'PDF'), ('png', 'PNG'), ('jpg', 'JPG'), ('gif', 'GIF'), ('tiff', 'TIFF'), ('txt', 'TXT'), ('csv', 'CSV'), ('md', 'MD')],
choices=[
("pdf", "PDF"),
("png", "PNG"),
("jpg", "JPG"),
("gif", "GIF"),
("tiff", "TIFF"),
("txt", "TXT"),
("csv", "CSV"),
("md", "MD"),
],
editable=False,
max_length=4,
null=True,
blank=True
blank=True,
),
),
migrations.RunPython(migrations.RunPython.noop, add_file_extensions),
migrations.RemoveField(
model_name='document',
name='file_type',
model_name="document",
name="file_type",
),
]

16
src/documents/migrations/1004_sanity_check_schedule.py
View File

@@ -7,20 +7,22 @@ from django_q.tasks import schedule
def add_schedules(apps, schema_editor):
schedule('documents.tasks.sanity_check', name="Perform sanity check", schedule_type=Schedule.WEEKLY)
schedule(
"documents.tasks.sanity_check",
name="Perform sanity check",
schedule_type=Schedule.WEEKLY,
)
def remove_schedules(apps, schema_editor):
Schedule.objects.filter(func='documents.tasks.sanity_check').delete()
Schedule.objects.filter(func="documents.tasks.sanity_check").delete()
class Migration(migrations.Migration):
dependencies = [
('documents', '1003_mime_types'),
('django_q', '0013_task_attempt_count'),
("documents", "1003_mime_types"),
("django_q", "0013_task_attempt_count"),
]
operations = [
RunPython(add_schedules, remove_schedules)
]
operations = [RunPython(add_schedules, remove_schedules)]

25
src/documents/migrations/1005_checksums.py
View File

@@ -6,18 +6,29 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '1004_sanity_check_schedule'),
("documents", "1004_sanity_check_schedule"),
]
operations = [
migrations.AddField(
model_name='document',
name='archive_checksum',
field=models.CharField(blank=True, editable=False, help_text='The checksum of the archived document.', max_length=32, null=True),
model_name="document",
name="archive_checksum",
field=models.CharField(
blank=True,
editable=False,
help_text="The checksum of the archived document.",
max_length=32,
null=True,
),
),
migrations.AlterField(
model_name='document',
name='checksum',
field=models.CharField(editable=False, help_text='The checksum of the original document.', max_length=32, unique=True),
model_name="document",
name="checksum",
field=models.CharField(
editable=False,
help_text="The checksum of the original document.",
max_length=32,
unique=True,
),
),
]

14
src/documents/migrations/1006_auto_20201208_2209.py
View File

@@ -6,20 +6,20 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('documents', '1005_checksums'),
("documents", "1005_checksums"),
]
operations = [
migrations.RemoveField(
model_name='correspondent',
name='slug',
model_name="correspondent",
name="slug",
),
migrations.RemoveField(
model_name='documenttype',
name='slug',
model_name="documenttype",
name="slug",
),
migrations.RemoveField(
model_name='tag',
name='slug',
model_name="tag",
name="slug",
),
]

81
src/documents/migrations/1007_savedview_savedviewfilterrule.py
View File

@@ -9,29 +9,82 @@ class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('documents', '1006_auto_20201208_2209'),
("documents", "1006_auto_20201208_2209"),
]
operations = [
migrations.CreateModel(
name='SavedView',
name="SavedView",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128)),
('show_on_dashboard', models.BooleanField()),
('show_in_sidebar', models.BooleanField()),
('sort_field', models.CharField(max_length=128)),
('sort_reverse', models.BooleanField(default=False)),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=128)),
("show_on_dashboard", models.BooleanField()),
("show_in_sidebar", models.BooleanField()),
("sort_field", models.CharField(max_length=128)),
("sort_reverse", models.BooleanField(default=False)),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
],
),
migrations.CreateModel(
name='SavedViewFilterRule',
name="SavedViewFilterRule",
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('rule_type', models.PositiveIntegerField(choices=[(0, 'Title contains'), (1, 'Content contains'), (2, 'ASN is'), (3, 'Correspondent is'), (4, 'Document type is'), (5, 'Is in inbox'), (6, 'Has tag'), (7, 'Has any tag'), (8, 'Created before'), (9, 'Created after'), (10, 'Created year is'), (11, 'Created month is'), (12, 'Created day is'), (13, 'Added before'), (14, 'Added after'), (15, 'Modified before'), (16, 'Modified after'), (17, 'Does not have tag')])),
('value', models.CharField(max_length=128)),
('saved_view', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='filter_rules', to='documents.savedview')),
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"rule_type",
models.PositiveIntegerField(
choices=[
(0, "Title contains"),
(1, "Content contains"),
(2, "ASN is"),
(3, "Correspondent is"),
(4, "Document type is"),
(5, "Is in inbox"),
(6, "Has tag"),
(7, "Has any tag"),
(8, "Created before"),
(9, "Created after"),
(10, "Created year is"),
(11, "Created month is"),
(12, "Created day is"),
(13, "Added before"),
(14, "Added after"),
(15, "Modified before"),
(16, "Modified after"),
(17, "Does not have tag"),
]
),
),
("value", models.CharField(max_length=128)),
(
"saved_view",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="filter_rules",
to="documents.savedview",
),
),
],
),
]

22
src/documents/migrations/1008_auto_20201216_1736.py
View File

@@ -7,28 +7,28 @@ import django.db.models.functions.text
class Migration(migrations.Migration):
dependencies = [
('documents', '1007_savedview_savedviewfilterrule'),
("documents", "1007_savedview_savedviewfilterrule"),
]
operations = [
migrations.AlterModelOptions(
name='correspondent',
options={'ordering': (django.db.models.functions.text.Lower('name'),)},
name="correspondent",
options={"ordering": (django.db.models.functions.text.Lower("name"),)},
),
migrations.AlterModelOptions(
name='document',
options={'ordering': ('-created',)},
name="document",
options={"ordering": ("-created",)},
),
migrations.AlterModelOptions(
name='documenttype',
options={'ordering': (django.db.models.functions.text.Lower('name'),)},
name="documenttype",
options={"ordering": (django.db.models.functions.text.Lower("name"),)},
),
migrations.AlterModelOptions(
name='savedview',
options={'ordering': (django.db.models.functions.text.Lower('name'),)},
name="savedview",
options={"ordering": (django.db.models.functions.text.Lower("name"),)},
),
migrations.AlterModelOptions(
name='tag',
options={'ordering': (django.db.models.functions.text.Lower('name'),)},
name="tag",
options={"ordering": (django.db.models.functions.text.Lower("name"),)},
),
]

18
src/documents/migrations/1009_auto_20201216_2005.py
View File

@@ -6,24 +6,24 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('documents', '1008_auto_20201216_1736'),
("documents", "1008_auto_20201216_1736"),
]
operations = [
migrations.AlterModelOptions(
name='correspondent',
options={'ordering': ('name',)},
name="correspondent",
options={"ordering": ("name",)},
),
migrations.AlterModelOptions(
name='documenttype',
options={'ordering': ('name',)},
name="documenttype",
options={"ordering": ("name",)},
),
migrations.AlterModelOptions(
name='savedview',
options={'ordering': ('name',)},
name="savedview",
options={"ordering": ("name",)},
),
migrations.AlterModelOptions(
name='tag',
options={'ordering': ('name',)},
name="tag",
options={"ordering": ("name",)},
),
]

6
src/documents/migrations/1010_auto_20210101_2159.py
View File

@@ -6,13 +6,13 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '1009_auto_20201216_2005'),
("documents", "1009_auto_20201216_2005"),
]
operations = [
migrations.AlterField(
model_name='savedviewfilterrule',
name='value',
model_name="savedviewfilterrule",
name="value",
field=models.CharField(blank=True, max_length=128, null=True),
),
]

468
src/documents/migrations/1011_auto_20210101_2340.py
View File

@@ -10,241 +10,433 @@ class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('documents', '1010_auto_20210101_2159'),
("documents", "1010_auto_20210101_2159"),
]
operations = [
migrations.AlterModelOptions(
name='correspondent',
options={'ordering': ('name',), 'verbose_name': 'correspondent', 'verbose_name_plural': 'correspondents'},
name="correspondent",
options={
"ordering": ("name",),
"verbose_name": "correspondent",
"verbose_name_plural": "correspondents",
},
),
migrations.AlterModelOptions(
name='document',
options={'ordering': ('-created',), 'verbose_name': 'document', 'verbose_name_plural': 'documents'},
name="document",
options={
"ordering": ("-created",),
"verbose_name": "document",
"verbose_name_plural": "documents",
},
),
migrations.AlterModelOptions(
name='documenttype',
options={'verbose_name': 'document type', 'verbose_name_plural': 'document types'},
name="documenttype",
options={
"verbose_name": "document type",
"verbose_name_plural": "document types",
},
),
migrations.AlterModelOptions(
name='log',
options={'ordering': ('-created',), 'verbose_name': 'log', 'verbose_name_plural': 'logs'},
name="log",
options={
"ordering": ("-created",),
"verbose_name": "log",
"verbose_name_plural": "logs",
},
),
migrations.AlterModelOptions(
name='savedview',
options={'ordering': ('name',), 'verbose_name': 'saved view', 'verbose_name_plural': 'saved views'},
name="savedview",
options={
"ordering": ("name",),
"verbose_name": "saved view",
"verbose_name_plural": "saved views",
},
),
migrations.AlterModelOptions(
name='savedviewfilterrule',
options={'verbose_name': 'filter rule', 'verbose_name_plural': 'filter rules'},
name="savedviewfilterrule",
options={
"verbose_name": "filter rule",
"verbose_name_plural": "filter rules",
},
),
migrations.AlterModelOptions(
name='tag',
options={'verbose_name': 'tag', 'verbose_name_plural': 'tags'},
name="tag",
options={"verbose_name": "tag", "verbose_name_plural": "tags"},
),
migrations.AlterField(
model_name='correspondent',
name='is_insensitive',
field=models.BooleanField(default=True, verbose_name='is insensitive'),
model_name="correspondent",
name="is_insensitive",
field=models.BooleanField(default=True, verbose_name="is insensitive"),
),
migrations.AlterField(
model_name='correspondent',
name='match',
field=models.CharField(blank=True, max_length=256, verbose_name='match'),
model_name="correspondent",
name="match",
field=models.CharField(blank=True, max_length=256, verbose_name="match"),
),
migrations.AlterField(
model_name='correspondent',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
model_name="correspondent",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any word"),
(2, "All words"),
(3, "Exact match"),
(4, "Regular expression"),
(5, "Fuzzy word"),
(6, "Automatic"),
],
default=1,
verbose_name="matching algorithm",
),
),
migrations.AlterField(
model_name='correspondent',
name='name',
field=models.CharField(max_length=128, unique=True, verbose_name='name'),
model_name="correspondent",
name="name",
field=models.CharField(max_length=128, unique=True, verbose_name="name"),
),
migrations.AlterField(
model_name='document',
name='added',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, editable=False, verbose_name='added'),
model_name="document",
name="added",
field=models.DateTimeField(
db_index=True,
default=django.utils.timezone.now,
editable=False,
verbose_name="added",
),
),
migrations.AlterField(
model_name='document',
name='archive_checksum',
field=models.CharField(blank=True, editable=False, help_text='The checksum of the archived document.', max_length=32, null=True, verbose_name='archive checksum'),
model_name="document",
name="archive_checksum",
field=models.CharField(
blank=True,
editable=False,
help_text="The checksum of the archived document.",
max_length=32,
null=True,
verbose_name="archive checksum",
),
),
migrations.AlterField(
model_name='document',
name='archive_serial_number',
field=models.IntegerField(blank=True, db_index=True, help_text='The position of this document in your physical document archive.', null=True, unique=True, verbose_name='archive serial number'),
model_name="document",
name="archive_serial_number",
field=models.IntegerField(
blank=True,
db_index=True,
help_text="The position of this document in your physical document archive.",
null=True,
unique=True,
verbose_name="archive serial number",
),
),
migrations.AlterField(
model_name='document',
name='checksum',
field=models.CharField(editable=False, help_text='The checksum of the original document.', max_length=32, unique=True, verbose_name='checksum'),
model_name="document",
name="checksum",
field=models.CharField(
editable=False,
help_text="The checksum of the original document.",
max_length=32,
unique=True,
verbose_name="checksum",
),
),
migrations.AlterField(
model_name='document',
name='content',
field=models.TextField(blank=True, help_text='The raw, text-only data of the document. This field is primarily used for searching.', verbose_name='content'),
model_name="document",
name="content",
field=models.TextField(
blank=True,
help_text="The raw, text-only data of the document. This field is primarily used for searching.",
verbose_name="content",
),
),
migrations.AlterField(
model_name='document',
name='correspondent',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='documents.correspondent', verbose_name='correspondent'),
model_name="document",
name="correspondent",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="documents",
to="documents.correspondent",
verbose_name="correspondent",
),
),
migrations.AlterField(
model_name='document',
name='created',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, verbose_name='created'),
model_name="document",
name="created",
field=models.DateTimeField(
db_index=True, default=django.utils.timezone.now, verbose_name="created"
),
),
migrations.AlterField(
model_name='document',
name='document_type',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to='documents.documenttype', verbose_name='document type'),
model_name="document",
name="document_type",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="documents",
to="documents.documenttype",
verbose_name="document type",
),
),
migrations.AlterField(
model_name='document',
name='filename',
field=models.FilePathField(default=None, editable=False, help_text='Current filename in storage', max_length=1024, null=True, verbose_name='filename'),
model_name="document",
name="filename",
field=models.FilePathField(
default=None,
editable=False,
help_text="Current filename in storage",
max_length=1024,
null=True,
verbose_name="filename",
),
),
migrations.AlterField(
model_name='document',
name='mime_type',
field=models.CharField(editable=False, max_length=256, verbose_name='mime type'),
model_name="document",
name="mime_type",
field=models.CharField(
editable=False, max_length=256, verbose_name="mime type"
),
),
migrations.AlterField(
model_name='document',
name='modified',
field=models.DateTimeField(auto_now=True, db_index=True, verbose_name='modified'),
model_name="document",
name="modified",
field=models.DateTimeField(
auto_now=True, db_index=True, verbose_name="modified"
),
),
migrations.AlterField(
model_name='document',
name='storage_type',
field=models.CharField(choices=[('unencrypted', 'Unencrypted'), ('gpg', 'Encrypted with GNU Privacy Guard')], default='unencrypted', editable=False, max_length=11, verbose_name='storage type'),
model_name="document",
name="storage_type",
field=models.CharField(
choices=[
("unencrypted", "Unencrypted"),
("gpg", "Encrypted with GNU Privacy Guard"),
],
default="unencrypted",
editable=False,
max_length=11,
verbose_name="storage type",
),
),
migrations.AlterField(
model_name='document',
name='tags',
field=models.ManyToManyField(blank=True, related_name='documents', to='documents.Tag', verbose_name='tags'),
model_name="document",
name="tags",
field=models.ManyToManyField(
blank=True,
related_name="documents",
to="documents.Tag",
verbose_name="tags",
),
),
migrations.AlterField(
model_name='document',
name='title',
field=models.CharField(blank=True, db_index=True, max_length=128, verbose_name='title'),
model_name="document",
name="title",
field=models.CharField(
blank=True, db_index=True, max_length=128, verbose_name="title"
),
),
migrations.AlterField(
model_name='documenttype',
name='is_insensitive',
field=models.BooleanField(default=True, verbose_name='is insensitive'),
model_name="documenttype",
name="is_insensitive",
field=models.BooleanField(default=True, verbose_name="is insensitive"),
),
migrations.AlterField(
model_name='documenttype',
name='match',
field=models.CharField(blank=True, max_length=256, verbose_name='match'),
model_name="documenttype",
name="match",
field=models.CharField(blank=True, max_length=256, verbose_name="match"),
),
migrations.AlterField(
model_name='documenttype',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
model_name="documenttype",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any word"),
(2, "All words"),
(3, "Exact match"),
(4, "Regular expression"),
(5, "Fuzzy word"),
(6, "Automatic"),
],
default=1,
verbose_name="matching algorithm",
),
),
migrations.AlterField(
model_name='documenttype',
name='name',
field=models.CharField(max_length=128, unique=True, verbose_name='name'),
model_name="documenttype",
name="name",
field=models.CharField(max_length=128, unique=True, verbose_name="name"),
),
migrations.AlterField(
model_name='log',
name='created',
field=models.DateTimeField(auto_now_add=True, verbose_name='created'),
model_name="log",
name="created",
field=models.DateTimeField(auto_now_add=True, verbose_name="created"),
),
migrations.AlterField(
model_name='log',
name='group',
field=models.UUIDField(blank=True, null=True, verbose_name='group'),
model_name="log",
name="group",
field=models.UUIDField(blank=True, null=True, verbose_name="group"),
),
migrations.AlterField(
model_name='log',
name='level',
field=models.PositiveIntegerField(choices=[(10, 'debug'), (20, 'information'), (30, 'warning'), (40, 'error'), (50, 'critical')], default=20, verbose_name='level'),
model_name="log",
name="level",
field=models.PositiveIntegerField(
choices=[
(10, "debug"),
(20, "information"),
(30, "warning"),
(40, "error"),
(50, "critical"),
],
default=20,
verbose_name="level",
),
),
migrations.AlterField(
model_name='log',
name='message',
field=models.TextField(verbose_name='message'),
model_name="log",
name="message",
field=models.TextField(verbose_name="message"),
),
migrations.AlterField(
model_name='savedview',
name='name',
field=models.CharField(max_length=128, verbose_name='name'),
model_name="savedview",
name="name",
field=models.CharField(max_length=128, verbose_name="name"),
),
migrations.AlterField(
model_name='savedview',
name='show_in_sidebar',
field=models.BooleanField(verbose_name='show in sidebar'),
model_name="savedview",
name="show_in_sidebar",
field=models.BooleanField(verbose_name="show in sidebar"),
),
migrations.AlterField(
model_name='savedview',
name='show_on_dashboard',
field=models.BooleanField(verbose_name='show on dashboard'),
model_name="savedview",
name="show_on_dashboard",
field=models.BooleanField(verbose_name="show on dashboard"),
),
migrations.AlterField(
model_name='savedview',
name='sort_field',
field=models.CharField(max_length=128, verbose_name='sort field'),
model_name="savedview",
name="sort_field",
field=models.CharField(max_length=128, verbose_name="sort field"),
),
migrations.AlterField(
model_name='savedview',
name='sort_reverse',
field=models.BooleanField(default=False, verbose_name='sort reverse'),
model_name="savedview",
name="sort_reverse",
field=models.BooleanField(default=False, verbose_name="sort reverse"),
),
migrations.AlterField(
model_name='savedview',
name='user',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='user'),
model_name="savedview",
name="user",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
verbose_name="user",
),
),
migrations.AlterField(
model_name='savedviewfilterrule',
name='rule_type',
field=models.PositiveIntegerField(choices=[(0, 'title contains'), (1, 'content contains'), (2, 'ASN is'), (3, 'correspondent is'), (4, 'document type is'), (5, 'is in inbox'), (6, 'has tag'), (7, 'has any tag'), (8, 'created before'), (9, 'created after'), (10, 'created year is'), (11, 'created month is'), (12, 'created day is'), (13, 'added before'), (14, 'added after'), (15, 'modified before'), (16, 'modified after'), (17, 'does not have tag')], verbose_name='rule type'),
model_name="savedviewfilterrule",
name="rule_type",
field=models.PositiveIntegerField(
choices=[
(0, "title contains"),
(1, "content contains"),
(2, "ASN is"),
(3, "correspondent is"),
(4, "document type is"),
(5, "is in inbox"),
(6, "has tag"),
(7, "has any tag"),
(8, "created before"),
(9, "created after"),
(10, "created year is"),
(11, "created month is"),
(12, "created day is"),
(13, "added before"),
(14, "added after"),
(15, "modified before"),
(16, "modified after"),
(17, "does not have tag"),
],
verbose_name="rule type",
),
),
migrations.AlterField(
model_name='savedviewfilterrule',
name='saved_view',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='filter_rules', to='documents.savedview', verbose_name='saved view'),
model_name="savedviewfilterrule",
name="saved_view",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="filter_rules",
to="documents.savedview",
verbose_name="saved view",
),
),
migrations.AlterField(
model_name='savedviewfilterrule',
name='value',
field=models.CharField(blank=True, max_length=128, null=True, verbose_name='value'),
model_name="savedviewfilterrule",
name="value",
field=models.CharField(
blank=True, max_length=128, null=True, verbose_name="value"
),
),
migrations.AlterField(
model_name='tag',
name='colour',
field=models.PositiveIntegerField(choices=[(1, '#a6cee3'), (2, '#1f78b4'), (3, '#b2df8a'), (4, '#33a02c'), (5, '#fb9a99'), (6, '#e31a1c'), (7, '#fdbf6f'), (8, '#ff7f00'), (9, '#cab2d6'), (10, '#6a3d9a'), (11, '#b15928'), (12, '#000000'), (13, '#cccccc')], default=1, verbose_name='color'),
model_name="tag",
name="colour",
field=models.PositiveIntegerField(
choices=[
(1, "#a6cee3"),
(2, "#1f78b4"),
(3, "#b2df8a"),
(4, "#33a02c"),
(5, "#fb9a99"),
(6, "#e31a1c"),
(7, "#fdbf6f"),
(8, "#ff7f00"),
(9, "#cab2d6"),
(10, "#6a3d9a"),
(11, "#b15928"),
(12, "#000000"),
(13, "#cccccc"),
],
default=1,
verbose_name="color",
),
),
migrations.AlterField(
model_name='tag',
name='is_inbox_tag',
field=models.BooleanField(default=False, help_text='Marks this tag as an inbox tag: All newly consumed documents will be tagged with inbox tags.', verbose_name='is inbox tag'),
model_name="tag",
name="is_inbox_tag",
field=models.BooleanField(
default=False,
help_text="Marks this tag as an inbox tag: All newly consumed documents will be tagged with inbox tags.",
verbose_name="is inbox tag",
),
),
migrations.AlterField(
model_name='tag',
name='is_insensitive',
field=models.BooleanField(default=True, verbose_name='is insensitive'),
model_name="tag",
name="is_insensitive",
field=models.BooleanField(default=True, verbose_name="is insensitive"),
),
migrations.AlterField(
model_name='tag',
name='match',
field=models.CharField(blank=True, max_length=256, verbose_name='match'),
model_name="tag",
name="match",
field=models.CharField(blank=True, max_length=256, verbose_name="match"),
),
migrations.AlterField(
model_name='tag',
name='matching_algorithm',
field=models.PositiveIntegerField(choices=[(1, 'Any word'), (2, 'All words'), (3, 'Exact match'), (4, 'Regular expression'), (5, 'Fuzzy word'), (6, 'Automatic')], default=1, verbose_name='matching algorithm'),
model_name="tag",
name="matching_algorithm",
field=models.PositiveIntegerField(
choices=[
(1, "Any word"),
(2, "All words"),
(3, "Exact match"),
(4, "Regular expression"),
(5, "Fuzzy word"),
(6, "Automatic"),
],
default=1,
verbose_name="matching algorithm",
),
),
migrations.AlterField(
model_name='tag',
name='name',
field=models.CharField(max_length=128, unique=True, verbose_name='name'),
model_name="tag",
name="name",
field=models.CharField(max_length=128, unique=True, verbose_name="name"),
),
]

115
src/documents/migrations/1012_fix_archive_files.py
View File

@@ -20,6 +20,7 @@ logger = logging.getLogger("paperless.migrations")
# This is code copied straight paperless before the change.
###############################################################################
def archive_name_from_filename(filename):
return os.path.splitext(filename)[0] + ".pdf"
@@ -30,10 +31,7 @@ def archive_path_old(doc):
else:
fname = "{:07}.pdf".format(doc.pk)
return os.path.join(
settings.ARCHIVE_DIR,
fname
)
return os.path.join(settings.ARCHIVE_DIR, fname)
STORAGE_TYPE_GPG = "gpg"
@@ -41,10 +39,7 @@ STORAGE_TYPE_GPG = "gpg"
def archive_path_new(doc):
if doc.archive_filename is not None:
return os.path.join(
settings.ARCHIVE_DIR,
str(doc.archive_filename)
)
return os.path.join(settings.ARCHIVE_DIR, str(doc.archive_filename))
else:
return None
@@ -57,10 +52,7 @@ def source_path(doc):
if doc.storage_type == STORAGE_TYPE_GPG:
fname += ".gpg" # pragma: no cover
return os.path.join(
settings.ORIGINALS_DIR,
fname
)
return os.path.join(settings.ORIGINALS_DIR, fname)
def generate_unique_filename(doc, archive_filename=False):
@@ -75,7 +67,8 @@ def generate_unique_filename(doc, archive_filename=False):
while True:
new_filename = generate_filename(
doc, counter, archive_filename=archive_filename)
doc, counter, archive_filename=archive_filename
)
if new_filename == old_filename:
# still the same as before.
return new_filename
@@ -91,14 +84,11 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
try:
if settings.PAPERLESS_FILENAME_FORMAT is not None:
tags = defaultdictNoStr(lambda: slugify(None),
many_to_dictionary(doc.tags))
tags = defaultdictNoStr(lambda: slugify(None), many_to_dictionary(doc.tags))
tag_list = pathvalidate.sanitize_filename(
",".join(sorted(
[tag.name for tag in doc.tags.all()]
)),
replacement_text="-"
",".join(sorted([tag.name for tag in doc.tags.all()])),
replacement_text="-",
)
if doc.correspondent:
@@ -116,20 +106,21 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
document_type = "none"
path = settings.PAPERLESS_FILENAME_FORMAT.format(
title=pathvalidate.sanitize_filename(
doc.title, replacement_text="-"),
title=pathvalidate.sanitize_filename(doc.title, replacement_text="-"),
correspondent=correspondent,
document_type=document_type,
created=datetime.date.isoformat(doc.created),
created_year=doc.created.year if doc.created else "none",
created_month=f"{doc.created.month:02}" if doc.created else "none", # NOQA: E501
created_month=f"{doc.created.month:02}"
if doc.created
else "none", # NOQA: E501
created_day=f"{doc.created.day:02}" if doc.created else "none",
added=datetime.date.isoformat(doc.added),
added_year=doc.added.year if doc.added else "none",
added_month=f"{doc.added.month:02}" if doc.added else "none",
added_day=f"{doc.added.day:02}" if doc.added else "none",
tags=tags,
tag_list=tag_list
tag_list=tag_list,
).strip()
path = path.strip(os.sep)
@@ -137,7 +128,8 @@ def generate_filename(doc, counter=0, append_gpg=True, archive_filename=False):
except (ValueError, KeyError, IndexError):
logger.warning(
f"Invalid PAPERLESS_FILENAME_FORMAT: "
f"{settings.PAPERLESS_FILENAME_FORMAT}, falling back to default")
f"{settings.PAPERLESS_FILENAME_FORMAT}, falling back to default"
)
counter_str = f"_{counter:02}" if counter else ""
@@ -166,29 +158,29 @@ def parse_wrapper(parser, path, mime_type, file_name):
def create_archive_version(doc, retry_count=3):
from documents.parsers import get_parser_class_for_mime_type, \
DocumentParser, \
ParseError
logger.info(
f"Regenerating archive document for document ID:{doc.id}"
from documents.parsers import (
get_parser_class_for_mime_type,
DocumentParser,
ParseError,
)
logger.info(f"Regenerating archive document for document ID:{doc.id}")
parser_class = get_parser_class_for_mime_type(doc.mime_type)
for try_num in range(retry_count):
parser: DocumentParser = parser_class(None, None)
try:
parse_wrapper(parser, source_path(doc), doc.mime_type,
os.path.basename(doc.filename))
parse_wrapper(
parser, source_path(doc), doc.mime_type, os.path.basename(doc.filename)
)
doc.content = parser.get_text()
if parser.get_archive_path() and os.path.isfile(
parser.get_archive_path()):
if parser.get_archive_path() and os.path.isfile(parser.get_archive_path()):
doc.archive_filename = generate_unique_filename(
doc, archive_filename=True)
doc, archive_filename=True
)
with open(parser.get_archive_path(), "rb") as f:
doc.archive_checksum = hashlib.md5(f.read()).hexdigest()
os.makedirs(os.path.dirname(archive_path_new(doc)),
exist_ok=True)
os.makedirs(os.path.dirname(archive_path_new(doc)), exist_ok=True)
shutil.copy2(parser.get_archive_path(), archive_path_new(doc))
else:
doc.archive_checksum = None
@@ -241,8 +233,8 @@ def move_old_to_new_locations(apps, schema_editor):
old_path = archive_path_old(doc)
if doc.id not in affected_document_ids and not os.path.isfile(old_path):
raise ValueError(
f"Archived document ID:{doc.id} does not exist at: "
f"{old_path}")
f"Archived document ID:{doc.id} does not exist at: " f"{old_path}"
)
# check that we can regenerate affected archive versions
for doc_id in affected_document_ids:
@@ -253,7 +245,8 @@ def move_old_to_new_locations(apps, schema_editor):
if not parser_class:
raise ValueError(
f"Document ID:{doc.id} has an invalid archived document, "
f"but no parsers are available. Cannot migrate.")
f"but no parsers are available. Cannot migrate."
)
for doc in Document.objects.filter(archive_checksum__isnull=False):
@@ -261,9 +254,7 @@ def move_old_to_new_locations(apps, schema_editor):
old_path = archive_path_old(doc)
# remove affected archive versions
if os.path.isfile(old_path):
logger.debug(
f"Removing {old_path}"
)
logger.debug(f"Removing {old_path}")
os.unlink(old_path)
else:
# Set archive path for unaffected files
@@ -290,7 +281,8 @@ def move_new_to_old_locations(apps, schema_editor):
raise ValueError(
f"Cannot migrate: Archive file name {old_archive_path} of "
f"document {doc.filename} would clash with another archive "
f"filename.")
f"filename."
)
old_archive_paths.add(old_archive_path)
if new_archive_path != old_archive_path and os.path.isfile(old_archive_path):
raise ValueError(
@@ -309,22 +301,35 @@ def move_new_to_old_locations(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '1011_auto_20210101_2340'),
("documents", "1011_auto_20210101_2340"),
]
operations = [
migrations.AddField(
model_name='document',
name='archive_filename',
field=models.FilePathField(default=None, editable=False, help_text='Current archive filename in storage', max_length=1024, null=True, unique=True, verbose_name='archive filename'),
model_name="document",
name="archive_filename",
field=models.FilePathField(
default=None,
editable=False,
help_text="Current archive filename in storage",
max_length=1024,
null=True,
unique=True,
verbose_name="archive filename",
),
),
migrations.AlterField(
model_name='document',
name='filename',
field=models.FilePathField(default=None, editable=False, help_text='Current filename in storage', max_length=1024, null=True, unique=True, verbose_name='filename'),
),
migrations.RunPython(
move_old_to_new_locations,
move_new_to_old_locations
model_name="document",
name="filename",
field=models.FilePathField(
default=None,
editable=False,
help_text="Current filename in storage",
max_length=1024,
null=True,
unique=True,
verbose_name="filename",
),
),
migrations.RunPython(move_old_to_new_locations, move_new_to_old_locations),
]

26
src/documents/migrations/1013_migrate_tag_colour.py
View File

@@ -20,7 +20,7 @@ COLOURS_OLD = {
def forward(apps, schema_editor):
Tag = apps.get_model('documents', 'Tag')
Tag = apps.get_model("documents", "Tag")
for tag in Tag.objects.all():
colour_old_id = tag.colour_old
@@ -30,7 +30,7 @@ def forward(apps, schema_editor):
def reverse(apps, schema_editor):
Tag = apps.get_model('documents', 'Tag')
Tag = apps.get_model("documents", "Tag")
def _get_colour_id(rdb):
for idx, rdbx in COLOURS_OLD.items():
@@ -48,23 +48,25 @@ def reverse(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('documents', '1012_fix_archive_files'),
("documents", "1012_fix_archive_files"),
]
operations = [
migrations.RenameField(
model_name='tag',
old_name='colour',
new_name='colour_old',
model_name="tag",
old_name="colour",
new_name="colour_old",
),
migrations.AddField(
model_name='tag',
name='color',
field=models.CharField(default='#a6cee3', max_length=7, verbose_name='color'),
model_name="tag",
name="color",
field=models.CharField(
default="#a6cee3", max_length=7, verbose_name="color"
),
),
migrations.RunPython(forward, reverse),
migrations.RemoveField(
model_name='tag',
name='colour_old',
)
model_name="tag",
name="colour_old",
),
]

32
src/documents/migrations/1014_auto_20210228_1614.py
View File

@@ -6,13 +6,37 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '1013_migrate_tag_colour'),
("documents", "1013_migrate_tag_colour"),
]
operations = [
migrations.AlterField(
model_name='savedviewfilterrule',
name='rule_type',
field=models.PositiveIntegerField(choices=[(0, 'title contains'), (1, 'content contains'), (2, 'ASN is'), (3, 'correspondent is'), (4, 'document type is'), (5, 'is in inbox'), (6, 'has tag'), (7, 'has any tag'), (8, 'created before'), (9, 'created after'), (10, 'created year is'), (11, 'created month is'), (12, 'created day is'), (13, 'added before'), (14, 'added after'), (15, 'modified before'), (16, 'modified after'), (17, 'does not have tag'), (18, 'does not have ASN'), (19, 'title or content contains')], verbose_name='rule type'),
model_name="savedviewfilterrule",
name="rule_type",
field=models.PositiveIntegerField(
choices=[
(0, "title contains"),
(1, "content contains"),
(2, "ASN is"),
(3, "correspondent is"),
(4, "document type is"),
(5, "is in inbox"),
(6, "has tag"),
(7, "has any tag"),
(8, "created before"),
(9, "created after"),
(10, "created year is"),
(11, "created month is"),
(12, "created day is"),
(13, "added before"),
(14, "added after"),
(15, "modified before"),
(16, "modified after"),
(17, "does not have tag"),
(18, "does not have ASN"),
(19, "title or content contains"),
],
verbose_name="rule type",
),
),
]

8
src/documents/migrations/1015_remove_null_characters.py
View File

@@ -8,20 +8,20 @@ logger = logging.getLogger("paperless.migrations")
def remove_null_characters(apps, schema_editor):
Document = apps.get_model('documents', 'Document')
Document = apps.get_model("documents", "Document")
for doc in Document.objects.all():
content: str = doc.content
if '\0' in content:
if "\0" in content:
logger.info(f"Removing null characters from document {doc}...")
doc.content = content.replace('\0', ' ')
doc.content = content.replace("\0", " ")
doc.save()
class Migration(migrations.Migration):
dependencies = [
('documents', '1014_auto_20210228_1614'),
("documents", "1014_auto_20210228_1614"),
]
operations = [

42
src/documents/migrations/1016_auto_20210317_1351.py
View File

@@ -6,18 +6,46 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('documents', '1015_remove_null_characters'),
("documents", "1015_remove_null_characters"),
]
operations = [
migrations.AlterField(
model_name='savedview',
name='sort_field',
field=models.CharField(blank=True, max_length=128, null=True, verbose_name='sort field'),
model_name="savedview",
name="sort_field",
field=models.CharField(
blank=True, max_length=128, null=True, verbose_name="sort field"
),
),
migrations.AlterField(
model_name='savedviewfilterrule',
name='rule_type',
field=models.PositiveIntegerField(choices=[(0, 'title contains'), (1, 'content contains'), (2, 'ASN is'), (3, 'correspondent is'), (4, 'document type is'), (5, 'is in inbox'), (6, 'has tag'), (7, 'has any tag'), (8, 'created before'), (9, 'created after'), (10, 'created year is'), (11, 'created month is'), (12, 'created day is'), (13, 'added before'), (14, 'added after'), (15, 'modified before'), (16, 'modified after'), (17, 'does not have tag'), (18, 'does not have ASN'), (19, 'title or content contains'), (20, 'fulltext query'), (21, 'more like this')], verbose_name='rule type'),
model_name="savedviewfilterrule",
name="rule_type",
field=models.PositiveIntegerField(
choices=[
(0, "title contains"),
(1, "content contains"),
(2, "ASN is"),
(3, "correspondent is"),
(4, "document type is"),
(5, "is in inbox"),
(6, "has tag"),
(7, "has any tag"),
(8, "created before"),
(9, "created after"),
(10, "created year is"),
(11, "created month is"),
(12, "created day is"),
(13, "added before"),
(14, "added after"),
(15, "modified before"),
(16, "modified after"),
(17, "does not have tag"),
(18, "does not have ASN"),
(19, "title or content contains"),
(20, "fulltext query"),
(21, "more like this"),
],
verbose_name="rule type",
),
),
]