mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-01-24 22:39:02 -06:00
Compare commits
6 Commits
chore/pyte
...
feature-be
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fe46cac55 | ||
|
|
c0c2202564 | ||
|
|
d65d9a2b88 | ||
|
|
8e12f3e93c | ||
|
|
cf89d81b9e | ||
|
|
d0032c18be |
@@ -4,8 +4,7 @@
|
|||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
for command in decrypt_documents \
|
for command in document_archiver \
|
||||||
document_archiver \
|
|
||||||
document_exporter \
|
document_exporter \
|
||||||
document_importer \
|
document_importer \
|
||||||
mail_fetcher \
|
mail_fetcher \
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
#!/command/with-contenv /usr/bin/bash
|
|
||||||
# shellcheck shell=bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
cd "${PAPERLESS_SRC_DIR}"
|
|
||||||
|
|
||||||
if [[ $(id -u) == 0 ]]; then
|
|
||||||
s6-setuidgid paperless python3 manage.py decrypt_documents "$@"
|
|
||||||
elif [[ $(id -un) == "paperless" ]]; then
|
|
||||||
python3 manage.py decrypt_documents "$@"
|
|
||||||
else
|
|
||||||
echo "Unknown user."
|
|
||||||
fi
|
|
||||||
@@ -580,36 +580,6 @@ document.
|
|||||||
documents, such as encrypted PDF documents. The archiver will skip over
|
documents, such as encrypted PDF documents. The archiver will skip over
|
||||||
these documents each time it sees them.
|
these documents each time it sees them.
|
||||||
|
|
||||||
### Managing encryption {#encryption}
|
|
||||||
|
|
||||||
!!! warning
|
|
||||||
|
|
||||||
Encryption was removed in [paperless-ng 0.9](changelog.md#paperless-ng-090)
|
|
||||||
because it did not really provide any additional security, the passphrase
|
|
||||||
was stored in a configuration file on the same system as the documents.
|
|
||||||
Furthermore, the entire text content of the documents is stored plain in
|
|
||||||
the database, even if your documents are encrypted. Filenames are not
|
|
||||||
encrypted as well. Finally, the web server provides transparent access to
|
|
||||||
your encrypted documents.
|
|
||||||
|
|
||||||
Consider running paperless on an encrypted filesystem instead, which
|
|
||||||
will then at least provide security against physical hardware theft.
|
|
||||||
|
|
||||||
#### Enabling encryption
|
|
||||||
|
|
||||||
Enabling encryption is no longer supported.
|
|
||||||
|
|
||||||
#### Disabling encryption
|
|
||||||
|
|
||||||
Basic usage to disable encryption of your document store:
|
|
||||||
|
|
||||||
(Note: If `PAPERLESS_PASSPHRASE` isn't set already, you need to specify
|
|
||||||
it here)
|
|
||||||
|
|
||||||
```
|
|
||||||
decrypt_documents [--passphrase SECR3TP4SSPHRA$E]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Detecting duplicates {#fuzzy_duplicate}
|
### Detecting duplicates {#fuzzy_duplicate}
|
||||||
|
|
||||||
Paperless already catches and prevents upload of exactly matching documents,
|
Paperless already catches and prevents upload of exactly matching documents,
|
||||||
|
|||||||
@@ -17,3 +17,9 @@ separating the directory ignore from the file ignore.
|
|||||||
| `CONSUMER_POLLING_RETRY_COUNT` | _Removed_ | Automatic with stability tracking |
|
| `CONSUMER_POLLING_RETRY_COUNT` | _Removed_ | Automatic with stability tracking |
|
||||||
| `CONSUMER_IGNORE_PATTERNS` | [`CONSUMER_IGNORE_PATTERNS`](configuration.md#PAPERLESS_CONSUMER_IGNORE_PATTERNS) | **Now regex, not fnmatch**; user patterns are added to (not replacing) default ones |
|
| `CONSUMER_IGNORE_PATTERNS` | [`CONSUMER_IGNORE_PATTERNS`](configuration.md#PAPERLESS_CONSUMER_IGNORE_PATTERNS) | **Now regex, not fnmatch**; user patterns are added to (not replacing) default ones |
|
||||||
| _New_ | [`CONSUMER_IGNORE_DIRS`](configuration.md#PAPERLESS_CONSUMER_IGNORE_DIRS) | Additional directories to ignore; user entries are added to (not replacing) defaults |
|
| _New_ | [`CONSUMER_IGNORE_DIRS`](configuration.md#PAPERLESS_CONSUMER_IGNORE_DIRS) | Additional directories to ignore; user entries are added to (not replacing) defaults |
|
||||||
|
|
||||||
|
## Encryption Support
|
||||||
|
|
||||||
|
Document and thumbnail encryption is no longer supported. This was previously deprecated in [paperless-ng 0.9.3](https://github.com/paperless-ngx/paperless-ngx/blob/dev/docs/changelog.md#paperless-ng-093)
|
||||||
|
|
||||||
|
Users must decrypt their document using the `decrypt_documents` command before upgrading.
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
# this is here so that django finds the checks.
|
# this is here so that django finds the checks.
|
||||||
from documents.checks import changed_password_check
|
|
||||||
from documents.checks import parser_check
|
from documents.checks import parser_check
|
||||||
|
|
||||||
__all__ = ["changed_password_check", "parser_check"]
|
__all__ = ["parser_check"]
|
||||||
|
|||||||
@@ -60,7 +60,6 @@ class DocumentAdmin(GuardedModelAdmin):
|
|||||||
"added",
|
"added",
|
||||||
"modified",
|
"modified",
|
||||||
"mime_type",
|
"mime_type",
|
||||||
"storage_type",
|
|
||||||
"filename",
|
"filename",
|
||||||
"checksum",
|
"checksum",
|
||||||
"archive_filename",
|
"archive_filename",
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from pikepdf import Pdf
|
|||||||
from documents.converters import convert_from_tiff_to_pdf
|
from documents.converters import convert_from_tiff_to_pdf
|
||||||
from documents.data_models import ConsumableDocument
|
from documents.data_models import ConsumableDocument
|
||||||
from documents.data_models import DocumentMetadataOverrides
|
from documents.data_models import DocumentMetadataOverrides
|
||||||
|
from documents.models import Document
|
||||||
from documents.models import Tag
|
from documents.models import Tag
|
||||||
from documents.plugins.base import ConsumeTaskPlugin
|
from documents.plugins.base import ConsumeTaskPlugin
|
||||||
from documents.plugins.base import StopConsumeTaskError
|
from documents.plugins.base import StopConsumeTaskError
|
||||||
@@ -115,6 +116,24 @@ class BarcodePlugin(ConsumeTaskPlugin):
|
|||||||
self._tiff_conversion_done = False
|
self._tiff_conversion_done = False
|
||||||
self.barcodes: list[Barcode] = []
|
self.barcodes: list[Barcode] = []
|
||||||
|
|
||||||
|
def _apply_detected_asn(self, detected_asn: int) -> None:
|
||||||
|
"""
|
||||||
|
Apply a detected ASN to metadata if allowed.
|
||||||
|
"""
|
||||||
|
if (
|
||||||
|
self.metadata.skip_asn_if_exists
|
||||||
|
and Document.global_objects.filter(
|
||||||
|
archive_serial_number=detected_asn,
|
||||||
|
).exists()
|
||||||
|
):
|
||||||
|
logger.info(
|
||||||
|
f"Found ASN in barcode {detected_asn} but skipping because it already exists.",
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info(f"Found ASN in barcode: {detected_asn}")
|
||||||
|
self.metadata.asn = detected_asn
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
# Some operations may use PIL, override pixel setting if needed
|
# Some operations may use PIL, override pixel setting if needed
|
||||||
maybe_override_pixel_limit()
|
maybe_override_pixel_limit()
|
||||||
@@ -186,13 +205,8 @@ class BarcodePlugin(ConsumeTaskPlugin):
|
|||||||
|
|
||||||
# Update/overwrite an ASN if possible
|
# Update/overwrite an ASN if possible
|
||||||
# After splitting, as otherwise each split document gets the same ASN
|
# After splitting, as otherwise each split document gets the same ASN
|
||||||
if (
|
if self.settings.barcode_enable_asn and (located_asn := self.asn) is not None:
|
||||||
self.settings.barcode_enable_asn
|
self._apply_detected_asn(located_asn)
|
||||||
and not self.metadata.skip_asn
|
|
||||||
and (located_asn := self.asn) is not None
|
|
||||||
):
|
|
||||||
logger.info(f"Found ASN in barcode: {located_asn}")
|
|
||||||
self.metadata.asn = located_asn
|
|
||||||
|
|
||||||
def cleanup(self) -> None:
|
def cleanup(self) -> None:
|
||||||
self.temp_dir.cleanup()
|
self.temp_dir.cleanup()
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ from pathlib import Path
|
|||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
from celery import chain
|
|
||||||
from celery import chord
|
from celery import chord
|
||||||
from celery import group
|
from celery import group
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
@@ -38,6 +37,42 @@ if TYPE_CHECKING:
|
|||||||
logger: logging.Logger = logging.getLogger("paperless.bulk_edit")
|
logger: logging.Logger = logging.getLogger("paperless.bulk_edit")
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(bind=True)
|
||||||
|
def restore_archive_serial_numbers_task(
|
||||||
|
self,
|
||||||
|
backup: dict[int, int],
|
||||||
|
*args,
|
||||||
|
**kwargs,
|
||||||
|
) -> None:
|
||||||
|
restore_archive_serial_numbers(backup)
|
||||||
|
|
||||||
|
|
||||||
|
def release_archive_serial_numbers(doc_ids: list[int]) -> dict[int, int]:
|
||||||
|
"""
|
||||||
|
Clears ASNs on documents that are about to be replaced so new documents
|
||||||
|
can be assigned ASNs without uniqueness collisions. Returns a backup map
|
||||||
|
of doc_id -> previous ASN for potential restoration.
|
||||||
|
"""
|
||||||
|
qs = Document.objects.filter(
|
||||||
|
id__in=doc_ids,
|
||||||
|
archive_serial_number__isnull=False,
|
||||||
|
).only("pk", "archive_serial_number")
|
||||||
|
backup = dict(qs.values_list("pk", "archive_serial_number"))
|
||||||
|
qs.update(archive_serial_number=None)
|
||||||
|
logger.info(f"Released archive serial numbers for documents {list(backup.keys())}")
|
||||||
|
return backup
|
||||||
|
|
||||||
|
|
||||||
|
def restore_archive_serial_numbers(backup: dict[int, int]) -> None:
|
||||||
|
"""
|
||||||
|
Restores ASNs using the provided backup map, intended for
|
||||||
|
rollback when replacement consumption fails.
|
||||||
|
"""
|
||||||
|
for doc_id, asn in backup.items():
|
||||||
|
Document.objects.filter(pk=doc_id).update(archive_serial_number=asn)
|
||||||
|
logger.info(f"Restored archive serial numbers for documents {list(backup.keys())}")
|
||||||
|
|
||||||
|
|
||||||
def set_correspondent(
|
def set_correspondent(
|
||||||
doc_ids: list[int],
|
doc_ids: list[int],
|
||||||
correspondent: Correspondent,
|
correspondent: Correspondent,
|
||||||
@@ -386,6 +421,7 @@ def merge(
|
|||||||
|
|
||||||
merged_pdf = pikepdf.new()
|
merged_pdf = pikepdf.new()
|
||||||
version: str = merged_pdf.pdf_version
|
version: str = merged_pdf.pdf_version
|
||||||
|
handoff_asn: int | None = None
|
||||||
# use doc_ids to preserve order
|
# use doc_ids to preserve order
|
||||||
for doc_id in doc_ids:
|
for doc_id in doc_ids:
|
||||||
doc = qs.get(id=doc_id)
|
doc = qs.get(id=doc_id)
|
||||||
@@ -401,6 +437,8 @@ def merge(
|
|||||||
version = max(version, pdf.pdf_version)
|
version = max(version, pdf.pdf_version)
|
||||||
merged_pdf.pages.extend(pdf.pages)
|
merged_pdf.pages.extend(pdf.pages)
|
||||||
affected_docs.append(doc.id)
|
affected_docs.append(doc.id)
|
||||||
|
if handoff_asn is None and doc.archive_serial_number is not None:
|
||||||
|
handoff_asn = doc.archive_serial_number
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
f"Error merging document {doc.id}, it will not be included in the merge: {e}",
|
f"Error merging document {doc.id}, it will not be included in the merge: {e}",
|
||||||
@@ -426,6 +464,8 @@ def merge(
|
|||||||
DocumentMetadataOverrides.from_document(metadata_document)
|
DocumentMetadataOverrides.from_document(metadata_document)
|
||||||
)
|
)
|
||||||
overrides.title = metadata_document.title + " (merged)"
|
overrides.title = metadata_document.title + " (merged)"
|
||||||
|
if metadata_document.archive_serial_number is not None:
|
||||||
|
handoff_asn = metadata_document.archive_serial_number
|
||||||
else:
|
else:
|
||||||
overrides = DocumentMetadataOverrides()
|
overrides = DocumentMetadataOverrides()
|
||||||
else:
|
else:
|
||||||
@@ -433,8 +473,11 @@ def merge(
|
|||||||
|
|
||||||
if user is not None:
|
if user is not None:
|
||||||
overrides.owner_id = user.id
|
overrides.owner_id = user.id
|
||||||
# Avoid copying or detecting ASN from merged PDFs to prevent collision
|
if not delete_originals:
|
||||||
overrides.skip_asn = True
|
overrides.skip_asn_if_exists = True
|
||||||
|
|
||||||
|
if delete_originals and handoff_asn is not None:
|
||||||
|
overrides.asn = handoff_asn
|
||||||
|
|
||||||
logger.info("Adding merged document to the task queue.")
|
logger.info("Adding merged document to the task queue.")
|
||||||
|
|
||||||
@@ -447,12 +490,20 @@ def merge(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if delete_originals:
|
if delete_originals:
|
||||||
|
backup = release_archive_serial_numbers(affected_docs)
|
||||||
logger.info(
|
logger.info(
|
||||||
"Queueing removal of original documents after consumption of merged document",
|
"Queueing removal of original documents after consumption of merged document",
|
||||||
)
|
)
|
||||||
chain(consume_task, delete.si(affected_docs)).delay()
|
try:
|
||||||
else:
|
consume_task.apply_async(
|
||||||
consume_task.delay()
|
link=[delete.si(affected_docs)],
|
||||||
|
link_error=[restore_archive_serial_numbers_task.s(backup)],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
restore_archive_serial_numbers(backup)
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
consume_task.delay()
|
||||||
|
|
||||||
return "OK"
|
return "OK"
|
||||||
|
|
||||||
@@ -494,6 +545,8 @@ def split(
|
|||||||
overrides.title = f"{doc.title} (split {idx + 1})"
|
overrides.title = f"{doc.title} (split {idx + 1})"
|
||||||
if user is not None:
|
if user is not None:
|
||||||
overrides.owner_id = user.id
|
overrides.owner_id = user.id
|
||||||
|
if not delete_originals:
|
||||||
|
overrides.skip_asn_if_exists = True
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Adding split document with pages {split_doc} to the task queue.",
|
f"Adding split document with pages {split_doc} to the task queue.",
|
||||||
)
|
)
|
||||||
@@ -508,10 +561,20 @@ def split(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if delete_originals:
|
if delete_originals:
|
||||||
|
backup = release_archive_serial_numbers([doc.id])
|
||||||
logger.info(
|
logger.info(
|
||||||
"Queueing removal of original document after consumption of the split documents",
|
"Queueing removal of original document after consumption of the split documents",
|
||||||
)
|
)
|
||||||
chord(header=consume_tasks, body=delete.si([doc.id])).delay()
|
try:
|
||||||
|
chord(
|
||||||
|
header=consume_tasks,
|
||||||
|
body=delete.si([doc.id]),
|
||||||
|
).apply_async(
|
||||||
|
link_error=[restore_archive_serial_numbers_task.s(backup)],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
restore_archive_serial_numbers(backup)
|
||||||
|
raise
|
||||||
else:
|
else:
|
||||||
group(consume_tasks).delay()
|
group(consume_tasks).delay()
|
||||||
|
|
||||||
@@ -614,7 +677,10 @@ def edit_pdf(
|
|||||||
)
|
)
|
||||||
if user is not None:
|
if user is not None:
|
||||||
overrides.owner_id = user.id
|
overrides.owner_id = user.id
|
||||||
|
if not delete_original:
|
||||||
|
overrides.skip_asn_if_exists = True
|
||||||
|
if delete_original and len(pdf_docs) == 1:
|
||||||
|
overrides.asn = doc.archive_serial_number
|
||||||
for idx, pdf in enumerate(pdf_docs, start=1):
|
for idx, pdf in enumerate(pdf_docs, start=1):
|
||||||
filepath: Path = (
|
filepath: Path = (
|
||||||
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
|
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
|
||||||
@@ -633,7 +699,17 @@ def edit_pdf(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if delete_original:
|
if delete_original:
|
||||||
chord(header=consume_tasks, body=delete.si([doc.id])).delay()
|
backup = release_archive_serial_numbers([doc.id])
|
||||||
|
try:
|
||||||
|
chord(
|
||||||
|
header=consume_tasks,
|
||||||
|
body=delete.si([doc.id]),
|
||||||
|
).apply_async(
|
||||||
|
link_error=[restore_archive_serial_numbers_task.s(backup)],
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
restore_archive_serial_numbers(backup)
|
||||||
|
raise
|
||||||
else:
|
else:
|
||||||
group(consume_tasks).delay()
|
group(consume_tasks).delay()
|
||||||
|
|
||||||
|
|||||||
@@ -1,60 +1,12 @@
|
|||||||
import textwrap
|
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core.checks import Error
|
from django.core.checks import Error
|
||||||
from django.core.checks import Warning
|
from django.core.checks import Warning
|
||||||
from django.core.checks import register
|
from django.core.checks import register
|
||||||
from django.core.exceptions import FieldError
|
|
||||||
from django.db.utils import OperationalError
|
|
||||||
from django.db.utils import ProgrammingError
|
|
||||||
|
|
||||||
from documents.signals import document_consumer_declaration
|
from documents.signals import document_consumer_declaration
|
||||||
from documents.templating.utils import convert_format_str_to_template_format
|
from documents.templating.utils import convert_format_str_to_template_format
|
||||||
|
|
||||||
|
|
||||||
@register()
|
|
||||||
def changed_password_check(app_configs, **kwargs):
|
|
||||||
from documents.models import Document
|
|
||||||
from paperless.db import GnuPG
|
|
||||||
|
|
||||||
try:
|
|
||||||
encrypted_doc = (
|
|
||||||
Document.objects.filter(
|
|
||||||
storage_type=Document.STORAGE_TYPE_GPG,
|
|
||||||
)
|
|
||||||
.only("pk", "storage_type")
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
except (OperationalError, ProgrammingError, FieldError):
|
|
||||||
return [] # No documents table yet
|
|
||||||
|
|
||||||
if encrypted_doc:
|
|
||||||
if not settings.PASSPHRASE:
|
|
||||||
return [
|
|
||||||
Error(
|
|
||||||
"The database contains encrypted documents but no password is set.",
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
if not GnuPG.decrypted(encrypted_doc.source_file):
|
|
||||||
return [
|
|
||||||
Error(
|
|
||||||
textwrap.dedent(
|
|
||||||
"""
|
|
||||||
The current password doesn't match the password of the
|
|
||||||
existing documents.
|
|
||||||
|
|
||||||
If you intend to change your password, you must first export
|
|
||||||
all of the old documents, start fresh with the new password
|
|
||||||
and then re-import them."
|
|
||||||
""",
|
|
||||||
),
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
@register()
|
@register()
|
||||||
def parser_check(app_configs, **kwargs):
|
def parser_check(app_configs, **kwargs):
|
||||||
parsers = []
|
parsers = []
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ def thumbnail_last_modified(request, pk: int) -> datetime | None:
|
|||||||
Cache should be (slightly?) faster than filesystem
|
Cache should be (slightly?) faster than filesystem
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
doc = Document.objects.only("storage_type").get(pk=pk)
|
doc = Document.objects.only("pk").get(pk=pk)
|
||||||
if not doc.thumbnail_path.exists():
|
if not doc.thumbnail_path.exists():
|
||||||
return None
|
return None
|
||||||
doc_key = get_thumbnail_modified_key(pk)
|
doc_key = get_thumbnail_modified_key(pk)
|
||||||
|
|||||||
@@ -497,7 +497,6 @@ class ConsumerPlugin(
|
|||||||
create_source_path_directory(document.source_path)
|
create_source_path_directory(document.source_path)
|
||||||
|
|
||||||
self._write(
|
self._write(
|
||||||
document.storage_type,
|
|
||||||
self.unmodified_original
|
self.unmodified_original
|
||||||
if self.unmodified_original is not None
|
if self.unmodified_original is not None
|
||||||
else self.working_copy,
|
else self.working_copy,
|
||||||
@@ -505,7 +504,6 @@ class ConsumerPlugin(
|
|||||||
)
|
)
|
||||||
|
|
||||||
self._write(
|
self._write(
|
||||||
document.storage_type,
|
|
||||||
thumbnail,
|
thumbnail,
|
||||||
document.thumbnail_path,
|
document.thumbnail_path,
|
||||||
)
|
)
|
||||||
@@ -517,7 +515,6 @@ class ConsumerPlugin(
|
|||||||
)
|
)
|
||||||
create_source_path_directory(document.archive_path)
|
create_source_path_directory(document.archive_path)
|
||||||
self._write(
|
self._write(
|
||||||
document.storage_type,
|
|
||||||
archive_path,
|
archive_path,
|
||||||
document.archive_path,
|
document.archive_path,
|
||||||
)
|
)
|
||||||
@@ -637,8 +634,6 @@ class ConsumerPlugin(
|
|||||||
)
|
)
|
||||||
self.log.debug(f"Creation date from st_mtime: {create_date}")
|
self.log.debug(f"Creation date from st_mtime: {create_date}")
|
||||||
|
|
||||||
storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
if self.metadata.filename:
|
if self.metadata.filename:
|
||||||
title = Path(self.metadata.filename).stem
|
title = Path(self.metadata.filename).stem
|
||||||
else:
|
else:
|
||||||
@@ -665,7 +660,6 @@ class ConsumerPlugin(
|
|||||||
checksum=hashlib.md5(file_for_checksum.read_bytes()).hexdigest(),
|
checksum=hashlib.md5(file_for_checksum.read_bytes()).hexdigest(),
|
||||||
created=create_date,
|
created=create_date,
|
||||||
modified=create_date,
|
modified=create_date,
|
||||||
storage_type=storage_type,
|
|
||||||
page_count=page_count,
|
page_count=page_count,
|
||||||
original_filename=self.filename,
|
original_filename=self.filename,
|
||||||
)
|
)
|
||||||
@@ -696,7 +690,7 @@ class ConsumerPlugin(
|
|||||||
pk=self.metadata.storage_path_id,
|
pk=self.metadata.storage_path_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.metadata.asn is not None and not self.metadata.skip_asn:
|
if self.metadata.asn is not None:
|
||||||
document.archive_serial_number = self.metadata.asn
|
document.archive_serial_number = self.metadata.asn
|
||||||
|
|
||||||
if self.metadata.owner_id:
|
if self.metadata.owner_id:
|
||||||
@@ -736,7 +730,7 @@ class ConsumerPlugin(
|
|||||||
}
|
}
|
||||||
CustomFieldInstance.objects.create(**args) # adds to document
|
CustomFieldInstance.objects.create(**args) # adds to document
|
||||||
|
|
||||||
def _write(self, storage_type, source, target):
|
def _write(self, source, target):
|
||||||
with (
|
with (
|
||||||
Path(source).open("rb") as read_file,
|
Path(source).open("rb") as read_file,
|
||||||
Path(target).open("wb") as write_file,
|
Path(target).open("wb") as write_file,
|
||||||
@@ -812,8 +806,8 @@ class ConsumerPreflightPlugin(
|
|||||||
"""
|
"""
|
||||||
Check that if override_asn is given, it is unique and within a valid range
|
Check that if override_asn is given, it is unique and within a valid range
|
||||||
"""
|
"""
|
||||||
if self.metadata.skip_asn or self.metadata.asn is None:
|
if self.metadata.asn is None:
|
||||||
# if skip is set or ASN is None
|
# if ASN is None
|
||||||
return
|
return
|
||||||
# Validate the range is above zero and less than uint32_t max
|
# Validate the range is above zero and less than uint32_t max
|
||||||
# otherwise, Whoosh can't handle it in the index
|
# otherwise, Whoosh can't handle it in the index
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ class DocumentMetadataOverrides:
|
|||||||
change_users: list[int] | None = None
|
change_users: list[int] | None = None
|
||||||
change_groups: list[int] | None = None
|
change_groups: list[int] | None = None
|
||||||
custom_fields: dict | None = None
|
custom_fields: dict | None = None
|
||||||
skip_asn: bool = False
|
skip_asn_if_exists: bool = False
|
||||||
|
|
||||||
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
|
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
|
||||||
"""
|
"""
|
||||||
@@ -50,8 +50,8 @@ class DocumentMetadataOverrides:
|
|||||||
self.storage_path_id = other.storage_path_id
|
self.storage_path_id = other.storage_path_id
|
||||||
if other.owner_id is not None:
|
if other.owner_id is not None:
|
||||||
self.owner_id = other.owner_id
|
self.owner_id = other.owner_id
|
||||||
if other.skip_asn:
|
if other.skip_asn_if_exists:
|
||||||
self.skip_asn = True
|
self.skip_asn_if_exists = True
|
||||||
|
|
||||||
# merge
|
# merge
|
||||||
if self.tag_ids is None:
|
if self.tag_ids is None:
|
||||||
|
|||||||
@@ -126,7 +126,6 @@ def generate_filename(
|
|||||||
doc: Document,
|
doc: Document,
|
||||||
*,
|
*,
|
||||||
counter=0,
|
counter=0,
|
||||||
append_gpg=True,
|
|
||||||
archive_filename=False,
|
archive_filename=False,
|
||||||
) -> Path:
|
) -> Path:
|
||||||
base_path: Path | None = None
|
base_path: Path | None = None
|
||||||
@@ -170,8 +169,4 @@ def generate_filename(
|
|||||||
final_filename = f"{doc.pk:07}{counter_str}{filetype_str}"
|
final_filename = f"{doc.pk:07}{counter_str}{filetype_str}"
|
||||||
full_path = Path(final_filename)
|
full_path = Path(final_filename)
|
||||||
|
|
||||||
# Add GPG extension if needed
|
|
||||||
if append_gpg and doc.storage_type == doc.STORAGE_TYPE_GPG:
|
|
||||||
full_path = full_path.with_suffix(full_path.suffix + ".gpg")
|
|
||||||
|
|
||||||
return full_path
|
return full_path
|
||||||
|
|||||||
@@ -1,93 +0,0 @@
|
|||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
from django.core.management.base import BaseCommand
|
|
||||||
from django.core.management.base import CommandError
|
|
||||||
|
|
||||||
from documents.models import Document
|
|
||||||
from paperless.db import GnuPG
|
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
|
||||||
help = (
|
|
||||||
"This is how you migrate your stored documents from an encrypted "
|
|
||||||
"state to an unencrypted one (or vice-versa)"
|
|
||||||
)
|
|
||||||
|
|
||||||
def add_arguments(self, parser) -> None:
|
|
||||||
parser.add_argument(
|
|
||||||
"--passphrase",
|
|
||||||
help=(
|
|
||||||
"If PAPERLESS_PASSPHRASE isn't set already, you need to specify it here"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
def handle(self, *args, **options) -> None:
|
|
||||||
try:
|
|
||||||
self.stdout.write(
|
|
||||||
self.style.WARNING(
|
|
||||||
"\n\n"
|
|
||||||
"WARNING: This script is going to work directly on your "
|
|
||||||
"document originals, so\n"
|
|
||||||
"WARNING: you probably shouldn't run "
|
|
||||||
"this unless you've got a recent backup\n"
|
|
||||||
"WARNING: handy. It "
|
|
||||||
"*should* work without a hitch, but be safe and backup your\n"
|
|
||||||
"WARNING: stuff first.\n\n"
|
|
||||||
"Hit Ctrl+C to exit now, or Enter to "
|
|
||||||
"continue.\n\n",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
_ = input()
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
return
|
|
||||||
|
|
||||||
passphrase = options["passphrase"] or settings.PASSPHRASE
|
|
||||||
if not passphrase:
|
|
||||||
raise CommandError(
|
|
||||||
"Passphrase not defined. Please set it with --passphrase or "
|
|
||||||
"by declaring it in your environment or your config.",
|
|
||||||
)
|
|
||||||
|
|
||||||
self.__gpg_to_unencrypted(passphrase)
|
|
||||||
|
|
||||||
def __gpg_to_unencrypted(self, passphrase: str) -> None:
|
|
||||||
encrypted_files = Document.objects.filter(
|
|
||||||
storage_type=Document.STORAGE_TYPE_GPG,
|
|
||||||
)
|
|
||||||
|
|
||||||
for document in encrypted_files:
|
|
||||||
self.stdout.write(f"Decrypting {document}")
|
|
||||||
|
|
||||||
old_paths = [document.source_path, document.thumbnail_path]
|
|
||||||
|
|
||||||
with document.source_file as file_handle:
|
|
||||||
raw_document = GnuPG.decrypted(file_handle, passphrase)
|
|
||||||
with document.thumbnail_file as file_handle:
|
|
||||||
raw_thumb = GnuPG.decrypted(file_handle, passphrase)
|
|
||||||
|
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
ext: str = Path(document.filename).suffix
|
|
||||||
|
|
||||||
if not ext == ".gpg":
|
|
||||||
raise CommandError(
|
|
||||||
f"Abort: encrypted file {document.source_path} does not "
|
|
||||||
f"end with .gpg",
|
|
||||||
)
|
|
||||||
|
|
||||||
document.filename = Path(document.filename).stem
|
|
||||||
|
|
||||||
with document.source_path.open("wb") as f:
|
|
||||||
f.write(raw_document)
|
|
||||||
|
|
||||||
with document.thumbnail_path.open("wb") as f:
|
|
||||||
f.write(raw_thumb)
|
|
||||||
|
|
||||||
Document.objects.filter(id=document.id).update(
|
|
||||||
storage_type=document.storage_type,
|
|
||||||
filename=document.filename,
|
|
||||||
)
|
|
||||||
|
|
||||||
for path in old_paths:
|
|
||||||
path.unlink()
|
|
||||||
@@ -3,7 +3,6 @@ import json
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
@@ -56,7 +55,6 @@ from documents.settings import EXPORTER_FILE_NAME
|
|||||||
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
from documents.settings import EXPORTER_THUMBNAIL_NAME
|
||||||
from documents.utils import copy_file_with_basic_stats
|
from documents.utils import copy_file_with_basic_stats
|
||||||
from paperless import version
|
from paperless import version
|
||||||
from paperless.db import GnuPG
|
|
||||||
from paperless.models import ApplicationConfiguration
|
from paperless.models import ApplicationConfiguration
|
||||||
from paperless_mail.models import MailAccount
|
from paperless_mail.models import MailAccount
|
||||||
from paperless_mail.models import MailRule
|
from paperless_mail.models import MailRule
|
||||||
@@ -316,20 +314,17 @@ class Command(CryptMixin, BaseCommand):
|
|||||||
total=len(document_manifest),
|
total=len(document_manifest),
|
||||||
disable=self.no_progress_bar,
|
disable=self.no_progress_bar,
|
||||||
):
|
):
|
||||||
# 3.1. store files unencrypted
|
|
||||||
document_dict["fields"]["storage_type"] = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
document = document_map[document_dict["pk"]]
|
document = document_map[document_dict["pk"]]
|
||||||
|
|
||||||
# 3.2. generate a unique filename
|
# 3.1. generate a unique filename
|
||||||
base_name = self.generate_base_name(document)
|
base_name = self.generate_base_name(document)
|
||||||
|
|
||||||
# 3.3. write filenames into manifest
|
# 3.2. write filenames into manifest
|
||||||
original_target, thumbnail_target, archive_target = (
|
original_target, thumbnail_target, archive_target = (
|
||||||
self.generate_document_targets(document, base_name, document_dict)
|
self.generate_document_targets(document, base_name, document_dict)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 3.4. write files to target folder
|
# 3.3. write files to target folder
|
||||||
if not self.data_only:
|
if not self.data_only:
|
||||||
self.copy_document_files(
|
self.copy_document_files(
|
||||||
document,
|
document,
|
||||||
@@ -423,7 +418,6 @@ class Command(CryptMixin, BaseCommand):
|
|||||||
base_name = generate_filename(
|
base_name = generate_filename(
|
||||||
document,
|
document,
|
||||||
counter=filename_counter,
|
counter=filename_counter,
|
||||||
append_gpg=False,
|
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
base_name = document.get_public_filename(counter=filename_counter)
|
base_name = document.get_public_filename(counter=filename_counter)
|
||||||
@@ -482,46 +476,24 @@ class Command(CryptMixin, BaseCommand):
|
|||||||
|
|
||||||
If the document is encrypted, the files are decrypted before copying them to the target location.
|
If the document is encrypted, the files are decrypted before copying them to the target location.
|
||||||
"""
|
"""
|
||||||
if document.storage_type == Document.STORAGE_TYPE_GPG:
|
self.check_and_copy(
|
||||||
t = int(time.mktime(document.created.timetuple()))
|
document.source_path,
|
||||||
|
document.checksum,
|
||||||
|
original_target,
|
||||||
|
)
|
||||||
|
|
||||||
original_target.parent.mkdir(parents=True, exist_ok=True)
|
if thumbnail_target:
|
||||||
with document.source_file as out_file:
|
self.check_and_copy(document.thumbnail_path, None, thumbnail_target)
|
||||||
original_target.write_bytes(GnuPG.decrypted(out_file))
|
|
||||||
os.utime(original_target, times=(t, t))
|
|
||||||
|
|
||||||
if thumbnail_target:
|
if archive_target:
|
||||||
thumbnail_target.parent.mkdir(parents=True, exist_ok=True)
|
if TYPE_CHECKING:
|
||||||
with document.thumbnail_file as out_file:
|
assert isinstance(document.archive_path, Path)
|
||||||
thumbnail_target.write_bytes(GnuPG.decrypted(out_file))
|
|
||||||
os.utime(thumbnail_target, times=(t, t))
|
|
||||||
|
|
||||||
if archive_target:
|
|
||||||
archive_target.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
assert isinstance(document.archive_path, Path)
|
|
||||||
with document.archive_path as out_file:
|
|
||||||
archive_target.write_bytes(GnuPG.decrypted(out_file))
|
|
||||||
os.utime(archive_target, times=(t, t))
|
|
||||||
else:
|
|
||||||
self.check_and_copy(
|
self.check_and_copy(
|
||||||
document.source_path,
|
document.archive_path,
|
||||||
document.checksum,
|
document.archive_checksum,
|
||||||
original_target,
|
archive_target,
|
||||||
)
|
)
|
||||||
|
|
||||||
if thumbnail_target:
|
|
||||||
self.check_and_copy(document.thumbnail_path, None, thumbnail_target)
|
|
||||||
|
|
||||||
if archive_target:
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
assert isinstance(document.archive_path, Path)
|
|
||||||
self.check_and_copy(
|
|
||||||
document.archive_path,
|
|
||||||
document.archive_checksum,
|
|
||||||
archive_target,
|
|
||||||
)
|
|
||||||
|
|
||||||
def check_and_write_json(
|
def check_and_write_json(
|
||||||
self,
|
self,
|
||||||
content: list[dict] | dict,
|
content: list[dict] | dict,
|
||||||
|
|||||||
@@ -383,8 +383,6 @@ class Command(CryptMixin, BaseCommand):
|
|||||||
else:
|
else:
|
||||||
archive_path = None
|
archive_path = None
|
||||||
|
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
with FileLock(settings.MEDIA_LOCK):
|
with FileLock(settings.MEDIA_LOCK):
|
||||||
if Path(document.source_path).is_file():
|
if Path(document.source_path).is_file():
|
||||||
raise FileExistsError(document.source_path)
|
raise FileExistsError(document.source_path)
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# Generated by Django 5.2.9 on 2026-01-24 23:05
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [
|
||||||
|
("documents", "0003_workflowaction_order"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RemoveField(
|
||||||
|
model_name="document",
|
||||||
|
name="storage_type",
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -154,13 +154,6 @@ class StoragePath(MatchingModel):
|
|||||||
|
|
||||||
|
|
||||||
class Document(SoftDeleteModel, ModelWithOwner):
|
class Document(SoftDeleteModel, ModelWithOwner):
|
||||||
STORAGE_TYPE_UNENCRYPTED = "unencrypted"
|
|
||||||
STORAGE_TYPE_GPG = "gpg"
|
|
||||||
STORAGE_TYPES = (
|
|
||||||
(STORAGE_TYPE_UNENCRYPTED, _("Unencrypted")),
|
|
||||||
(STORAGE_TYPE_GPG, _("Encrypted with GNU Privacy Guard")),
|
|
||||||
)
|
|
||||||
|
|
||||||
correspondent = models.ForeignKey(
|
correspondent = models.ForeignKey(
|
||||||
Correspondent,
|
Correspondent,
|
||||||
blank=True,
|
blank=True,
|
||||||
@@ -250,14 +243,6 @@ class Document(SoftDeleteModel, ModelWithOwner):
|
|||||||
db_index=True,
|
db_index=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
storage_type = models.CharField(
|
|
||||||
_("storage type"),
|
|
||||||
max_length=11,
|
|
||||||
choices=STORAGE_TYPES,
|
|
||||||
default=STORAGE_TYPE_UNENCRYPTED,
|
|
||||||
editable=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
added = models.DateTimeField(
|
added = models.DateTimeField(
|
||||||
_("added"),
|
_("added"),
|
||||||
default=timezone.now,
|
default=timezone.now,
|
||||||
@@ -353,12 +338,7 @@ class Document(SoftDeleteModel, ModelWithOwner):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def source_path(self) -> Path:
|
def source_path(self) -> Path:
|
||||||
if self.filename:
|
fname = str(self.filename) if self.filename else f"{self.pk:07}{self.file_type}"
|
||||||
fname = str(self.filename)
|
|
||||||
else:
|
|
||||||
fname = f"{self.pk:07}{self.file_type}"
|
|
||||||
if self.storage_type == self.STORAGE_TYPE_GPG:
|
|
||||||
fname += ".gpg" # pragma: no cover
|
|
||||||
|
|
||||||
return (settings.ORIGINALS_DIR / Path(fname)).resolve()
|
return (settings.ORIGINALS_DIR / Path(fname)).resolve()
|
||||||
|
|
||||||
@@ -407,8 +387,6 @@ class Document(SoftDeleteModel, ModelWithOwner):
|
|||||||
@property
|
@property
|
||||||
def thumbnail_path(self) -> Path:
|
def thumbnail_path(self) -> Path:
|
||||||
webp_file_name = f"{self.pk:07}.webp"
|
webp_file_name = f"{self.pk:07}.webp"
|
||||||
if self.storage_type == self.STORAGE_TYPE_GPG:
|
|
||||||
webp_file_name += ".gpg"
|
|
||||||
|
|
||||||
webp_file_path = settings.THUMBNAIL_DIR / Path(webp_file_name)
|
webp_file_path = settings.THUMBNAIL_DIR / Path(webp_file_name)
|
||||||
|
|
||||||
|
|||||||
@@ -108,7 +108,6 @@ def create_dummy_document():
|
|||||||
page_count=5,
|
page_count=5,
|
||||||
created=timezone.now(),
|
created=timezone.now(),
|
||||||
modified=timezone.now(),
|
modified=timezone.now(),
|
||||||
storage_type=Document.STORAGE_TYPE_UNENCRYPTED,
|
|
||||||
added=timezone.now(),
|
added=timezone.now(),
|
||||||
filename="/dummy/filename.pdf",
|
filename="/dummy/filename.pdf",
|
||||||
archive_filename="/dummy/archive_filename.pdf",
|
archive_filename="/dummy/archive_filename.pdf",
|
||||||
|
|||||||
BIN
src/documents/tests/samples/documents/originals/0000004.pdf
Normal file
BIN
src/documents/tests/samples/documents/originals/0000004.pdf
Normal file
Binary file not shown.
Binary file not shown.
BIN
src/documents/tests/samples/documents/thumbnails/0000004.webp
Normal file
BIN
src/documents/tests/samples/documents/thumbnails/0000004.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 2.6 KiB |
Binary file not shown.
@@ -603,23 +603,21 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
expected_filename,
|
expected_filename,
|
||||||
)
|
)
|
||||||
self.assertEqual(consume_file_args[1].title, None)
|
self.assertEqual(consume_file_args[1].title, None)
|
||||||
self.assertTrue(consume_file_args[1].skip_asn)
|
# No metadata_document_id, delete_originals False, so ASN should be None
|
||||||
|
self.assertIsNone(consume_file_args[1].asn)
|
||||||
|
|
||||||
# With metadata_document_id overrides
|
# With metadata_document_id overrides
|
||||||
result = bulk_edit.merge(doc_ids, metadata_document_id=metadata_document_id)
|
result = bulk_edit.merge(doc_ids, metadata_document_id=metadata_document_id)
|
||||||
consume_file_args, _ = mock_consume_file.call_args
|
consume_file_args, _ = mock_consume_file.call_args
|
||||||
self.assertEqual(consume_file_args[1].title, "B (merged)")
|
self.assertEqual(consume_file_args[1].title, "B (merged)")
|
||||||
self.assertEqual(consume_file_args[1].created, self.doc2.created)
|
self.assertEqual(consume_file_args[1].created, self.doc2.created)
|
||||||
self.assertTrue(consume_file_args[1].skip_asn)
|
|
||||||
|
|
||||||
self.assertEqual(result, "OK")
|
self.assertEqual(result, "OK")
|
||||||
|
|
||||||
@mock.patch("documents.bulk_edit.delete.si")
|
@mock.patch("documents.bulk_edit.delete.si")
|
||||||
@mock.patch("documents.tasks.consume_file.s")
|
@mock.patch("documents.tasks.consume_file.s")
|
||||||
@mock.patch("documents.bulk_edit.chain")
|
|
||||||
def test_merge_and_delete_originals(
|
def test_merge_and_delete_originals(
|
||||||
self,
|
self,
|
||||||
mock_chain,
|
|
||||||
mock_consume_file,
|
mock_consume_file,
|
||||||
mock_delete_documents,
|
mock_delete_documents,
|
||||||
):
|
):
|
||||||
@@ -633,6 +631,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
- Document deletion task should be called
|
- Document deletion task should be called
|
||||||
"""
|
"""
|
||||||
doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id]
|
doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id]
|
||||||
|
self.doc1.archive_serial_number = 101
|
||||||
|
self.doc2.archive_serial_number = 102
|
||||||
|
self.doc3.archive_serial_number = 103
|
||||||
|
self.doc1.save()
|
||||||
|
self.doc2.save()
|
||||||
|
self.doc3.save()
|
||||||
|
|
||||||
result = bulk_edit.merge(doc_ids, delete_originals=True)
|
result = bulk_edit.merge(doc_ids, delete_originals=True)
|
||||||
self.assertEqual(result, "OK")
|
self.assertEqual(result, "OK")
|
||||||
@@ -643,7 +647,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
|
|
||||||
mock_consume_file.assert_called()
|
mock_consume_file.assert_called()
|
||||||
mock_delete_documents.assert_called()
|
mock_delete_documents.assert_called()
|
||||||
mock_chain.assert_called_once()
|
consume_sig = mock_consume_file.return_value
|
||||||
|
consume_sig.apply_async.assert_called_once()
|
||||||
|
|
||||||
consume_file_args, _ = mock_consume_file.call_args
|
consume_file_args, _ = mock_consume_file.call_args
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -651,7 +656,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
expected_filename,
|
expected_filename,
|
||||||
)
|
)
|
||||||
self.assertEqual(consume_file_args[1].title, None)
|
self.assertEqual(consume_file_args[1].title, None)
|
||||||
self.assertTrue(consume_file_args[1].skip_asn)
|
self.assertEqual(consume_file_args[1].asn, 101)
|
||||||
|
|
||||||
delete_documents_args, _ = mock_delete_documents.call_args
|
delete_documents_args, _ = mock_delete_documents.call_args
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -659,6 +664,92 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
doc_ids,
|
doc_ids,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.doc1.refresh_from_db()
|
||||||
|
self.doc2.refresh_from_db()
|
||||||
|
self.doc3.refresh_from_db()
|
||||||
|
self.assertIsNone(self.doc1.archive_serial_number)
|
||||||
|
self.assertIsNone(self.doc2.archive_serial_number)
|
||||||
|
self.assertIsNone(self.doc3.archive_serial_number)
|
||||||
|
|
||||||
|
@mock.patch("documents.bulk_edit.delete.si")
|
||||||
|
@mock.patch("documents.tasks.consume_file.s")
|
||||||
|
def test_merge_and_delete_originals_restore_on_failure(
|
||||||
|
self,
|
||||||
|
mock_consume_file,
|
||||||
|
mock_delete_documents,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Existing documents
|
||||||
|
WHEN:
|
||||||
|
- Merge action with deleting documents is called with 1 document
|
||||||
|
- Error occurs when queuing consume file task
|
||||||
|
THEN:
|
||||||
|
- Archive serial numbers are restored
|
||||||
|
"""
|
||||||
|
doc_ids = [self.doc1.id]
|
||||||
|
self.doc1.archive_serial_number = 111
|
||||||
|
self.doc1.save()
|
||||||
|
sig = mock.Mock()
|
||||||
|
sig.apply_async.side_effect = Exception("boom")
|
||||||
|
mock_consume_file.return_value = sig
|
||||||
|
|
||||||
|
with self.assertRaises(Exception):
|
||||||
|
bulk_edit.merge(doc_ids, delete_originals=True)
|
||||||
|
|
||||||
|
self.doc1.refresh_from_db()
|
||||||
|
self.assertEqual(self.doc1.archive_serial_number, 111)
|
||||||
|
|
||||||
|
@mock.patch("documents.bulk_edit.delete.si")
|
||||||
|
@mock.patch("documents.tasks.consume_file.s")
|
||||||
|
def test_merge_and_delete_originals_metadata_handoff(
|
||||||
|
self,
|
||||||
|
mock_consume_file,
|
||||||
|
mock_delete_documents,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Existing documents with ASNs
|
||||||
|
WHEN:
|
||||||
|
- Merge with delete_originals=True and metadata_document_id set
|
||||||
|
THEN:
|
||||||
|
- Handoff ASN uses metadata document ASN
|
||||||
|
"""
|
||||||
|
doc_ids = [self.doc1.id, self.doc2.id]
|
||||||
|
self.doc1.archive_serial_number = 101
|
||||||
|
self.doc2.archive_serial_number = 202
|
||||||
|
self.doc1.save()
|
||||||
|
self.doc2.save()
|
||||||
|
|
||||||
|
result = bulk_edit.merge(
|
||||||
|
doc_ids,
|
||||||
|
metadata_document_id=self.doc2.id,
|
||||||
|
delete_originals=True,
|
||||||
|
)
|
||||||
|
self.assertEqual(result, "OK")
|
||||||
|
|
||||||
|
consume_file_args, _ = mock_consume_file.call_args
|
||||||
|
self.assertEqual(consume_file_args[1].asn, 202)
|
||||||
|
|
||||||
|
def test_restore_archive_serial_numbers_task(self):
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Existing document with no archive serial number
|
||||||
|
WHEN:
|
||||||
|
- Restore archive serial number task is called with backup data
|
||||||
|
THEN:
|
||||||
|
- Document archive serial number is restored
|
||||||
|
"""
|
||||||
|
self.doc1.archive_serial_number = 444
|
||||||
|
self.doc1.save()
|
||||||
|
Document.objects.filter(pk=self.doc1.id).update(archive_serial_number=None)
|
||||||
|
|
||||||
|
backup = {self.doc1.id: 444}
|
||||||
|
bulk_edit.restore_archive_serial_numbers_task(backup)
|
||||||
|
|
||||||
|
self.doc1.refresh_from_db()
|
||||||
|
self.assertEqual(self.doc1.archive_serial_number, 444)
|
||||||
|
|
||||||
@mock.patch("documents.tasks.consume_file.s")
|
@mock.patch("documents.tasks.consume_file.s")
|
||||||
def test_merge_with_archive_fallback(self, mock_consume_file):
|
def test_merge_with_archive_fallback(self, mock_consume_file):
|
||||||
"""
|
"""
|
||||||
@@ -727,6 +818,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
self.assertEqual(mock_consume_file.call_count, 2)
|
self.assertEqual(mock_consume_file.call_count, 2)
|
||||||
consume_file_args, _ = mock_consume_file.call_args
|
consume_file_args, _ = mock_consume_file.call_args
|
||||||
self.assertEqual(consume_file_args[1].title, "B (split 2)")
|
self.assertEqual(consume_file_args[1].title, "B (split 2)")
|
||||||
|
self.assertIsNone(consume_file_args[1].asn)
|
||||||
|
|
||||||
self.assertEqual(result, "OK")
|
self.assertEqual(result, "OK")
|
||||||
|
|
||||||
@@ -751,6 +843,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
"""
|
"""
|
||||||
doc_ids = [self.doc2.id]
|
doc_ids = [self.doc2.id]
|
||||||
pages = [[1, 2], [3]]
|
pages = [[1, 2], [3]]
|
||||||
|
self.doc2.archive_serial_number = 200
|
||||||
|
self.doc2.save()
|
||||||
|
|
||||||
result = bulk_edit.split(doc_ids, pages, delete_originals=True)
|
result = bulk_edit.split(doc_ids, pages, delete_originals=True)
|
||||||
self.assertEqual(result, "OK")
|
self.assertEqual(result, "OK")
|
||||||
@@ -768,6 +862,42 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
doc_ids,
|
doc_ids,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.doc2.refresh_from_db()
|
||||||
|
self.assertIsNone(self.doc2.archive_serial_number)
|
||||||
|
|
||||||
|
@mock.patch("documents.bulk_edit.delete.si")
|
||||||
|
@mock.patch("documents.tasks.consume_file.s")
|
||||||
|
@mock.patch("documents.bulk_edit.chord")
|
||||||
|
def test_split_restore_on_failure(
|
||||||
|
self,
|
||||||
|
mock_chord,
|
||||||
|
mock_consume_file,
|
||||||
|
mock_delete_documents,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Existing documents
|
||||||
|
WHEN:
|
||||||
|
- Split action with deleting documents is called with 1 document and 2 page groups
|
||||||
|
- Error occurs when queuing chord task
|
||||||
|
THEN:
|
||||||
|
- Archive serial numbers are restored
|
||||||
|
"""
|
||||||
|
doc_ids = [self.doc2.id]
|
||||||
|
pages = [[1, 2]]
|
||||||
|
self.doc2.archive_serial_number = 222
|
||||||
|
self.doc2.save()
|
||||||
|
|
||||||
|
sig = mock.Mock()
|
||||||
|
sig.apply_async.side_effect = Exception("boom")
|
||||||
|
mock_chord.return_value = sig
|
||||||
|
|
||||||
|
result = bulk_edit.split(doc_ids, pages, delete_originals=True)
|
||||||
|
self.assertEqual(result, "OK")
|
||||||
|
|
||||||
|
self.doc2.refresh_from_db()
|
||||||
|
self.assertEqual(self.doc2.archive_serial_number, 222)
|
||||||
|
|
||||||
@mock.patch("documents.tasks.consume_file.delay")
|
@mock.patch("documents.tasks.consume_file.delay")
|
||||||
@mock.patch("pikepdf.Pdf.save")
|
@mock.patch("pikepdf.Pdf.save")
|
||||||
def test_split_with_errors(self, mock_save_pdf, mock_consume_file):
|
def test_split_with_errors(self, mock_save_pdf, mock_consume_file):
|
||||||
@@ -968,10 +1098,49 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
mock_chord.return_value.delay.return_value = None
|
mock_chord.return_value.delay.return_value = None
|
||||||
doc_ids = [self.doc2.id]
|
doc_ids = [self.doc2.id]
|
||||||
operations = [{"page": 1}, {"page": 2}]
|
operations = [{"page": 1}, {"page": 2}]
|
||||||
|
self.doc2.archive_serial_number = 250
|
||||||
|
self.doc2.save()
|
||||||
|
|
||||||
result = bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
|
result = bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
|
||||||
self.assertEqual(result, "OK")
|
self.assertEqual(result, "OK")
|
||||||
mock_chord.assert_called_once()
|
mock_chord.assert_called_once()
|
||||||
|
consume_file_args, _ = mock_consume_file.call_args
|
||||||
|
self.assertEqual(consume_file_args[1].asn, 250)
|
||||||
|
self.doc2.refresh_from_db()
|
||||||
|
self.assertIsNone(self.doc2.archive_serial_number)
|
||||||
|
|
||||||
|
@mock.patch("documents.bulk_edit.delete.si")
|
||||||
|
@mock.patch("documents.tasks.consume_file.s")
|
||||||
|
@mock.patch("documents.bulk_edit.chord")
|
||||||
|
def test_edit_pdf_restore_on_failure(
|
||||||
|
self,
|
||||||
|
mock_chord,
|
||||||
|
mock_consume_file,
|
||||||
|
mock_delete_documents,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Existing document
|
||||||
|
WHEN:
|
||||||
|
- edit_pdf is called with delete_original=True
|
||||||
|
- Error occurs when queuing chord task
|
||||||
|
THEN:
|
||||||
|
- Archive serial numbers are restored
|
||||||
|
"""
|
||||||
|
doc_ids = [self.doc2.id]
|
||||||
|
operations = [{"page": 1}]
|
||||||
|
self.doc2.archive_serial_number = 333
|
||||||
|
self.doc2.save()
|
||||||
|
|
||||||
|
sig = mock.Mock()
|
||||||
|
sig.apply_async.side_effect = Exception("boom")
|
||||||
|
mock_chord.return_value = sig
|
||||||
|
|
||||||
|
with self.assertRaises(Exception):
|
||||||
|
bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
|
||||||
|
|
||||||
|
self.doc2.refresh_from_db()
|
||||||
|
self.assertEqual(self.doc2.archive_serial_number, 333)
|
||||||
|
|
||||||
@mock.patch("documents.tasks.update_document_content_maybe_archive_file.delay")
|
@mock.patch("documents.tasks.update_document_content_maybe_archive_file.delay")
|
||||||
def test_edit_pdf_with_update_document(self, mock_update_document):
|
def test_edit_pdf_with_update_document(self, mock_update_document):
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import textwrap
|
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
from django.core.checks import Error
|
from django.core.checks import Error
|
||||||
@@ -6,60 +5,11 @@ from django.core.checks import Warning
|
|||||||
from django.test import TestCase
|
from django.test import TestCase
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
|
|
||||||
from documents.checks import changed_password_check
|
|
||||||
from documents.checks import filename_format_check
|
from documents.checks import filename_format_check
|
||||||
from documents.checks import parser_check
|
from documents.checks import parser_check
|
||||||
from documents.models import Document
|
|
||||||
from documents.tests.factories import DocumentFactory
|
|
||||||
|
|
||||||
|
|
||||||
class TestDocumentChecks(TestCase):
|
class TestDocumentChecks(TestCase):
|
||||||
def test_changed_password_check_empty_db(self):
|
|
||||||
self.assertListEqual(changed_password_check(None), [])
|
|
||||||
|
|
||||||
def test_changed_password_check_no_encryption(self):
|
|
||||||
DocumentFactory.create(storage_type=Document.STORAGE_TYPE_UNENCRYPTED)
|
|
||||||
self.assertListEqual(changed_password_check(None), [])
|
|
||||||
|
|
||||||
def test_encrypted_missing_passphrase(self):
|
|
||||||
DocumentFactory.create(storage_type=Document.STORAGE_TYPE_GPG)
|
|
||||||
msgs = changed_password_check(None)
|
|
||||||
self.assertEqual(len(msgs), 1)
|
|
||||||
msg_text = msgs[0].msg
|
|
||||||
self.assertEqual(
|
|
||||||
msg_text,
|
|
||||||
"The database contains encrypted documents but no password is set.",
|
|
||||||
)
|
|
||||||
|
|
||||||
@override_settings(
|
|
||||||
PASSPHRASE="test",
|
|
||||||
)
|
|
||||||
@mock.patch("paperless.db.GnuPG.decrypted")
|
|
||||||
@mock.patch("documents.models.Document.source_file")
|
|
||||||
def test_encrypted_decrypt_fails(self, mock_decrypted, mock_source_file):
|
|
||||||
mock_decrypted.return_value = None
|
|
||||||
mock_source_file.return_value = b""
|
|
||||||
|
|
||||||
DocumentFactory.create(storage_type=Document.STORAGE_TYPE_GPG)
|
|
||||||
|
|
||||||
msgs = changed_password_check(None)
|
|
||||||
|
|
||||||
self.assertEqual(len(msgs), 1)
|
|
||||||
msg_text = msgs[0].msg
|
|
||||||
self.assertEqual(
|
|
||||||
msg_text,
|
|
||||||
textwrap.dedent(
|
|
||||||
"""
|
|
||||||
The current password doesn't match the password of the
|
|
||||||
existing documents.
|
|
||||||
|
|
||||||
If you intend to change your password, you must first export
|
|
||||||
all of the old documents, start fresh with the new password
|
|
||||||
and then re-import them."
|
|
||||||
""",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_parser_check(self):
|
def test_parser_check(self):
|
||||||
self.assertEqual(parser_check(None), [])
|
self.assertEqual(parser_check(None), [])
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from django.test import override_settings
|
|||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from guardian.core import ObjectPermissionChecker
|
from guardian.core import ObjectPermissionChecker
|
||||||
|
|
||||||
|
from documents.barcodes import BarcodePlugin
|
||||||
from documents.consumer import ConsumerError
|
from documents.consumer import ConsumerError
|
||||||
from documents.data_models import DocumentMetadataOverrides
|
from documents.data_models import DocumentMetadataOverrides
|
||||||
from documents.data_models import DocumentSource
|
from documents.data_models import DocumentSource
|
||||||
@@ -412,14 +413,6 @@ class TestConsumer(
|
|||||||
self.assertEqual(document.archive_serial_number, 123)
|
self.assertEqual(document.archive_serial_number, 123)
|
||||||
self._assert_first_last_send_progress()
|
self._assert_first_last_send_progress()
|
||||||
|
|
||||||
def testMetadataOverridesSkipAsnPropagation(self):
|
|
||||||
overrides = DocumentMetadataOverrides()
|
|
||||||
incoming = DocumentMetadataOverrides(skip_asn=True)
|
|
||||||
|
|
||||||
overrides.update(incoming)
|
|
||||||
|
|
||||||
self.assertTrue(overrides.skip_asn)
|
|
||||||
|
|
||||||
def testOverrideTitlePlaceholders(self):
|
def testOverrideTitlePlaceholders(self):
|
||||||
c = Correspondent.objects.create(name="Correspondent Name")
|
c = Correspondent.objects.create(name="Correspondent Name")
|
||||||
dt = DocumentType.objects.create(name="DocType Name")
|
dt = DocumentType.objects.create(name="DocType Name")
|
||||||
@@ -1240,3 +1233,46 @@ class PostConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
|
|||||||
r"sample\.pdf: Error while executing post-consume script: Command '\[.*\]' returned non-zero exit status \d+\.",
|
r"sample\.pdf: Error while executing post-consume script: Command '\[.*\]' returned non-zero exit status \d+\.",
|
||||||
):
|
):
|
||||||
consumer.run_post_consume_script(doc)
|
consumer.run_post_consume_script(doc)
|
||||||
|
|
||||||
|
|
||||||
|
class TestMetadataOverrides(TestCase):
|
||||||
|
def test_update_skip_asn_if_exists(self):
|
||||||
|
base = DocumentMetadataOverrides()
|
||||||
|
incoming = DocumentMetadataOverrides(skip_asn_if_exists=True)
|
||||||
|
base.update(incoming)
|
||||||
|
self.assertTrue(base.skip_asn_if_exists)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBarcodeApplyDetectedASN(TestCase):
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Existing Documents with ASN 123
|
||||||
|
WHEN:
|
||||||
|
- A BarcodePlugin which detected an ASN
|
||||||
|
THEN:
|
||||||
|
- If skip_asn_if_exists is set, and ASN exists, do not set ASN
|
||||||
|
- If skip_asn_if_exists is set, and ASN does not exist, set ASN
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_apply_detected_asn_skips_existing_when_flag_set(self):
|
||||||
|
doc = Document.objects.create(
|
||||||
|
checksum="X1",
|
||||||
|
title="D1",
|
||||||
|
archive_serial_number=123,
|
||||||
|
)
|
||||||
|
metadata = DocumentMetadataOverrides(skip_asn_if_exists=True)
|
||||||
|
plugin = BarcodePlugin(
|
||||||
|
input_doc=mock.Mock(),
|
||||||
|
metadata=metadata,
|
||||||
|
status_mgr=mock.Mock(),
|
||||||
|
base_tmp_dir=tempfile.gettempdir(),
|
||||||
|
task_id="test-task",
|
||||||
|
)
|
||||||
|
|
||||||
|
plugin._apply_detected_asn(123)
|
||||||
|
self.assertIsNone(plugin.metadata.asn)
|
||||||
|
|
||||||
|
doc.hard_delete()
|
||||||
|
|
||||||
|
plugin._apply_detected_asn(123)
|
||||||
|
self.assertEqual(plugin.metadata.asn, 123)
|
||||||
|
|||||||
@@ -34,22 +34,14 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
def test_generate_source_filename(self):
|
def test_generate_source_filename(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
self.assertEqual(generate_filename(document), Path(f"{document.pk:07d}.pdf"))
|
self.assertEqual(generate_filename(document), Path(f"{document.pk:07d}.pdf"))
|
||||||
|
|
||||||
document.storage_type = Document.STORAGE_TYPE_GPG
|
|
||||||
self.assertEqual(
|
|
||||||
generate_filename(document),
|
|
||||||
Path(f"{document.pk:07d}.pdf.gpg"),
|
|
||||||
)
|
|
||||||
|
|
||||||
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
|
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
|
||||||
def test_file_renaming(self):
|
def test_file_renaming(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Test default source_path
|
# Test default source_path
|
||||||
@@ -63,11 +55,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
self.assertEqual(document.filename, Path("none/none.pdf"))
|
self.assertEqual(document.filename, Path("none/none.pdf"))
|
||||||
|
|
||||||
# Enable encryption and check again
|
|
||||||
document.storage_type = Document.STORAGE_TYPE_GPG
|
|
||||||
document.filename = generate_filename(document)
|
|
||||||
self.assertEqual(document.filename, Path("none/none.pdf.gpg"))
|
|
||||||
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# test that creating dirs for the source_path creates the correct directory
|
# test that creating dirs for the source_path creates the correct directory
|
||||||
@@ -87,14 +74,14 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
settings.ORIGINALS_DIR / "none",
|
settings.ORIGINALS_DIR / "none",
|
||||||
)
|
)
|
||||||
self.assertIsFile(
|
self.assertIsFile(
|
||||||
settings.ORIGINALS_DIR / "test" / "test.pdf.gpg",
|
settings.ORIGINALS_DIR / "test" / "test.pdf",
|
||||||
)
|
)
|
||||||
|
|
||||||
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
|
@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
|
||||||
def test_file_renaming_missing_permissions(self):
|
def test_file_renaming_missing_permissions(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
@@ -128,14 +115,13 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
def test_file_renaming_database_error(self):
|
def test_file_renaming_database_error(self):
|
||||||
Document.objects.create(
|
Document.objects.create(
|
||||||
mime_type="application/pdf",
|
mime_type="application/pdf",
|
||||||
storage_type=Document.STORAGE_TYPE_UNENCRYPTED,
|
|
||||||
checksum="AAAAA",
|
checksum="AAAAA",
|
||||||
)
|
)
|
||||||
|
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.checksum = "BBBBB"
|
document.checksum = "BBBBB"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
@@ -170,7 +156,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
def test_document_delete(self):
|
def test_document_delete(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
@@ -196,7 +182,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
def test_document_delete_trash_dir(self):
|
def test_document_delete_trash_dir(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
@@ -221,7 +207,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
# Create an identical document and ensure it is trashed under a new name
|
# Create an identical document and ensure it is trashed under a new name
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
document.filename = generate_filename(document)
|
document.filename = generate_filename(document)
|
||||||
document.save()
|
document.save()
|
||||||
@@ -235,7 +221,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
def test_document_delete_nofile(self):
|
def test_document_delete_nofile(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
document.delete()
|
document.delete()
|
||||||
@@ -245,7 +231,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
def test_directory_not_empty(self):
|
def test_directory_not_empty(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
@@ -362,7 +348,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
def test_nested_directory_cleanup(self):
|
def test_nested_directory_cleanup(self):
|
||||||
document = Document()
|
document = Document()
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
@@ -390,7 +376,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
document = Document()
|
document = Document()
|
||||||
document.pk = 1
|
document.pk = 1
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
||||||
|
|
||||||
@@ -403,7 +388,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
document = Document()
|
document = Document()
|
||||||
document.pk = 1
|
document.pk = 1
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
||||||
|
|
||||||
@@ -429,7 +413,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
document = Document()
|
document = Document()
|
||||||
document.pk = 1
|
document.pk = 1
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
||||||
|
|
||||||
@@ -438,7 +421,6 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
document = Document()
|
document = Document()
|
||||||
document.pk = 1
|
document.pk = 1
|
||||||
document.mime_type = "application/pdf"
|
document.mime_type = "application/pdf"
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
|
|
||||||
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
self.assertEqual(generate_filename(document), Path("0000001.pdf"))
|
||||||
|
|
||||||
@@ -1258,7 +1240,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase):
|
|||||||
title="doc1",
|
title="doc1",
|
||||||
mime_type="application/pdf",
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
|
|
||||||
document.save()
|
document.save()
|
||||||
|
|
||||||
# Ensure that filename is properly generated
|
# Ensure that filename is properly generated
|
||||||
@@ -1732,7 +1714,6 @@ class TestPathDateLocalization:
|
|||||||
document = DocumentFactory.create(
|
document = DocumentFactory.create(
|
||||||
title="My Document",
|
title="My Document",
|
||||||
mime_type="application/pdf",
|
mime_type="application/pdf",
|
||||||
storage_type=Document.STORAGE_TYPE_UNENCRYPTED,
|
|
||||||
created=self.TEST_DATE, # 2023-10-26 (which is a Thursday)
|
created=self.TEST_DATE, # 2023-10-26 (which is a Thursday)
|
||||||
)
|
)
|
||||||
with override_settings(FILENAME_FORMAT=filename_format):
|
with override_settings(FILENAME_FORMAT=filename_format):
|
||||||
|
|||||||
@@ -1,7 +1,5 @@
|
|||||||
import filecmp
|
import filecmp
|
||||||
import hashlib
|
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
@@ -96,66 +94,6 @@ class TestArchiver(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
|
|||||||
self.assertEqual(doc2.archive_filename, "document_01.pdf")
|
self.assertEqual(doc2.archive_filename, "document_01.pdf")
|
||||||
|
|
||||||
|
|
||||||
class TestDecryptDocuments(FileSystemAssertsMixin, TestCase):
|
|
||||||
@mock.patch("documents.management.commands.decrypt_documents.input")
|
|
||||||
def test_decrypt(self, m):
|
|
||||||
media_dir = tempfile.mkdtemp()
|
|
||||||
originals_dir = Path(media_dir) / "documents" / "originals"
|
|
||||||
thumb_dir = Path(media_dir) / "documents" / "thumbnails"
|
|
||||||
originals_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
thumb_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
with override_settings(
|
|
||||||
ORIGINALS_DIR=originals_dir,
|
|
||||||
THUMBNAIL_DIR=thumb_dir,
|
|
||||||
PASSPHRASE="test",
|
|
||||||
FILENAME_FORMAT=None,
|
|
||||||
):
|
|
||||||
doc = Document.objects.create(
|
|
||||||
checksum="82186aaa94f0b98697d704b90fd1c072",
|
|
||||||
title="wow",
|
|
||||||
filename="0000004.pdf.gpg",
|
|
||||||
mime_type="application/pdf",
|
|
||||||
storage_type=Document.STORAGE_TYPE_GPG,
|
|
||||||
)
|
|
||||||
|
|
||||||
shutil.copy(
|
|
||||||
(
|
|
||||||
Path(__file__).parent
|
|
||||||
/ "samples"
|
|
||||||
/ "documents"
|
|
||||||
/ "originals"
|
|
||||||
/ "0000004.pdf.gpg"
|
|
||||||
),
|
|
||||||
originals_dir / "0000004.pdf.gpg",
|
|
||||||
)
|
|
||||||
shutil.copy(
|
|
||||||
(
|
|
||||||
Path(__file__).parent
|
|
||||||
/ "samples"
|
|
||||||
/ "documents"
|
|
||||||
/ "thumbnails"
|
|
||||||
/ "0000004.webp.gpg"
|
|
||||||
),
|
|
||||||
thumb_dir / f"{doc.id:07}.webp.gpg",
|
|
||||||
)
|
|
||||||
|
|
||||||
call_command("decrypt_documents")
|
|
||||||
|
|
||||||
doc.refresh_from_db()
|
|
||||||
|
|
||||||
self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
|
|
||||||
self.assertEqual(doc.filename, "0000004.pdf")
|
|
||||||
self.assertIsFile(Path(originals_dir) / "0000004.pdf")
|
|
||||||
self.assertIsFile(doc.source_path)
|
|
||||||
self.assertIsFile(Path(thumb_dir) / f"{doc.id:07}.webp")
|
|
||||||
self.assertIsFile(doc.thumbnail_path)
|
|
||||||
|
|
||||||
with doc.source_file as f:
|
|
||||||
checksum: str = hashlib.md5(f.read()).hexdigest()
|
|
||||||
self.assertEqual(checksum, doc.checksum)
|
|
||||||
|
|
||||||
|
|
||||||
class TestMakeIndex(TestCase):
|
class TestMakeIndex(TestCase):
|
||||||
@mock.patch("documents.management.commands.document_index.index_reindex")
|
@mock.patch("documents.management.commands.document_index.index_reindex")
|
||||||
def test_reindex(self, m):
|
def test_reindex(self, m):
|
||||||
|
|||||||
@@ -86,9 +86,8 @@ class TestExportImport(
|
|||||||
content="Content",
|
content="Content",
|
||||||
checksum="82186aaa94f0b98697d704b90fd1c072",
|
checksum="82186aaa94f0b98697d704b90fd1c072",
|
||||||
title="wow_dec",
|
title="wow_dec",
|
||||||
filename="0000004.pdf.gpg",
|
filename="0000004.pdf",
|
||||||
mime_type="application/pdf",
|
mime_type="application/pdf",
|
||||||
storage_type=Document.STORAGE_TYPE_GPG,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
self.note = Note.objects.create(
|
self.note = Note.objects.create(
|
||||||
@@ -242,11 +241,6 @@ class TestExportImport(
|
|||||||
checksum = hashlib.md5(f.read()).hexdigest()
|
checksum = hashlib.md5(f.read()).hexdigest()
|
||||||
self.assertEqual(checksum, element["fields"]["checksum"])
|
self.assertEqual(checksum, element["fields"]["checksum"])
|
||||||
|
|
||||||
self.assertEqual(
|
|
||||||
element["fields"]["storage_type"],
|
|
||||||
Document.STORAGE_TYPE_UNENCRYPTED,
|
|
||||||
)
|
|
||||||
|
|
||||||
if document_exporter.EXPORTER_ARCHIVE_NAME in element:
|
if document_exporter.EXPORTER_ARCHIVE_NAME in element:
|
||||||
fname = (
|
fname = (
|
||||||
self.target / element[document_exporter.EXPORTER_ARCHIVE_NAME]
|
self.target / element[document_exporter.EXPORTER_ARCHIVE_NAME]
|
||||||
@@ -436,7 +430,7 @@ class TestExportImport(
|
|||||||
Document.objects.create(
|
Document.objects.create(
|
||||||
checksum="AAAAAAAAAAAAAAAAA",
|
checksum="AAAAAAAAAAAAAAAAA",
|
||||||
title="wow",
|
title="wow",
|
||||||
filename="0000004.pdf",
|
filename="0000010.pdf",
|
||||||
mime_type="application/pdf",
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
self.assertRaises(FileNotFoundError, call_command, "document_exporter", target)
|
self.assertRaises(FileNotFoundError, call_command, "document_exporter", target)
|
||||||
|
|||||||
@@ -195,7 +195,6 @@ from paperless import version
|
|||||||
from paperless.celery import app as celery_app
|
from paperless.celery import app as celery_app
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
from paperless.config import GeneralConfig
|
from paperless.config import GeneralConfig
|
||||||
from paperless.db import GnuPG
|
|
||||||
from paperless.models import ApplicationConfiguration
|
from paperless.models import ApplicationConfiguration
|
||||||
from paperless.serialisers import GroupSerializer
|
from paperless.serialisers import GroupSerializer
|
||||||
from paperless.serialisers import UserSerializer
|
from paperless.serialisers import UserSerializer
|
||||||
@@ -1071,10 +1070,8 @@ class DocumentViewSet(
|
|||||||
doc,
|
doc,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
if doc.storage_type == Document.STORAGE_TYPE_GPG:
|
|
||||||
handle = GnuPG.decrypted(doc.thumbnail_file)
|
handle = doc.thumbnail_file
|
||||||
else:
|
|
||||||
handle = doc.thumbnail_file
|
|
||||||
|
|
||||||
return HttpResponse(handle, content_type="image/webp")
|
return HttpResponse(handle, content_type="image/webp")
|
||||||
except (FileNotFoundError, Document.DoesNotExist):
|
except (FileNotFoundError, Document.DoesNotExist):
|
||||||
@@ -2824,9 +2821,6 @@ def serve_file(*, doc: Document, use_archive: bool, disposition: str):
|
|||||||
if mime_type in {"application/csv", "text/csv"} and disposition == "inline":
|
if mime_type in {"application/csv", "text/csv"} and disposition == "inline":
|
||||||
mime_type = "text/plain"
|
mime_type = "text/plain"
|
||||||
|
|
||||||
if doc.storage_type == Document.STORAGE_TYPE_GPG:
|
|
||||||
file_handle = GnuPG.decrypted(file_handle)
|
|
||||||
|
|
||||||
response = HttpResponse(file_handle, content_type=mime_type)
|
response = HttpResponse(file_handle, content_type=mime_type)
|
||||||
# Firefox is not able to handle unicode characters in filename field
|
# Firefox is not able to handle unicode characters in filename field
|
||||||
# RFC 5987 addresses this issue
|
# RFC 5987 addresses this issue
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,17 +0,0 @@
|
|||||||
import gnupg
|
|
||||||
from django.conf import settings
|
|
||||||
|
|
||||||
|
|
||||||
class GnuPG:
|
|
||||||
"""
|
|
||||||
A handy singleton to use when handling encrypted files.
|
|
||||||
"""
|
|
||||||
|
|
||||||
gpg = gnupg.GPG(gnupghome=settings.GNUPG_HOME)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def decrypted(cls, file_handle, passphrase=None):
|
|
||||||
if not passphrase:
|
|
||||||
passphrase = settings.PASSPHRASE
|
|
||||||
|
|
||||||
return cls.gpg.decrypt_file(file_handle, passphrase=passphrase).data
|
|
||||||
@@ -1203,19 +1203,6 @@ EMAIL_PARSE_DEFAULT_LAYOUT = __get_int(
|
|||||||
1, # MailRule.PdfLayout.TEXT_HTML but that can't be imported here
|
1, # MailRule.PdfLayout.TEXT_HTML but that can't be imported here
|
||||||
)
|
)
|
||||||
|
|
||||||
# Pre-2.x versions of Paperless stored your documents locally with GPG
|
|
||||||
# encryption, but that is no longer the default. This behaviour is still
|
|
||||||
# available, but it must be explicitly enabled by setting
|
|
||||||
# `PAPERLESS_PASSPHRASE` in your environment or config file. The default is to
|
|
||||||
# store these files unencrypted.
|
|
||||||
#
|
|
||||||
# Translation:
|
|
||||||
# * If you're a new user, you can safely ignore this setting.
|
|
||||||
# * If you're upgrading from 1.x, this must be set, OR you can run
|
|
||||||
# `./manage.py change_storage_type gpg unencrypted` to decrypt your files,
|
|
||||||
# after which you can unset this value.
|
|
||||||
PASSPHRASE = os.getenv("PAPERLESS_PASSPHRASE")
|
|
||||||
|
|
||||||
# Trigger a script after every successful document consumption?
|
# Trigger a script after every successful document consumption?
|
||||||
PRE_CONSUME_SCRIPT = os.getenv("PAPERLESS_PRE_CONSUME_SCRIPT")
|
PRE_CONSUME_SCRIPT = os.getenv("PAPERLESS_PRE_CONSUME_SCRIPT")
|
||||||
POST_CONSUME_SCRIPT = os.getenv("PAPERLESS_POST_CONSUME_SCRIPT")
|
POST_CONSUME_SCRIPT = os.getenv("PAPERLESS_POST_CONSUME_SCRIPT")
|
||||||
|
|||||||
Reference in New Issue
Block a user