mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-04-02 13:45:10 -05:00
Merge branch 'dev' into quick-filters
This commit is contained in:
Michael Shamoon
GitHub
commit
7a75d3f41b
@ -28,6 +28,7 @@ Here's what you get:
|
||||
# Features
|
||||
|
||||
* Performs OCR on your documents, adds selectable text to image only documents and adds tags, correspondents and document types to your documents.
|
||||
* Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and can be configured freely.
|
||||
* Single page application front end. Should be pretty snappy. Will be mobile friendly in the future.
|
||||
* Includes a dashboard that shows basic statistics and has document upload.
|
||||
* Filtering by tags, correspondents, types, and more.
|
||||
|
||||
@ -25,6 +25,11 @@ wait_for_postgres() {
|
||||
host="${PAPERLESS_DBHOST}"
|
||||
port="${PAPERLESS_DBPORT}"
|
||||
|
||||
if [[ -z $port ]] ;
|
||||
then
|
||||
port="5432"
|
||||
fi
|
||||
|
||||
while !</dev/tcp/$host/$port ;
|
||||
do
|
||||
|
||||
@ -114,13 +119,13 @@ install_languages() {
|
||||
done
|
||||
}
|
||||
|
||||
initialize
|
||||
|
||||
# Install additional languages if specified
|
||||
if [[ ! -z "$PAPERLESS_OCR_LANGUAGES" ]]; then
|
||||
install_languages "$PAPERLESS_OCR_LANGUAGES"
|
||||
fi
|
||||
|
||||
initialize
|
||||
|
||||
if [[ "$1" != "/"* ]]; then
|
||||
exec sudo -HEu paperless python3 manage.py "$@"
|
||||
else
|
||||
|
||||
@ -15,7 +15,7 @@ services:
|
||||
POSTGRES_PASSWORD: paperless
|
||||
|
||||
webserver:
|
||||
image: jonaswinkler/paperless-ng:0.9.5
|
||||
image: jonaswinkler/paperless-ng:0.9.6
|
||||
restart: always
|
||||
depends_on:
|
||||
- db
|
||||
|
||||
@ -5,7 +5,7 @@ services:
|
||||
restart: always
|
||||
|
||||
webserver:
|
||||
image: jonaswinkler/paperless-ng:0.9.5
|
||||
image: jonaswinkler/paperless-ng:0.9.6
|
||||
restart: always
|
||||
depends_on:
|
||||
- broker
|
||||
|
||||
@ -32,6 +32,7 @@ This release focusses primarily on many small issues with the UI.
|
||||
* Other
|
||||
|
||||
* Fixed an issue with the docker image when a non-standard PostgreSQL port was used.
|
||||
* The docker image was trying check for installed languages before actually installing them.
|
||||
* ``FILENAME_FORMAT`` placeholder for document types.
|
||||
* The filename formatter is now less restrictive with file names and tries to
|
||||
conserve the original correspondents, types and titles as much as possible.
|
||||
|
||||
@ -152,6 +152,16 @@ PAPERLESS_AUTO_LOGIN_USERNAME=<username>
|
||||
|
||||
Defaults to none, which disables this feature.
|
||||
|
||||
|
||||
PAPERLESS_COOKIE_PREFIX=<str>
|
||||
Specify a prefix that is added to the cookies used by paperless to identify
|
||||
the currently logged in user. This is useful for when you're running two
|
||||
instances of paperless on the same host.
|
||||
|
||||
After changing this, you will have to login again.
|
||||
|
||||
Defaults to ``""``, which does not alter the cookie names.
|
||||
|
||||
.. _configuration-ocr:
|
||||
|
||||
OCR settings
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
#PAPERLESS_FORCE_SCRIPT_NAME=
|
||||
#PAPERLESS_STATIC_URL=/static/
|
||||
#PAPERLESS_AUTO_LOGIN_USERNAME=
|
||||
#PAPERLESS_COOKIE_PREFIX=
|
||||
|
||||
# OCR settings
|
||||
|
||||
|
||||
63
src-ui/package-lock.json
generated
63
src-ui/package-lock.json
generated
@ -2215,6 +2215,11 @@
|
||||
"integrity": "sha512-UV1/ZJMC+HcP902wWdpC43cAcGu0IQk/I5bXjP2aSuCjsk3cE74mDvFrLKga7oDC170ugOAYBwfT4DSQW3akDA==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/pdfjs-dist": {
|
||||
"version": "2.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/pdfjs-dist/-/pdfjs-dist-2.1.7.tgz",
|
||||
"integrity": "sha512-nQIwcPUhkAIyn7x9NS0lR/qxYfd5unRtfGkMjvpgF4Sh28IXftRymaNmFKTTdejDNY25NDGSIyjwj/BRwAPexg=="
|
||||
},
|
||||
"@types/q": {
|
||||
"version": "1.5.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz",
|
||||
@ -3023,6 +3028,16 @@
|
||||
"integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
|
||||
"dev": true
|
||||
},
|
||||
"bindings": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
|
||||
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"file-uri-to-path": "1.0.0"
|
||||
}
|
||||
},
|
||||
"blob": {
|
||||
"version": "0.0.5",
|
||||
"resolved": "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz",
|
||||
@ -5508,6 +5523,13 @@
|
||||
"schema-utils": "^2.6.5"
|
||||
}
|
||||
},
|
||||
"file-uri-to-path": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
|
||||
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"fill-range": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
|
||||
@ -8208,6 +8230,13 @@
|
||||
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==",
|
||||
"dev": true
|
||||
},
|
||||
"nan": {
|
||||
"version": "2.14.2",
|
||||
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
|
||||
"integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==",
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"nanomatch": {
|
||||
"version": "1.2.13",
|
||||
"resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
|
||||
@ -8260,6 +8289,23 @@
|
||||
"moment": "2.18.1"
|
||||
}
|
||||
},
|
||||
"ng2-pdf-viewer": {
|
||||
"version": "6.3.2",
|
||||
"resolved": "https://registry.npmjs.org/ng2-pdf-viewer/-/ng2-pdf-viewer-6.3.2.tgz",
|
||||
"integrity": "sha512-H2tBhDd+Lq6CUzK2g54HsCcZDR2wTn1sDjYqKY3yF0Ydasl2R5ppCKynZBU/zge4EKvmHglJI120FbQMpJKDYQ==",
|
||||
"requires": {
|
||||
"@types/pdfjs-dist": "^2.1.4",
|
||||
"pdfjs-dist": "^2.4.456",
|
||||
"tslib": "^1.10.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"tslib": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
|
||||
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"ngx-cookie-service": {
|
||||
"version": "10.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ngx-cookie-service/-/ngx-cookie-service-10.1.1.tgz",
|
||||
@ -9270,6 +9316,11 @@
|
||||
"sha.js": "^2.4.8"
|
||||
}
|
||||
},
|
||||
"pdfjs-dist": {
|
||||
"version": "2.5.207",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.5.207.tgz",
|
||||
"integrity": "sha512-xGDUhnCYPfHy+unMXCLCJtlpZaaZ17Ew3WIL0tnSgKFUZXHAPD49GO9xScyszSsQMoutNDgRb+rfBXIaX/lJbw=="
|
||||
},
|
||||
"performance-now": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
|
||||
@ -13228,7 +13279,11 @@
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz",
|
||||
"integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==",
|
||||
"dev": true,
|
||||
"optional": true
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"bindings": "^1.5.0",
|
||||
"nan": "^2.12.1"
|
||||
}
|
||||
},
|
||||
"glob-parent": {
|
||||
"version": "3.1.0",
|
||||
@ -13832,7 +13887,11 @@
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz",
|
||||
"integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==",
|
||||
"dev": true,
|
||||
"optional": true
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"bindings": "^1.5.0",
|
||||
"nan": "^2.12.1"
|
||||
}
|
||||
},
|
||||
"glob-parent": {
|
||||
"version": "3.1.0",
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
"@ng-bootstrap/ng-bootstrap": "^8.0.0",
|
||||
"bootstrap": "^4.5.0",
|
||||
"ng-bootstrap": "^1.6.3",
|
||||
"ng2-pdf-viewer": "^6.3.2",
|
||||
"ngx-cookie-service": "^10.1.1",
|
||||
"ngx-file-drop": "^10.0.0",
|
||||
"ngx-infinite-scroll": "^9.1.0",
|
||||
|
||||
@ -14,10 +14,9 @@ import { LogsComponent } from './components/manage/logs/logs.component';
|
||||
import { SettingsComponent } from './components/manage/settings/settings.component';
|
||||
import { FormsModule, ReactiveFormsModule } from '@angular/forms';
|
||||
import { DatePipe } from '@angular/common';
|
||||
import { SafePipe } from './pipes/safe.pipe';
|
||||
import { NotFoundComponent } from './components/not-found/not-found.component';
|
||||
import { CorrespondentListComponent } from './components/manage/correspondent-list/correspondent-list.component';
|
||||
import { DeleteDialogComponent } from './components/common/delete-dialog/delete-dialog.component';
|
||||
import { ConfirmDialogComponent } from './components/common/confirm-dialog/confirm-dialog.component';
|
||||
import { CorrespondentEditDialogComponent } from './components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component';
|
||||
import { TagEditDialogComponent } from './components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component';
|
||||
import { DocumentTypeEditDialogComponent } from './components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
|
||||
@ -48,10 +47,13 @@ import { SavedViewWidgetComponent } from './components/dashboard/widgets/saved-v
|
||||
import { StatisticsWidgetComponent } from './components/dashboard/widgets/statistics-widget/statistics-widget.component';
|
||||
import { UploadFileWidgetComponent } from './components/dashboard/widgets/upload-file-widget/upload-file-widget.component';
|
||||
import { WidgetFrameComponent } from './components/dashboard/widgets/widget-frame/widget-frame.component';
|
||||
import { PdfViewerModule } from 'ng2-pdf-viewer';
|
||||
import { WelcomeWidgetComponent } from './components/dashboard/widgets/welcome-widget/welcome-widget.component';
|
||||
import { YesNoPipe } from './pipes/yes-no.pipe';
|
||||
import { FileSizePipe } from './pipes/file-size.pipe';
|
||||
import { FilterPipe } from './pipes/filter.pipe';
|
||||
import { DocumentTitlePipe } from './pipes/document-title.pipe';
|
||||
import { MetadataCollapseComponent } from './components/document-detail/metadata-collapse/metadata-collapse.component';
|
||||
|
||||
@NgModule({
|
||||
declarations: [
|
||||
@ -64,10 +66,9 @@ import { FilterPipe } from './pipes/filter.pipe';
|
||||
DocumentTypeListComponent,
|
||||
LogsComponent,
|
||||
SettingsComponent,
|
||||
SafePipe,
|
||||
NotFoundComponent,
|
||||
CorrespondentEditDialogComponent,
|
||||
DeleteDialogComponent,
|
||||
ConfirmDialogComponent,
|
||||
TagEditDialogComponent,
|
||||
DocumentTypeEditDialogComponent,
|
||||
TagComponent,
|
||||
@ -97,6 +98,8 @@ import { FilterPipe } from './pipes/filter.pipe';
|
||||
YesNoPipe,
|
||||
FileSizePipe,
|
||||
FilterPipe
|
||||
DocumentTitlePipe,
|
||||
MetadataCollapseComponent
|
||||
],
|
||||
imports: [
|
||||
BrowserModule,
|
||||
@ -106,7 +109,8 @@ import { FilterPipe } from './pipes/filter.pipe';
|
||||
FormsModule,
|
||||
ReactiveFormsModule,
|
||||
NgxFileDropModule,
|
||||
InfiniteScrollModule
|
||||
InfiniteScrollModule,
|
||||
PdfViewerModule
|
||||
],
|
||||
providers: [
|
||||
DatePipe,
|
||||
|
||||
@ -132,7 +132,7 @@
|
||||
</h6>
|
||||
<ul class="nav flex-column mb-2">
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="https://paperless-ng.readthedocs.io/en/latest/">
|
||||
<a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://paperless-ng.readthedocs.io/en/latest/">
|
||||
<svg class="sidebaricon" fill="currentColor">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#question-circle"/>
|
||||
</svg>
|
||||
@ -140,7 +140,7 @@
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="https://github.com/jonaswinkler/paperless-ng">
|
||||
<a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://github.com/jonaswinkler/paperless-ng">
|
||||
<svg class="sidebaricon" fill="currentColor">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#link"/>
|
||||
</svg>
|
||||
|
||||
@ -5,10 +5,10 @@
|
||||
</button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p><b>{{message}}</b></p>
|
||||
<p *ngIf="message2">{{message2}}</p>
|
||||
<p *ngIf="messageBold"><b>{{messageBold}}</b></p>
|
||||
<p *ngIf="message">{{message}}</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-dark" (click)="cancelClicked()">Cancel</button>
|
||||
<button type="button" class="btn btn-danger" (click)="deleteClicked.emit()">Delete</button>
|
||||
<button type="button" class="btn" [class]="btnClass" (click)="confirmClicked.emit()">{{btnCaption}}</button>
|
||||
</div>
|
||||
@ -1,20 +1,20 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { DeleteDialogComponent } from './delete-dialog.component';
|
||||
import { ConfirmDialogComponent } from './confirm-dialog.component';
|
||||
|
||||
describe('DeleteDialogComponent', () => {
|
||||
let component: DeleteDialogComponent;
|
||||
let fixture: ComponentFixture<DeleteDialogComponent>;
|
||||
describe('ConfirmDialogComponent', () => {
|
||||
let component: ConfirmDialogComponent;
|
||||
let fixture: ComponentFixture<ConfirmDialogComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
declarations: [ DeleteDialogComponent ]
|
||||
declarations: [ ConfirmDialogComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(DeleteDialogComponent);
|
||||
fixture = TestBed.createComponent(ConfirmDialogComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
@ -0,0 +1,37 @@
|
||||
import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
|
||||
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
|
||||
|
||||
@Component({
|
||||
selector: 'app-confirm-dialog',
|
||||
templateUrl: './confirm-dialog.component.html',
|
||||
styleUrls: ['./confirm-dialog.component.scss']
|
||||
})
|
||||
export class ConfirmDialogComponent implements OnInit {
|
||||
|
||||
constructor(public activeModal: NgbActiveModal) { }
|
||||
|
||||
@Output()
|
||||
public confirmClicked = new EventEmitter()
|
||||
|
||||
@Input()
|
||||
title = "Confirmation"
|
||||
|
||||
@Input()
|
||||
messageBold
|
||||
|
||||
@Input()
|
||||
message
|
||||
|
||||
@Input()
|
||||
btnClass = "btn-primary"
|
||||
|
||||
@Input()
|
||||
btnCaption = "Confirm"
|
||||
|
||||
ngOnInit(): void {
|
||||
}
|
||||
|
||||
cancelClicked() {
|
||||
this.activeModal.close()
|
||||
}
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
|
||||
import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
|
||||
|
||||
@Component({
|
||||
selector: 'app-delete-dialog',
|
||||
templateUrl: './delete-dialog.component.html',
|
||||
styleUrls: ['./delete-dialog.component.scss']
|
||||
})
|
||||
export class DeleteDialogComponent implements OnInit {
|
||||
|
||||
constructor(public activeModal: NgbActiveModal) { }
|
||||
|
||||
@Output()
|
||||
public deleteClicked = new EventEmitter()
|
||||
|
||||
@Input()
|
||||
title = "Delete confirmation"
|
||||
|
||||
@Input()
|
||||
message = "Do you really want to delete this?"
|
||||
|
||||
@Input()
|
||||
message2
|
||||
|
||||
ngOnInit(): void {
|
||||
}
|
||||
|
||||
cancelClicked() {
|
||||
this.activeModal.close()
|
||||
}
|
||||
}
|
||||
@ -8,7 +8,7 @@
|
||||
|
||||
<div class="input-group-append" ngbDropdown placement="top-right">
|
||||
<button class="btn btn-outline-secondary" type="button" ngbDropdownToggle></button>
|
||||
<div ngbDropdownMenu class="scrollable-menu">
|
||||
<div ngbDropdownMenu class="scrollable-menu shadow">
|
||||
<button type="button" *ngFor="let tag of tags" ngbDropdownItem (click)="addTag(tag.id)">
|
||||
<app-tag [tag]="tag"></app-tag>
|
||||
</button>
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
<div class="card mb-3 shadow">
|
||||
<div class="card mb-3 shadow-sm">
|
||||
<div class="card-header">
|
||||
<div class="d-flex justify-content-between align-items-center">
|
||||
<h5 class="card-title mb-0">{{title}}</h5>
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
|
||||
<div class="btn-group" ngbDropdown role="group" *ngIf="metadata?.has_archive_version">
|
||||
<button class="btn btn-sm btn-outline-primary dropdown-toggle-split" ngbDropdownToggle></button>
|
||||
<div class="dropdown-menu" ngbDropdownMenu>
|
||||
<div class="dropdown-menu shadow" ngbDropdownMenu>
|
||||
<a ngbDropdownItem [href]="downloadOriginalUrl">Download original</a>
|
||||
</div>
|
||||
</div>
|
||||
@ -35,7 +35,7 @@
|
||||
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl">
|
||||
<div class="col mb-4">
|
||||
|
||||
<form [formGroup]='documentForm' (ngSubmit)="save()">
|
||||
|
||||
@ -110,53 +110,8 @@
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h6 *ngIf="metadata?.original_metadata.length > 0">
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm mr-2"
|
||||
(click)="expandOriginalMetadata = !expandOriginalMetadata" aria-controls="collapseExample">
|
||||
<svg class="buttonicon" fill="currentColor" *ngIf="!expandOriginalMetadata">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#caret-down" />
|
||||
</svg>
|
||||
<svg class="buttonicon" fill="currentColor" *ngIf="expandOriginalMetadata">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#caret-up" />
|
||||
</svg>
|
||||
</button>
|
||||
Original document metadata
|
||||
</h6>
|
||||
|
||||
<div #collapse="ngbCollapse" [(ngbCollapse)]="!expandOriginalMetadata">
|
||||
<table class="table table-borderless">
|
||||
<tbody>
|
||||
<tr *ngFor="let m of metadata?.original_metadata">
|
||||
<td>{{m.prefix}}:{{m.key}}</td>
|
||||
<td>{{m.value}}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<h6 *ngIf="metadata?.has_archive_version && metadata?.archive_metadata.length > 0">
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm mr-2"
|
||||
(click)="expandArchivedMetadata = !expandArchivedMetadata" aria-controls="collapseExample">
|
||||
<svg class="buttonicon" fill="currentColor" *ngIf="!expandArchivedMetadata">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#caret-down" />
|
||||
</svg>
|
||||
<svg class="buttonicon" fill="currentColor" *ngIf="expandArchivedMetadata">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#caret-up" />
|
||||
</svg>
|
||||
</button>
|
||||
Archived document metadata
|
||||
</h6>
|
||||
|
||||
<div #collapse="ngbCollapse" [(ngbCollapse)]="!expandArchivedMetadata">
|
||||
<table class="table table-borderless">
|
||||
<tbody>
|
||||
<tr *ngFor="let m of metadata?.archive_metadata">
|
||||
<td>{{m.prefix}}:{{m.key}}</td>
|
||||
<td>{{m.value}}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<app-metadata-collapse title="Original document metadata" [metadata]="metadata.original_metadata" *ngIf="metadata?.original_metadata.length > 0"></app-metadata-collapse>
|
||||
<app-metadata-collapse title="Archived document metadata" [metadata]="metadata.archive_metadata" *ngIf="metadata?.archive_metadata.length > 0"></app-metadata-collapse>
|
||||
|
||||
</ng-template>
|
||||
</li>
|
||||
@ -171,11 +126,9 @@
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="col-xl d-none d-xl-block document-preview">
|
||||
<object [data]="previewUrl | safe" type="application/pdf" width="100%" height="100%">
|
||||
<p>Your browser does not support PDFs.
|
||||
<a href="previewUrl">Download the PDF</a>.</p>
|
||||
</object>
|
||||
|
||||
<div class="col-md-6 col-xl-8 mb-3">
|
||||
<div class="pdf-viewer-container" *ngIf="getContentType() == 'application/pdf'">
|
||||
<pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true"></pdf-viewer>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
.document-preview {
|
||||
height: calc(100vh - 180px);
|
||||
.pdf-viewer-container {
|
||||
height: calc(100vh - 160px);
|
||||
top: 70px;
|
||||
position: sticky;
|
||||
}
|
||||
background-color: gray;
|
||||
}
|
||||
|
||||
@ -13,7 +13,7 @@ import { CorrespondentService } from 'src/app/services/rest/correspondent.servic
|
||||
import { DocumentTypeService } from 'src/app/services/rest/document-type.service';
|
||||
import { DocumentService } from 'src/app/services/rest/document.service';
|
||||
import { environment } from 'src/environments/environment';
|
||||
import { DeleteDialogComponent } from '../common/delete-dialog/delete-dialog.component';
|
||||
import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component';
|
||||
import { CorrespondentEditDialogComponent } from '../manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component';
|
||||
import { DocumentTypeEditDialogComponent } from '../manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
|
||||
|
||||
@ -59,6 +59,10 @@ export class DocumentDetailComponent implements OnInit {
|
||||
private documentListViewService: DocumentListViewService,
|
||||
private titleService: Title) { }
|
||||
|
||||
getContentType() {
|
||||
return this.metadata?.has_archive_version ? 'application/pdf' : this.metadata?.original_mime_type
|
||||
}
|
||||
|
||||
ngOnInit(): void {
|
||||
this.documentForm.valueChanges.subscribe(wow => {
|
||||
Object.assign(this.document, this.documentForm.value)
|
||||
@ -151,10 +155,13 @@ export class DocumentDetailComponent implements OnInit {
|
||||
}
|
||||
|
||||
delete() {
|
||||
let modal = this.modalService.open(DeleteDialogComponent, {backdrop: 'static'})
|
||||
modal.componentInstance.message = `Do you really want to delete document '${this.document.title}'?`
|
||||
modal.componentInstance.message2 = `The files for this document will be deleted permanently. This operation cannot be undone.`
|
||||
modal.componentInstance.deleteClicked.subscribe(() => {
|
||||
let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'})
|
||||
modal.componentInstance.title = "Confirm delete"
|
||||
modal.componentInstance.messageBold = `Do you really want to delete document '${this.document.title}'?`
|
||||
modal.componentInstance.message = `The files for this document will be deleted permanently. This operation cannot be undone.`
|
||||
modal.componentInstance.btnClass = "btn-danger"
|
||||
modal.componentInstance.btnCaption = "Delete document"
|
||||
modal.componentInstance.confirmClicked.subscribe(() => {
|
||||
this.documentsService.delete(this.document).subscribe(() => {
|
||||
modal.close()
|
||||
this.close()
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
<h6>
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm mr-2"
|
||||
(click)="expand = !expand">
|
||||
<svg class="buttonicon" fill="currentColor" *ngIf="!expand">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#caret-down" />
|
||||
</svg>
|
||||
<svg class="buttonicon" fill="currentColor" *ngIf="expand">
|
||||
<use xlink:href="assets/bootstrap-icons.svg#caret-up" />
|
||||
</svg>
|
||||
</button>
|
||||
{{title}}
|
||||
</h6>
|
||||
|
||||
<div #collapse="ngbCollapse" [(ngbCollapse)]="!expand">
|
||||
<table class="table table-borderless">
|
||||
<tbody>
|
||||
<tr *ngFor="let m of metadata">
|
||||
<td>{{m.prefix}}:{{m.key}}</td>
|
||||
<td>{{m.value}}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@ -0,0 +1,25 @@
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
|
||||
import { MetadataCollapseComponent } from './metadata-collapse.component';
|
||||
|
||||
describe('MetadataCollapseComponent', () => {
|
||||
let component: MetadataCollapseComponent;
|
||||
let fixture: ComponentFixture<MetadataCollapseComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
declarations: [ MetadataCollapseComponent ]
|
||||
})
|
||||
.compileComponents();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
fixture = TestBed.createComponent(MetadataCollapseComponent);
|
||||
component = fixture.componentInstance;
|
||||
fixture.detectChanges();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
expect(component).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,23 @@
|
||||
import { Component, Input, OnInit } from '@angular/core';
|
||||
|
||||
@Component({
|
||||
selector: 'app-metadata-collapse',
|
||||
templateUrl: './metadata-collapse.component.html',
|
||||
styleUrls: ['./metadata-collapse.component.scss']
|
||||
})
|
||||
export class MetadataCollapseComponent implements OnInit {
|
||||
|
||||
constructor() { }
|
||||
|
||||
expand = false
|
||||
|
||||
@Input()
|
||||
metadata
|
||||
|
||||
@Input()
|
||||
title = "Metadata"
|
||||
|
||||
ngOnInit(): void {
|
||||
}
|
||||
|
||||
}
|
||||
@ -12,7 +12,7 @@
|
||||
<a *ngIf="clickCorrespondent.observers.length ; else nolink" [routerLink]="" title="Filter by correspondent" (click)="clickCorrespondent.emit(document.correspondent)" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>
|
||||
<ng-template #nolink>{{(document.correspondent$ | async)?.name}}</ng-template>:
|
||||
</ng-container>
|
||||
{{document.title}}
|
||||
{{document.title | documentTitle}}
|
||||
<app-tag [tag]="t" linkTitle="Filter by tag" *ngFor="let t of document.tags$ | async" class="ml-1" (click)="clickTag.emit(t.id)" [clickable]="clickTag.observers.length"></app-tag>
|
||||
</h5>
|
||||
<h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archive_serial_number}}</h5>
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
<div class="col p-2 h-100" style="width: 16rem;">
|
||||
<div class="card h-100 shadow-sm">
|
||||
<div class=" border-bottom pr-1">
|
||||
<div class="border-bottom">
|
||||
<img class="card-img doc-img" [src]="getThumbUrl()">
|
||||
<div style="top: 0; right: 0; font-size: large" class="text-right position-absolute mr-1">
|
||||
<div *ngFor="let t of getTagsLimited$() | async">
|
||||
@ -17,7 +17,7 @@
|
||||
<ng-container *ngIf="document.correspondent">
|
||||
<a [routerLink]="" title="Filter by correspondent" (click)="clickCorrespondent.emit(document.correspondent)" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>:
|
||||
</ng-container>
|
||||
{{document.title}}
|
||||
{{document.title | documentTitle}}
|
||||
</p>
|
||||
</div>
|
||||
<div class="card-footer">
|
||||
|
||||
@ -24,7 +24,7 @@
|
||||
<div class="btn-group btn-group-toggle ml-2" ngbRadioGroup [(ngModel)]="list.sortDirection">
|
||||
<div ngbDropdown class="btn-group">
|
||||
<button class="btn btn-outline-primary btn-sm" id="dropdownBasic1" ngbDropdownToggle>Sort by</button>
|
||||
<div ngbDropdownMenu aria-labelledby="dropdownBasic1">
|
||||
<div ngbDropdownMenu aria-labelledby="dropdownBasic1" class="shadow">
|
||||
<button *ngFor="let f of getSortFields()" ngbDropdownItem (click)="list.sortField = f.field"
|
||||
[class.active]="list.sortField == f.field">{{f.name}}</button>
|
||||
</div>
|
||||
@ -79,7 +79,7 @@
|
||||
</app-document-card-large>
|
||||
</div>
|
||||
|
||||
<table class="table table-sm border shadow" *ngIf="displayMode == 'details'">
|
||||
<table class="table table-sm border shadow-sm" *ngIf="displayMode == 'details'">
|
||||
<thead>
|
||||
<th class="d-none d-lg-table-cell">ASN</th>
|
||||
<th class="d-none d-md-table-cell">Correspondent</th>
|
||||
@ -99,7 +99,7 @@
|
||||
</ng-container>
|
||||
</td>
|
||||
<td>
|
||||
<a routerLink="/documents/{{d.id}}" title="Edit document" style="overflow-wrap: anywhere;">{{d.title}}</a>
|
||||
<a routerLink="/documents/{{d.id}}" title="Edit document" style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
|
||||
<app-tag [tag]="t" *ngFor="let t of d.tags$ | async" class="ml-1" clickable="true" linkTitle="Filter by tag" (click)="clickTag(t)"></app-tag>
|
||||
</td>
|
||||
<td class="d-none d-xl-table-cell">
|
||||
|
||||
@ -4,7 +4,7 @@ import { MatchingModel, MATCHING_ALGORITHMS, MATCH_AUTO } from 'src/app/data/mat
|
||||
import { ObjectWithId } from 'src/app/data/object-with-id';
|
||||
import { SortableDirective, SortEvent } from 'src/app/directives/sortable.directive';
|
||||
import { AbstractPaperlessService } from 'src/app/services/rest/abstract-paperless-service';
|
||||
import { DeleteDialogComponent } from '../../common/delete-dialog/delete-dialog.component';
|
||||
import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component';
|
||||
|
||||
@Directive()
|
||||
export abstract class GenericListComponent<T extends ObjectWithId> implements OnInit {
|
||||
@ -88,10 +88,13 @@ export abstract class GenericListComponent<T extends ObjectWithId> implements On
|
||||
}
|
||||
|
||||
openDeleteDialog(object: T) {
|
||||
var activeModal = this.modalService.open(DeleteDialogComponent, {backdrop: 'static'})
|
||||
activeModal.componentInstance.message = `Do you really want to delete ${this.getObjectName(object)}?`
|
||||
activeModal.componentInstance.message2 = "Associated documents will not be deleted."
|
||||
activeModal.componentInstance.deleteClicked.subscribe(() => {
|
||||
var activeModal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'})
|
||||
activeModal.componentInstance.title = "Confirm delete"
|
||||
activeModal.componentInstance.messageBold = `Do you really want to delete ${this.getObjectName(object)}?`
|
||||
activeModal.componentInstance.message = "Associated documents will not be deleted."
|
||||
activeModal.componentInstance.btnClass = "btn-danger"
|
||||
activeModal.componentInstance.btnCaption = "Delete"
|
||||
activeModal.componentInstance.confirmPressed.subscribe(() => {
|
||||
this.service.delete(object).subscribe(_ => {
|
||||
activeModal.close()
|
||||
this.reloadData()
|
||||
|
||||
8
src-ui/src/app/pipes/document-title.pipe.spec.ts
Normal file
8
src-ui/src/app/pipes/document-title.pipe.spec.ts
Normal file
@ -0,0 +1,8 @@
|
||||
import { DocumentTitlePipe } from './document-title.pipe';
|
||||
|
||||
describe('DocumentTitlePipe', () => {
|
||||
it('create an instance', () => {
|
||||
const pipe = new DocumentTitlePipe();
|
||||
expect(pipe).toBeTruthy();
|
||||
});
|
||||
});
|
||||
16
src-ui/src/app/pipes/document-title.pipe.ts
Normal file
16
src-ui/src/app/pipes/document-title.pipe.ts
Normal file
@ -0,0 +1,16 @@
|
||||
import { Pipe, PipeTransform } from '@angular/core';
|
||||
|
||||
@Pipe({
|
||||
name: 'documentTitle'
|
||||
})
|
||||
export class DocumentTitlePipe implements PipeTransform {
|
||||
|
||||
transform(value: string): unknown {
|
||||
if (value) {
|
||||
return value
|
||||
} else {
|
||||
return "(no title)"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
import { SafePipe } from './safe.pipe';
|
||||
|
||||
describe('SafePipe', () => {
|
||||
it('create an instance', () => {
|
||||
const pipe = new SafePipe();
|
||||
expect(pipe).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@ -1,19 +0,0 @@
|
||||
import { Pipe, PipeTransform } from '@angular/core';
|
||||
import { DomSanitizer } from '@angular/platform-browser';
|
||||
|
||||
@Pipe({
|
||||
name: 'safe'
|
||||
})
|
||||
export class SafePipe implements PipeTransform {
|
||||
|
||||
constructor(private sanitizer: DomSanitizer) { }
|
||||
|
||||
transform(url) {
|
||||
if (url == null) {
|
||||
return this.sanitizer.bypassSecurityTrustResourceUrl("")
|
||||
} else {
|
||||
return this.sanitizer.bypassSecurityTrustResourceUrl(url);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -17,8 +17,6 @@ class CorrespondentAdmin(admin.ModelAdmin):
|
||||
list_filter = ("matching_algorithm",)
|
||||
list_editable = ("match", "matching_algorithm")
|
||||
|
||||
readonly_fields = ("slug",)
|
||||
|
||||
|
||||
class TagAdmin(admin.ModelAdmin):
|
||||
|
||||
@ -31,8 +29,6 @@ class TagAdmin(admin.ModelAdmin):
|
||||
list_filter = ("colour", "matching_algorithm")
|
||||
list_editable = ("colour", "match", "matching_algorithm")
|
||||
|
||||
readonly_fields = ("slug", )
|
||||
|
||||
|
||||
class DocumentTypeAdmin(admin.ModelAdmin):
|
||||
|
||||
@ -44,8 +40,6 @@ class DocumentTypeAdmin(admin.ModelAdmin):
|
||||
list_filter = ("matching_algorithm",)
|
||||
list_editable = ("match", "matching_algorithm")
|
||||
|
||||
readonly_fields = ("slug",)
|
||||
|
||||
|
||||
class DocumentAdmin(admin.ModelAdmin):
|
||||
|
||||
@ -106,7 +100,7 @@ class DocumentAdmin(admin.ModelAdmin):
|
||||
for tag in obj.tags.all():
|
||||
r += self._html_tag(
|
||||
"span",
|
||||
tag.slug + ", "
|
||||
tag.name + ", "
|
||||
)
|
||||
return r
|
||||
|
||||
|
||||
@ -248,7 +248,7 @@ class Consumer(LoggingMixin):
|
||||
with open(self.path, "rb") as f:
|
||||
document = Document.objects.create(
|
||||
correspondent=file_info.correspondent,
|
||||
title=file_info.title,
|
||||
title=(self.override_title or file_info.title)[:127],
|
||||
content=text,
|
||||
mime_type=mime_type,
|
||||
checksum=hashlib.md5(f.read()).hexdigest(),
|
||||
@ -259,18 +259,17 @@ class Consumer(LoggingMixin):
|
||||
|
||||
relevant_tags = set(file_info.tags)
|
||||
if relevant_tags:
|
||||
tag_names = ", ".join([t.slug for t in relevant_tags])
|
||||
tag_names = ", ".join([t.name for t in relevant_tags])
|
||||
self.log("debug", "Tagging with {}".format(tag_names))
|
||||
document.tags.add(*relevant_tags)
|
||||
|
||||
self.apply_overrides(document)
|
||||
|
||||
document.save()
|
||||
|
||||
return document
|
||||
|
||||
def apply_overrides(self, document):
|
||||
if self.override_title:
|
||||
document.title = self.override_title
|
||||
|
||||
if self.override_correspondent_id:
|
||||
document.correspondent = Correspondent.objects.get(
|
||||
pk=self.override_correspondent_id)
|
||||
|
||||
@ -51,7 +51,6 @@ class TagsFilter(Filter):
|
||||
return qs
|
||||
|
||||
for tag_id in tag_ids:
|
||||
print(self.exclude, tag_id)
|
||||
if self.exclude:
|
||||
qs = qs.exclude(tags__id=tag_id)
|
||||
else:
|
||||
|
||||
@ -82,7 +82,8 @@ class Command(BaseCommand):
|
||||
with open(document.thumbnail_path, "wb") as f:
|
||||
f.write(raw_thumb)
|
||||
|
||||
document.save(update_fields=("storage_type", "filename"))
|
||||
Document.objects.filter(id=document.id).update(
|
||||
storage_type=document.storage_type, filename=document.filename)
|
||||
|
||||
for path in old_paths:
|
||||
os.unlink(path)
|
||||
|
||||
@ -29,10 +29,9 @@ def _tags_from_path(filepath):
|
||||
path_parts = Path(filepath).relative_to(
|
||||
settings.CONSUMPTION_DIR).parent.parts
|
||||
for part in path_parts:
|
||||
tag_ids.add(Tag.objects.get_or_create(
|
||||
slug=slugify(part),
|
||||
defaults={"name": part},
|
||||
)[0].pk)
|
||||
tag_ids.add(Tag.objects.get_or_create(name__iexact=part, defaults={
|
||||
"name": part
|
||||
})[0].pk)
|
||||
|
||||
return tag_ids
|
||||
|
||||
|
||||
@ -1,4 +1,8 @@
|
||||
import logging
|
||||
|
||||
import tqdm
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db.models.signals import post_save
|
||||
|
||||
from documents.models import Document
|
||||
from ...mixins import Renderable
|
||||
@ -18,6 +22,7 @@ class Command(Renderable, BaseCommand):
|
||||
|
||||
self.verbosity = options["verbosity"]
|
||||
|
||||
for document in Document.objects.all():
|
||||
# Saving the document again will generate a new filename and rename
|
||||
document.save()
|
||||
logging.getLogger().handlers[0].level = logging.ERROR
|
||||
|
||||
for document in tqdm.tqdm(Document.objects.all()):
|
||||
post_save.send(Document, instance=document)
|
||||
|
||||
25
src/documents/migrations/1006_auto_20201208_2209.py
Normal file
25
src/documents/migrations/1006_auto_20201208_2209.py
Normal file
@ -0,0 +1,25 @@
|
||||
# Generated by Django 3.1.4 on 2020-12-08 22:09
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('documents', '1005_checksums'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='correspondent',
|
||||
name='slug',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='documenttype',
|
||||
name='slug',
|
||||
),
|
||||
migrations.RemoveField(
|
||||
model_name='tag',
|
||||
name='slug',
|
||||
),
|
||||
]
|
||||
@ -36,7 +36,6 @@ class MatchingModel(models.Model):
|
||||
)
|
||||
|
||||
name = models.CharField(max_length=128, unique=True)
|
||||
slug = models.SlugField(blank=True, editable=False)
|
||||
|
||||
match = models.CharField(max_length=256, blank=True)
|
||||
matching_algorithm = models.PositiveIntegerField(
|
||||
@ -69,7 +68,6 @@ class MatchingModel(models.Model):
|
||||
def save(self, *args, **kwargs):
|
||||
|
||||
self.match = self.match.lower()
|
||||
self.slug = slugify(self.name)
|
||||
|
||||
models.Model.save(self, *args, **kwargs)
|
||||
|
||||
@ -384,9 +382,7 @@ class FileInfo:
|
||||
def _get_correspondent(cls, name):
|
||||
if not name:
|
||||
return None
|
||||
return Correspondent.objects.get_or_create(name=name, defaults={
|
||||
"slug": slugify(name)
|
||||
})[0]
|
||||
return Correspondent.objects.get_or_create(name=name)[0]
|
||||
|
||||
@classmethod
|
||||
def _get_title(cls, title):
|
||||
@ -396,10 +392,7 @@ class FileInfo:
|
||||
def _get_tags(cls, tags):
|
||||
r = []
|
||||
for t in tags.split(","):
|
||||
r.append(Tag.objects.get_or_create(
|
||||
slug=slugify(t),
|
||||
defaults={"name": t}
|
||||
)[0])
|
||||
r.append(Tag.objects.get_or_create(name=t)[0])
|
||||
return tuple(r)
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -210,6 +210,7 @@ class DocumentParser(LoggingMixin):
|
||||
def __init__(self, logging_group):
|
||||
super().__init__()
|
||||
self.logging_group = logging_group
|
||||
os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
|
||||
self.tempdir = tempfile.mkdtemp(
|
||||
prefix="paperless-", dir=settings.SCRATCH_DIR)
|
||||
|
||||
@ -217,6 +218,9 @@ class DocumentParser(LoggingMixin):
|
||||
self.text = None
|
||||
self.date = None
|
||||
|
||||
def extract_metadata(self, document_path, mime_type):
|
||||
return []
|
||||
|
||||
def parse(self, document_path, mime_type):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@ -46,6 +46,10 @@ def check_sanity():
|
||||
for f in files:
|
||||
present_files.append(os.path.normpath(os.path.join(root, f)))
|
||||
|
||||
lockfile = os.path.normpath(settings.MEDIA_LOCK)
|
||||
if lockfile in present_files:
|
||||
present_files.remove(lockfile)
|
||||
|
||||
for doc in Document.objects.all():
|
||||
# Check sanity of the thumbnail
|
||||
if not os.path.isfile(doc.thumbnail_path):
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import magic
|
||||
from pathvalidate import validate_filename, ValidationError
|
||||
from django.utils.text import slugify
|
||||
from rest_framework import serializers
|
||||
from rest_framework.fields import SerializerMethodField
|
||||
|
||||
@ -7,12 +7,16 @@ from .models import Correspondent, Tag, Document, Log, DocumentType
|
||||
from .parsers import is_mime_type_supported
|
||||
|
||||
|
||||
class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
|
||||
class CorrespondentSerializer(serializers.ModelSerializer):
|
||||
|
||||
document_count = serializers.IntegerField(read_only=True)
|
||||
|
||||
last_correspondence = serializers.DateTimeField(read_only=True)
|
||||
|
||||
def get_slug(self, obj):
|
||||
return slugify(obj.name)
|
||||
slug = SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Correspondent
|
||||
fields = (
|
||||
@ -27,10 +31,14 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
|
||||
)
|
||||
|
||||
|
||||
class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
|
||||
class DocumentTypeSerializer(serializers.ModelSerializer):
|
||||
|
||||
document_count = serializers.IntegerField(read_only=True)
|
||||
|
||||
def get_slug(self, obj):
|
||||
return slugify(obj.name)
|
||||
slug = SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = DocumentType
|
||||
fields = (
|
||||
@ -44,10 +52,14 @@ class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
|
||||
)
|
||||
|
||||
|
||||
class TagSerializer(serializers.HyperlinkedModelSerializer):
|
||||
class TagSerializer(serializers.ModelSerializer):
|
||||
|
||||
document_count = serializers.IntegerField(read_only=True)
|
||||
|
||||
def get_slug(self, obj):
|
||||
return slugify(obj.name)
|
||||
slug = SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Tag
|
||||
fields = (
|
||||
@ -166,12 +178,6 @@ class PostDocumentSerializer(serializers.Serializer):
|
||||
)
|
||||
|
||||
def validate_document(self, document):
|
||||
|
||||
try:
|
||||
validate_filename(document.name)
|
||||
except ValidationError:
|
||||
raise serializers.ValidationError("Invalid filename.")
|
||||
|
||||
document_data = document.file.read()
|
||||
mime_type = magic.from_buffer(document_data, mime=True)
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@ from django.contrib.admin.models import ADDITION, LogEntry
|
||||
from django.contrib.auth.models import User
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.db import models, DatabaseError
|
||||
from django.db.models import Q
|
||||
from django.dispatch import receiver
|
||||
from django.utils import timezone
|
||||
from filelock import FileLock
|
||||
@ -121,11 +122,14 @@ def set_tags(sender,
|
||||
classifier=None,
|
||||
replace=False,
|
||||
**kwargs):
|
||||
|
||||
if replace:
|
||||
document.tags.clear()
|
||||
current_tags = set([])
|
||||
else:
|
||||
current_tags = set(document.tags.all())
|
||||
Document.tags.through.objects.filter(document=document).exclude(
|
||||
Q(tag__is_inbox_tag=True)).exclude(
|
||||
Q(tag__match="") & ~Q(tag__matching_algorithm=Tag.MATCH_AUTO)
|
||||
).delete()
|
||||
|
||||
current_tags = set(document.tags.all())
|
||||
|
||||
matched_tags = matching.match_tags(document.content, classifier)
|
||||
|
||||
@ -136,7 +140,7 @@ def set_tags(sender,
|
||||
|
||||
message = 'Tagging "{}" with "{}"'
|
||||
logger(
|
||||
message.format(document, ", ".join([t.slug for t in relevant_tags])),
|
||||
message.format(document, ", ".join([t.name for t in relevant_tags])),
|
||||
logging_group
|
||||
)
|
||||
|
||||
@ -165,35 +169,36 @@ def run_post_consume_script(sender, document, **kwargs):
|
||||
reverse("document-download", kwargs={"pk": document.pk}),
|
||||
reverse("document-thumb", kwargs={"pk": document.pk}),
|
||||
str(document.correspondent),
|
||||
str(",".join(document.tags.all().values_list("slug", flat=True)))
|
||||
str(",".join(document.tags.all().values_list("name", flat=True)))
|
||||
)).wait()
|
||||
|
||||
|
||||
@receiver(models.signals.post_delete, sender=Document)
|
||||
def cleanup_document_deletion(sender, instance, using, **kwargs):
|
||||
for f in (instance.source_path,
|
||||
instance.archive_path,
|
||||
instance.thumbnail_path):
|
||||
if os.path.isfile(f):
|
||||
try:
|
||||
os.unlink(f)
|
||||
logging.getLogger(__name__).debug(
|
||||
f"Deleted file {f}.")
|
||||
except OSError as e:
|
||||
logging.getLogger(__name__).warning(
|
||||
f"While deleting document {str(instance)}, the file "
|
||||
f"{f} could not be deleted: {e}"
|
||||
)
|
||||
with FileLock(settings.MEDIA_LOCK):
|
||||
for f in (instance.source_path,
|
||||
instance.archive_path,
|
||||
instance.thumbnail_path):
|
||||
if os.path.isfile(f):
|
||||
try:
|
||||
os.unlink(f)
|
||||
logging.getLogger(__name__).debug(
|
||||
f"Deleted file {f}.")
|
||||
except OSError as e:
|
||||
logging.getLogger(__name__).warning(
|
||||
f"While deleting document {str(instance)}, the file "
|
||||
f"{f} could not be deleted: {e}"
|
||||
)
|
||||
|
||||
delete_empty_directories(
|
||||
os.path.dirname(instance.source_path),
|
||||
root=settings.ORIGINALS_DIR
|
||||
)
|
||||
delete_empty_directories(
|
||||
os.path.dirname(instance.source_path),
|
||||
root=settings.ORIGINALS_DIR
|
||||
)
|
||||
|
||||
delete_empty_directories(
|
||||
os.path.dirname(instance.archive_path),
|
||||
root=settings.ARCHIVE_DIR
|
||||
)
|
||||
delete_empty_directories(
|
||||
os.path.dirname(instance.archive_path),
|
||||
root=settings.ARCHIVE_DIR
|
||||
)
|
||||
|
||||
|
||||
def validate_move(instance, old_path, new_path):
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import logging
|
||||
|
||||
import tqdm
|
||||
from django.conf import settings
|
||||
from whoosh.writing import AsyncWriter
|
||||
|
||||
@ -23,7 +24,7 @@ def index_reindex():
|
||||
ix = index.open_index(recreate=True)
|
||||
|
||||
with AsyncWriter(ix) as writer:
|
||||
for document in documents:
|
||||
for document in tqdm.tqdm(documents):
|
||||
index.update_document(writer, document)
|
||||
|
||||
|
||||
|
||||
@ -403,16 +403,6 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(response.status_code, 400)
|
||||
m.assert_not_called()
|
||||
|
||||
@mock.patch("documents.views.async_task")
|
||||
@mock.patch("documents.serialisers.validate_filename")
|
||||
def test_upload_invalid_filename(self, validate_filename, async_task):
|
||||
validate_filename.side_effect = ValidationError()
|
||||
with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f:
|
||||
response = self.client.post("/api/documents/post_document/", {"document": f})
|
||||
self.assertEqual(response.status_code, 400)
|
||||
|
||||
async_task.assert_not_called()
|
||||
|
||||
@mock.patch("documents.views.async_task")
|
||||
def test_upload_with_title(self, async_task):
|
||||
with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f:
|
||||
|
||||
@ -27,7 +27,7 @@ class TestAttributes(TestCase):
|
||||
|
||||
self.assertEqual(file_info.title, title, filename)
|
||||
|
||||
self.assertEqual(tuple([t.slug for t in file_info.tags]), tags, filename)
|
||||
self.assertEqual(tuple([t.name for t in file_info.tags]), tags, filename)
|
||||
|
||||
def test_guess_attributes_from_name0(self):
|
||||
self._test_guess_attributes_from_name(
|
||||
@ -188,7 +188,7 @@ class TestFieldPermutations(TestCase):
|
||||
self.assertEqual(info.tags, (), filename)
|
||||
else:
|
||||
self.assertEqual(
|
||||
[t.slug for t in info.tags], tags.split(','),
|
||||
[t.name for t in info.tags], tags.split(','),
|
||||
filename
|
||||
)
|
||||
|
||||
@ -342,8 +342,8 @@ class TestFieldPermutations(TestCase):
|
||||
info = FileInfo.from_filename(filename)
|
||||
self.assertEqual(info.title, "0001")
|
||||
self.assertEqual(len(info.tags), 2)
|
||||
self.assertEqual(info.tags[0].slug, "tag1")
|
||||
self.assertEqual(info.tags[1].slug, "tag2")
|
||||
self.assertEqual(info.tags[0].name, "tag1")
|
||||
self.assertEqual(info.tags[1].name, "tag2")
|
||||
self.assertIsNone(info.created)
|
||||
|
||||
# Complex transformation with date in replacement string
|
||||
@ -356,8 +356,8 @@ class TestFieldPermutations(TestCase):
|
||||
info = FileInfo.from_filename(filename)
|
||||
self.assertEqual(info.title, "0001")
|
||||
self.assertEqual(len(info.tags), 2)
|
||||
self.assertEqual(info.tags[0].slug, "tag1")
|
||||
self.assertEqual(info.tags[1].slug, "tag2")
|
||||
self.assertEqual(info.tags[0].name, "tag1")
|
||||
self.assertEqual(info.tags[1].name, "tag2")
|
||||
self.assertEqual(info.created.year, 2019)
|
||||
self.assertEqual(info.created.month, 9)
|
||||
self.assertEqual(info.created.day, 8)
|
||||
|
||||
@ -3,7 +3,6 @@ import hashlib
|
||||
import os
|
||||
import random
|
||||
import uuid
|
||||
from concurrent.futures.thread import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
@ -15,7 +14,6 @@ from .utils import DirectoriesMixin
|
||||
from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \
|
||||
generate_unique_filename
|
||||
from ..models import Document, Correspondent
|
||||
from ..sanity_checker import check_sanity
|
||||
|
||||
|
||||
class TestFileHandling(DirectoriesMixin, TestCase):
|
||||
@ -573,21 +571,3 @@ def run():
|
||||
for i in range(30):
|
||||
doc.title = str(random.randrange(1, 5))
|
||||
doc.save()
|
||||
|
||||
|
||||
class TestSuperMassive(DirectoriesMixin, TestCase):
|
||||
|
||||
@override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
|
||||
def test_super_massive(self):
|
||||
# try to save as many documents in parallel as possible.
|
||||
# try to make the system fail.
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as executor:
|
||||
results = [executor.submit(run) for i in range(16)]
|
||||
|
||||
for r in results:
|
||||
if r.exception():
|
||||
raise r.exception()
|
||||
|
||||
# nope, everything still good. Thank you, lockfiles.
|
||||
self.assertEqual(len(check_sanity()), 0)
|
||||
|
||||
@ -16,25 +16,23 @@ sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
|
||||
class TestArchiver(DirectoriesMixin, TestCase):
|
||||
|
||||
def make_models(self):
|
||||
self.d1 = Document.objects.create(checksum="A", title="A", content="first document", pk=1, mime_type="application/pdf")
|
||||
#self.d2 = Document.objects.create(checksum="B", title="B", content="second document")
|
||||
#self.d3 = Document.objects.create(checksum="C", title="C", content="unrelated document")
|
||||
return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf")
|
||||
|
||||
def test_archiver(self):
|
||||
|
||||
shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
|
||||
self.make_models()
|
||||
doc = self.make_models()
|
||||
shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
|
||||
|
||||
call_command('document_archiver')
|
||||
|
||||
def test_handle_document(self):
|
||||
|
||||
shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
|
||||
self.make_models()
|
||||
doc = self.make_models()
|
||||
shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
|
||||
|
||||
handle_document(self.d1.pk)
|
||||
handle_document(doc.pk)
|
||||
|
||||
doc = Document.objects.get(id=self.d1.id)
|
||||
doc = Document.objects.get(id=doc.id)
|
||||
|
||||
self.assertIsNotNone(doc.checksum)
|
||||
self.assertTrue(os.path.isfile(doc.archive_path))
|
||||
|
||||
@ -230,7 +230,7 @@ class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
|
||||
|
||||
tag_names = ("existingTag", "Space Tag")
|
||||
# Create a Tag prior to consuming a file using it in path
|
||||
tag_ids = [Tag.objects.create(name=tag_names[0]).pk,]
|
||||
tag_ids = [Tag.objects.create(name="existingtag").pk,]
|
||||
|
||||
self.t_start()
|
||||
|
||||
|
||||
@ -35,20 +35,20 @@ class TestDecryptDocuments(TestCase):
|
||||
PASSPHRASE="test"
|
||||
).enable()
|
||||
|
||||
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
|
||||
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg"))
|
||||
doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
|
||||
|
||||
Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
|
||||
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
|
||||
shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg"))
|
||||
|
||||
call_command('decrypt_documents')
|
||||
|
||||
doc = Document.objects.get(id=2)
|
||||
doc.refresh_from_db()
|
||||
|
||||
self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
|
||||
self.assertEqual(doc.filename, "0000002.pdf")
|
||||
self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf")))
|
||||
self.assertTrue(os.path.isfile(doc.source_path))
|
||||
self.assertTrue(os.path.isfile(os.path.join(thumb_dir, "0000002.png")))
|
||||
self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png")))
|
||||
self.assertTrue(os.path.isfile(doc.thumbnail_path))
|
||||
|
||||
with doc.source_file as f:
|
||||
|
||||
@ -24,13 +24,14 @@ class TestExportImport(DirectoriesMixin, TestCase):
|
||||
|
||||
file = os.path.join(self.dirs.originals_dir, "0000001.pdf")
|
||||
|
||||
Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", id=1, mime_type="application/pdf")
|
||||
Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
|
||||
Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf")
|
||||
Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
|
||||
Tag.objects.create(name="t")
|
||||
DocumentType.objects.create(name="dt")
|
||||
Correspondent.objects.create(name="c")
|
||||
|
||||
target = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, target)
|
||||
|
||||
call_command('document_exporter', target)
|
||||
|
||||
@ -66,9 +67,6 @@ class TestExportImport(DirectoriesMixin, TestCase):
|
||||
def test_export_missing_files(self):
|
||||
|
||||
target = tempfile.mkdtemp()
|
||||
Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", id=3, mime_type="application/pdf")
|
||||
self.addCleanup(shutil.rmtree, target)
|
||||
Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", mime_type="application/pdf")
|
||||
self.assertRaises(FileNotFoundError, call_command, 'document_exporter', target)
|
||||
|
||||
def test_duplicate_titles(self):
|
||||
# TODO
|
||||
pass
|
||||
|
||||
@ -14,6 +14,12 @@ class TestRetagger(DirectoriesMixin, TestCase):
|
||||
|
||||
self.tag_first = Tag.objects.create(name="tag1", match="first", matching_algorithm=Tag.MATCH_ANY)
|
||||
self.tag_second = Tag.objects.create(name="tag2", match="second", matching_algorithm=Tag.MATCH_ANY)
|
||||
self.tag_inbox = Tag.objects.create(name="test", is_inbox_tag=True)
|
||||
self.tag_no_match = Tag.objects.create(name="test2")
|
||||
|
||||
self.d3.tags.add(self.tag_inbox)
|
||||
self.d3.tags.add(self.tag_no_match)
|
||||
|
||||
|
||||
self.correspondent_first = Correspondent.objects.create(
|
||||
name="c1", match="first", matching_algorithm=Correspondent.MATCH_ANY)
|
||||
@ -38,7 +44,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
|
||||
|
||||
self.assertEqual(d_first.tags.count(), 1)
|
||||
self.assertEqual(d_second.tags.count(), 1)
|
||||
self.assertEqual(d_unrelated.tags.count(), 0)
|
||||
self.assertEqual(d_unrelated.tags.count(), 2)
|
||||
|
||||
self.assertEqual(d_first.tags.first(), self.tag_first)
|
||||
self.assertEqual(d_second.tags.first(), self.tag_second)
|
||||
@ -56,3 +62,17 @@ class TestRetagger(DirectoriesMixin, TestCase):
|
||||
|
||||
self.assertEqual(d_first.correspondent, self.correspondent_first)
|
||||
self.assertEqual(d_second.correspondent, self.correspondent_second)
|
||||
|
||||
def test_overwrite_preserve_inbox(self):
|
||||
self.d1.tags.add(self.tag_second)
|
||||
|
||||
call_command('document_retagger', '--tags', '--overwrite')
|
||||
|
||||
d_first, d_second, d_unrelated = self.get_updated_docs()
|
||||
|
||||
self.assertIsNotNone(Tag.objects.get(id=self.tag_second.id))
|
||||
|
||||
self.assertCountEqual([tag.id for tag in d_first.tags.all()], [self.tag_first.id])
|
||||
self.assertCountEqual([tag.id for tag in d_second.tags.all()], [self.tag_second.id])
|
||||
self.assertCountEqual([tag.id for tag in d_unrelated.tags.all()], [self.tag_inbox.id, self.tag_no_match.id])
|
||||
|
||||
|
||||
@ -1,11 +1,8 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from time import mktime
|
||||
|
||||
import pikepdf
|
||||
from django.conf import settings
|
||||
from django.db.models import Count, Max
|
||||
from django.http import HttpResponse, HttpResponseBadRequest, Http404
|
||||
@ -42,6 +39,7 @@ from .filters import (
|
||||
LogFilterSet
|
||||
)
|
||||
from .models import Correspondent, Document, Log, Tag, DocumentType
|
||||
from .parsers import get_parser_class_for_mime_type
|
||||
from .serialisers import (
|
||||
CorrespondentSerializer,
|
||||
DocumentSerializer,
|
||||
@ -163,34 +161,16 @@ class DocumentViewSet(RetrieveModelMixin,
|
||||
disposition, filename)
|
||||
return response
|
||||
|
||||
def get_metadata(self, file, type):
|
||||
def get_metadata(self, file, mime_type):
|
||||
if not os.path.isfile(file):
|
||||
return None
|
||||
|
||||
namespace_pattern = re.compile(r"\{(.*)\}(.*)")
|
||||
|
||||
result = []
|
||||
if type == 'application/pdf':
|
||||
pdf = pikepdf.open(file)
|
||||
meta = pdf.open_metadata()
|
||||
for key, value in meta.items():
|
||||
if isinstance(value, list):
|
||||
value = " ".join([str(e) for e in value])
|
||||
value = str(value)
|
||||
try:
|
||||
m = namespace_pattern.match(key)
|
||||
result.append({
|
||||
"namespace": m.group(1),
|
||||
"prefix": meta.REVERSE_NS[m.group(1)],
|
||||
"key": m.group(2),
|
||||
"value": value
|
||||
})
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(
|
||||
f"Error while reading metadata {key}: {value}. Error: "
|
||||
f"{e}"
|
||||
)
|
||||
return result
|
||||
parser_class = get_parser_class_for_mime_type(mime_type)
|
||||
if parser_class:
|
||||
parser = parser_class(logging_group=None)
|
||||
return parser.extract_metadata(file, mime_type)
|
||||
else:
|
||||
return []
|
||||
|
||||
@action(methods=['get'], detail=True)
|
||||
def metadata(self, request, pk=None):
|
||||
|
||||
@ -210,6 +210,12 @@ AUTH_PASSWORD_VALIDATORS = [
|
||||
|
||||
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
|
||||
|
||||
COOKIE_PREFIX = os.getenv("PAPERLESS_COOKIE_PREFIX", "")
|
||||
|
||||
CSRF_COOKIE_NAME = f"{COOKIE_PREFIX}csrftoken"
|
||||
SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid"
|
||||
LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language"
|
||||
|
||||
###############################################################################
|
||||
# Database #
|
||||
###############################################################################
|
||||
|
||||
@ -1 +1 @@
|
||||
__version__ = (0, 9, 5)
|
||||
__version__ = (0, 9, 6)
|
||||
|
||||
@ -103,10 +103,7 @@ class MailAccountHandler(LoggingMixin):
|
||||
|
||||
def _correspondent_from_name(self, name):
|
||||
try:
|
||||
return Correspondent.objects.get_or_create(
|
||||
name=name, defaults={
|
||||
"slug": slugify(name)
|
||||
})[0]
|
||||
return Correspondent.objects.get_or_create(name=name)[0]
|
||||
except DatabaseError as e:
|
||||
self.log(
|
||||
"error",
|
||||
|
||||
@ -5,6 +5,7 @@ import subprocess
|
||||
|
||||
import ocrmypdf
|
||||
import pdftotext
|
||||
import pikepdf
|
||||
from PIL import Image
|
||||
from django.conf import settings
|
||||
from ocrmypdf import InputFileError, EncryptedPdfError
|
||||
@ -18,6 +19,33 @@ class RasterisedDocumentParser(DocumentParser):
|
||||
image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.)
|
||||
"""
|
||||
|
||||
def extract_metadata(self, document_path, mime_type):
|
||||
namespace_pattern = re.compile(r"\{(.*)\}(.*)")
|
||||
|
||||
result = []
|
||||
if mime_type == 'application/pdf':
|
||||
pdf = pikepdf.open(document_path)
|
||||
meta = pdf.open_metadata()
|
||||
for key, value in meta.items():
|
||||
if isinstance(value, list):
|
||||
value = " ".join([str(e) for e in value])
|
||||
value = str(value)
|
||||
try:
|
||||
m = namespace_pattern.match(key)
|
||||
result.append({
|
||||
"namespace": m.group(1),
|
||||
"prefix": meta.REVERSE_NS[m.group(1)],
|
||||
"key": m.group(2),
|
||||
"value": value
|
||||
})
|
||||
except Exception as e:
|
||||
self.log(
|
||||
"warning",
|
||||
f"Error while reading metadata {key}: {value}. Error: "
|
||||
f"{e}"
|
||||
)
|
||||
return result
|
||||
|
||||
def get_thumbnail(self, document_path, mime_type):
|
||||
"""
|
||||
The thumbnail of a PDF is just a 500px wide image of the first page.
|
||||
@ -82,6 +110,24 @@ class RasterisedDocumentParser(DocumentParser):
|
||||
f"Error while getting DPI from image {image}: {e}")
|
||||
return None
|
||||
|
||||
def calculate_a4_dpi(self, image):
|
||||
try:
|
||||
with Image.open(image) as im:
|
||||
width, height = im.size
|
||||
# divide image width by A4 width (210mm) in inches.
|
||||
dpi = int(width / (21 / 2.54))
|
||||
self.log(
|
||||
'debug',
|
||||
f"Estimated DPI {dpi} based on image width {width}"
|
||||
)
|
||||
return dpi
|
||||
|
||||
except Exception as e:
|
||||
self.log(
|
||||
'warning',
|
||||
f"Error while calculating DPI for image {image}: {e}")
|
||||
return None
|
||||
|
||||
def parse(self, document_path, mime_type):
|
||||
mode = settings.OCR_MODE
|
||||
|
||||
@ -134,6 +180,7 @@ class RasterisedDocumentParser(DocumentParser):
|
||||
|
||||
if self.is_image(mime_type):
|
||||
dpi = self.get_dpi(document_path)
|
||||
a4_dpi = self.calculate_a4_dpi(document_path)
|
||||
if dpi:
|
||||
self.log(
|
||||
"debug",
|
||||
@ -142,6 +189,8 @@ class RasterisedDocumentParser(DocumentParser):
|
||||
ocr_args['image_dpi'] = dpi
|
||||
elif settings.OCR_IMAGE_DPI:
|
||||
ocr_args['image_dpi'] = settings.OCR_IMAGE_DPI
|
||||
elif a4_dpi:
|
||||
ocr_args['image_dpi'] = a4_dpi
|
||||
else:
|
||||
raise ParseError(
|
||||
f"Cannot produce archive PDF for image {document_path}, "
|
||||
@ -213,6 +262,9 @@ def strip_excess_whitespace(text):
|
||||
|
||||
def get_text_from_pdf(pdf_file):
|
||||
|
||||
if not os.path.isfile(pdf_file):
|
||||
return None
|
||||
|
||||
with open(pdf_file, "rb") as f:
|
||||
try:
|
||||
pdf = pdftotext.PDF(f)
|
||||
|
||||
@ -164,8 +164,21 @@ class TestParser(DirectoriesMixin, TestCase):
|
||||
|
||||
self.assertRaises(ParseError, f)
|
||||
|
||||
@mock.patch("paperless_tesseract.parsers.ocrmypdf.ocr")
|
||||
def test_image_calc_a4_dpi(self, m):
|
||||
parser = RasterisedDocumentParser(None)
|
||||
|
||||
def test_image_no_dpi_fail(self):
|
||||
parser.parse(os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"), "image/png")
|
||||
|
||||
m.assert_called_once()
|
||||
|
||||
args, kwargs = m.call_args
|
||||
|
||||
self.assertEqual(kwargs['image_dpi'], 62)
|
||||
|
||||
@mock.patch("paperless_tesseract.parsers.RasterisedDocumentParser.calculate_a4_dpi")
|
||||
def test_image_dpi_fail(self, m):
|
||||
m.return_value = None
|
||||
parser = RasterisedDocumentParser(None)
|
||||
|
||||
def f():
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user