Merge branch 'dev' into quick-filters

README.md

@@ -28,6 +28,7 @@ Here's what you get:
 # Features
 
 * Performs OCR on your documents, adds selectable text to image only documents and adds tags, correspondents and document types to your documents.
+* Paperless stores your documents plain on disk. Filenames and folders are managed by paperless and can be configured freely.
 * Single page application front end. Should be pretty snappy. Will be mobile friendly in the future.
 	* Includes a dashboard that shows basic statistics and has document upload.
 	* Filtering by tags, correspondents, types, and more.

docker/docker-entrypoint.sh

@@ -25,6 +25,11 @@ wait_for_postgres() {
 	host="${PAPERLESS_DBHOST}"
 	port="${PAPERLESS_DBPORT}"
 
+	if [[ -z $port ]] ;
+	then
+		port="5432"
+	fi
+
 	while !</dev/tcp/$host/$port ;
 	do
 
@@ -114,13 +119,13 @@ install_languages() {
     done
 }
 
-initialize
-
 # Install additional languages if specified
 if [[ ! -z "$PAPERLESS_OCR_LANGUAGES"  ]]; then
 		install_languages "$PAPERLESS_OCR_LANGUAGES"
 fi
 
+initialize
+
 if [[ "$1" != "/"* ]]; then
 	exec sudo -HEu paperless python3 manage.py "$@"
 else

docker/hub/docker-compose.postgres.yml

@@ -15,7 +15,7 @@ services:
       POSTGRES_PASSWORD: paperless
 
   webserver:
-    image: jonaswinkler/paperless-ng:0.9.5
+    image: jonaswinkler/paperless-ng:0.9.6
     restart: always
     depends_on:
       - db

docker/hub/docker-compose.sqlite.yml

@@ -5,7 +5,7 @@ services:
     restart: always
 
   webserver:
-    image: jonaswinkler/paperless-ng:0.9.5
+    image: jonaswinkler/paperless-ng:0.9.6
     restart: always
     depends_on:
       - broker

docs/changelog.rst

@@ -32,6 +32,7 @@ This release focusses primarily on many small issues with the UI.
 * Other
 
   * Fixed an issue with the docker image when a non-standard PostgreSQL port was used.
+  * The docker image was trying check for installed languages before actually installing them.
   * ``FILENAME_FORMAT`` placeholder for document types.
   * The filename formatter is now less restrictive with file names and tries to
     conserve the original correspondents, types and titles as much as possible.

docs/configuration.rst

@@ -152,6 +152,16 @@ PAPERLESS_AUTO_LOGIN_USERNAME=<username>
 
     Defaults to none, which disables this feature.
 
+
+PAPERLESS_COOKIE_PREFIX=<str>
+    Specify a prefix that is added to the cookies used by paperless to identify
+    the currently logged in user. This is useful for when you're running two
+    instances of paperless on the same host.
+
+    After changing this, you will have to login again.
+
+    Defaults to ``""``, which does not alter the cookie names.
+
 .. _configuration-ocr:
 
 OCR settings

paperless.conf.example

@@ -30,6 +30,7 @@
 #PAPERLESS_FORCE_SCRIPT_NAME=
 #PAPERLESS_STATIC_URL=/static/
 #PAPERLESS_AUTO_LOGIN_USERNAME=
+#PAPERLESS_COOKIE_PREFIX=
 
 # OCR settings
 

src-ui/package-lock.json

@@ -2215,6 +2215,11 @@
       "integrity": "sha512-UV1/ZJMC+HcP902wWdpC43cAcGu0IQk/I5bXjP2aSuCjsk3cE74mDvFrLKga7oDC170ugOAYBwfT4DSQW3akDA==",
       "dev": true
     },
+    "@types/pdfjs-dist": {
+      "version": "2.1.7",
+      "resolved": "https://registry.npmjs.org/@types/pdfjs-dist/-/pdfjs-dist-2.1.7.tgz",
+      "integrity": "sha512-nQIwcPUhkAIyn7x9NS0lR/qxYfd5unRtfGkMjvpgF4Sh28IXftRymaNmFKTTdejDNY25NDGSIyjwj/BRwAPexg=="
+    },
     "@types/q": {
       "version": "1.5.4",
       "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz",
@@ -3023,6 +3028,16 @@
       "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
       "dev": true
     },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "dev": true,
+      "optional": true,
+      "requires": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
     "blob": {
       "version": "0.0.5",
       "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.5.tgz",
@@ -5508,6 +5523,13 @@
         "schema-utils": "^2.6.5"
       }
     },
+    "file-uri-to-path": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+      "dev": true,
+      "optional": true
+    },
     "fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -8208,6 +8230,13 @@
       "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==",
       "dev": true
     },
+    "nan": {
+      "version": "2.14.2",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz",
+      "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==",
+      "dev": true,
+      "optional": true
+    },
     "nanomatch": {
       "version": "1.2.13",
       "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz",
@@ -8260,6 +8289,23 @@
         "moment": "2.18.1"
       }
     },
+    "ng2-pdf-viewer": {
+      "version": "6.3.2",
+      "resolved": "https://registry.npmjs.org/ng2-pdf-viewer/-/ng2-pdf-viewer-6.3.2.tgz",
+      "integrity": "sha512-H2tBhDd+Lq6CUzK2g54HsCcZDR2wTn1sDjYqKY3yF0Ydasl2R5ppCKynZBU/zge4EKvmHglJI120FbQMpJKDYQ==",
+      "requires": {
+        "@types/pdfjs-dist": "^2.1.4",
+        "pdfjs-dist": "^2.4.456",
+        "tslib": "^1.10.0"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "1.14.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
+          "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
+        }
+      }
+    },
     "ngx-cookie-service": {
       "version": "10.1.1",
       "resolved": "https://registry.npmjs.org/ngx-cookie-service/-/ngx-cookie-service-10.1.1.tgz",
@@ -9270,6 +9316,11 @@
         "sha.js": "^2.4.8"
       }
     },
+    "pdfjs-dist": {
+      "version": "2.5.207",
+      "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.5.207.tgz",
+      "integrity": "sha512-xGDUhnCYPfHy+unMXCLCJtlpZaaZ17Ew3WIL0tnSgKFUZXHAPD49GO9xScyszSsQMoutNDgRb+rfBXIaX/lJbw=="
+    },
     "performance-now": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
@@ -13228,7 +13279,11 @@
           "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz",
           "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==",
           "dev": true,
-          "optional": true
+          "optional": true,
+          "requires": {
+            "bindings": "^1.5.0",
+            "nan": "^2.12.1"
+          }
         },
         "glob-parent": {
           "version": "3.1.0",
@@ -13832,7 +13887,11 @@
           "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz",
           "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==",
           "dev": true,
-          "optional": true
+          "optional": true,
+          "requires": {
+            "bindings": "^1.5.0",
+            "nan": "^2.12.1"
+          }
         },
         "glob-parent": {
           "version": "3.1.0",

src-ui/package.json

@@ -23,6 +23,7 @@
     "@ng-bootstrap/ng-bootstrap": "^8.0.0",
     "bootstrap": "^4.5.0",
     "ng-bootstrap": "^1.6.3",
+    "ng2-pdf-viewer": "^6.3.2",
     "ngx-cookie-service": "^10.1.1",
     "ngx-file-drop": "^10.0.0",
     "ngx-infinite-scroll": "^9.1.0",

src-ui/src/app/app.module.ts

@@ -14,10 +14,9 @@ import { LogsComponent } from './components/manage/logs/logs.component';
 import { SettingsComponent } from './components/manage/settings/settings.component';
 import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { DatePipe } from '@angular/common';
-import { SafePipe } from './pipes/safe.pipe';
 import { NotFoundComponent } from './components/not-found/not-found.component';
 import { CorrespondentListComponent } from './components/manage/correspondent-list/correspondent-list.component';
-import { DeleteDialogComponent } from './components/common/delete-dialog/delete-dialog.component';
+import { ConfirmDialogComponent } from './components/common/confirm-dialog/confirm-dialog.component';
 import { CorrespondentEditDialogComponent } from './components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component';
 import { TagEditDialogComponent } from './components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component';
 import { DocumentTypeEditDialogComponent } from './components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
@@ -48,10 +47,13 @@ import { SavedViewWidgetComponent } from './components/dashboard/widgets/saved-v
 import { StatisticsWidgetComponent } from './components/dashboard/widgets/statistics-widget/statistics-widget.component';
 import { UploadFileWidgetComponent } from './components/dashboard/widgets/upload-file-widget/upload-file-widget.component';
 import { WidgetFrameComponent } from './components/dashboard/widgets/widget-frame/widget-frame.component';
+import { PdfViewerModule } from 'ng2-pdf-viewer';
 import { WelcomeWidgetComponent } from './components/dashboard/widgets/welcome-widget/welcome-widget.component';
 import { YesNoPipe } from './pipes/yes-no.pipe';
 import { FileSizePipe } from './pipes/file-size.pipe';
 import { FilterPipe } from './pipes/filter.pipe';
+import { DocumentTitlePipe } from './pipes/document-title.pipe';
+import { MetadataCollapseComponent } from './components/document-detail/metadata-collapse/metadata-collapse.component';
 
 @NgModule({
   declarations: [
@@ -64,10 +66,9 @@ import { FilterPipe } from './pipes/filter.pipe';
     DocumentTypeListComponent,
     LogsComponent,
     SettingsComponent,
-    SafePipe,
     NotFoundComponent,
     CorrespondentEditDialogComponent,
-    DeleteDialogComponent,
+    ConfirmDialogComponent,
     TagEditDialogComponent,
     DocumentTypeEditDialogComponent,
     TagComponent,
@@ -97,6 +98,8 @@ import { FilterPipe } from './pipes/filter.pipe';
     YesNoPipe,
     FileSizePipe,
     FilterPipe
+    DocumentTitlePipe,
+    MetadataCollapseComponent
   ],
   imports: [
     BrowserModule,
@@ -106,7 +109,8 @@ import { FilterPipe } from './pipes/filter.pipe';
     FormsModule,
     ReactiveFormsModule,
     NgxFileDropModule,
-    InfiniteScrollModule
+    InfiniteScrollModule,
+    PdfViewerModule
   ],
   providers: [
     DatePipe,

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -132,7 +132,7 @@
         </h6>
         <ul class="nav flex-column mb-2">
           <li class="nav-item">
-            <a class="nav-link" href="https://paperless-ng.readthedocs.io/en/latest/">
+            <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://paperless-ng.readthedocs.io/en/latest/">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#question-circle"/>
               </svg>
@@ -140,7 +140,7 @@
             </a>
           </li>
           <li class="nav-item">
-            <a class="nav-link" href="https://github.com/jonaswinkler/paperless-ng">
+            <a class="nav-link" target="_blank" rel="noopener noreferrer" href="https://github.com/jonaswinkler/paperless-ng">
               <svg class="sidebaricon" fill="currentColor">
                 <use xlink:href="assets/bootstrap-icons.svg#link"/>
               </svg>

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.html

@@ -5,10 +5,10 @@
       </button>
     </div>
     <div class="modal-body">
-      <p><b>{{message}}</b></p>
+      <p *ngIf="messageBold"><b>{{messageBold}}</b></p>
-      <p *ngIf="message2">{{message2}}</p>
+      <p *ngIf="message">{{message}}</p>
     </div>
     <div class="modal-footer">
       <button type="button" class="btn btn-outline-dark" (click)="cancelClicked()">Cancel</button>
-      <button type="button" class="btn btn-danger" (click)="deleteClicked.emit()">Delete</button>
+      <button type="button" class="btn" [class]="btnClass" (click)="confirmClicked.emit()">{{btnCaption}}</button>
     </div>

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.spec.ts

@@ -1,20 +1,20 @@
 import { ComponentFixture, TestBed } from '@angular/core/testing';
 
-import { DeleteDialogComponent } from './delete-dialog.component';
+import { ConfirmDialogComponent } from './confirm-dialog.component';
 
-describe('DeleteDialogComponent', () => {
+describe('ConfirmDialogComponent', () => {
-  let component: DeleteDialogComponent;
+  let component: ConfirmDialogComponent;
-  let fixture: ComponentFixture<DeleteDialogComponent>;
+  let fixture: ComponentFixture<ConfirmDialogComponent>;
 
   beforeEach(async () => {
     await TestBed.configureTestingModule({
-      declarations: [ DeleteDialogComponent ]
+      declarations: [ ConfirmDialogComponent ]
     })
     .compileComponents();
   });
 
   beforeEach(() => {
-    fixture = TestBed.createComponent(DeleteDialogComponent);
+    fixture = TestBed.createComponent(ConfirmDialogComponent);
     component = fixture.componentInstance;
     fixture.detectChanges();
   });

src-ui/src/app/components/common/confirm-dialog/confirm-dialog.component.ts

@@ -0,0 +1,37 @@
+import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+
+@Component({
+  selector: 'app-confirm-dialog',
+  templateUrl: './confirm-dialog.component.html',
+  styleUrls: ['./confirm-dialog.component.scss']
+})
+export class ConfirmDialogComponent implements OnInit {
+
+  constructor(public activeModal: NgbActiveModal) { }
+
+  @Output()
+  public confirmClicked = new EventEmitter()
+
+  @Input()
+  title = "Confirmation"
+
+  @Input()
+  messageBold
+
+  @Input()
+  message
+
+  @Input()
+  btnClass = "btn-primary"
+
+  @Input()
+  btnCaption = "Confirm"
+
+  ngOnInit(): void {
+  }
+
+  cancelClicked() {
+    this.activeModal.close()
+  }
+}

src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts

@@ -1,31 +0,0 @@
-import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
-import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
-
-@Component({
-  selector: 'app-delete-dialog',
-  templateUrl: './delete-dialog.component.html',
-  styleUrls: ['./delete-dialog.component.scss']
-})
-export class DeleteDialogComponent implements OnInit {
-
-  constructor(public activeModal: NgbActiveModal) { }
-
-  @Output()
-  public deleteClicked = new EventEmitter()
-
-  @Input()
-  title = "Delete confirmation"
-
-  @Input()
-  message = "Do you really want to delete this?"
-
-  @Input()
-  message2
-
-  ngOnInit(): void {
-  }
-
-  cancelClicked() {
-    this.activeModal.close()
-  }
-}

src-ui/src/app/components/common/input/tags/tags.component.html

@@ -8,7 +8,7 @@
 
     <div class="input-group-append" ngbDropdown placement="top-right">
       <button class="btn btn-outline-secondary" type="button" ngbDropdownToggle></button>
-      <div ngbDropdownMenu class="scrollable-menu">
+      <div ngbDropdownMenu class="scrollable-menu shadow">
         <button type="button" *ngFor="let tag of tags" ngbDropdownItem (click)="addTag(tag.id)">
           <app-tag [tag]="tag"></app-tag>
         </button>

src-ui/src/app/components/dashboard/widgets/widget-frame/widget-frame.component.html

@@ -1,4 +1,4 @@
-<div class="card mb-3 shadow">
+<div class="card mb-3 shadow-sm">
   <div class="card-header">
     <div class="d-flex justify-content-between align-items-center">
       <h5 class="card-title mb-0">{{title}}</h5>

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -17,7 +17,7 @@
 
         <div class="btn-group" ngbDropdown role="group" *ngIf="metadata?.has_archive_version">
             <button class="btn btn-sm btn-outline-primary dropdown-toggle-split" ngbDropdownToggle></button>
-            <div class="dropdown-menu" ngbDropdownMenu>
+            <div class="dropdown-menu shadow" ngbDropdownMenu>
                 <a ngbDropdownItem [href]="downloadOriginalUrl">Download original</a>
             </div>
         </div>
@@ -35,7 +35,7 @@
 
 
 <div class="row">
-    <div class="col-xl">
+    <div class="col mb-4">
 
         <form [formGroup]='documentForm' (ngSubmit)="save()">
 
@@ -110,53 +110,8 @@
                             </tbody>
                         </table>
 
-                        <h6 *ngIf="metadata?.original_metadata.length > 0">
+                        <app-metadata-collapse title="Original document metadata" [metadata]="metadata.original_metadata" *ngIf="metadata?.original_metadata.length > 0"></app-metadata-collapse>
-                            <button type="button" class="btn btn-outline-secondary btn-sm mr-2"
+                        <app-metadata-collapse title="Archived document metadata" [metadata]="metadata.archive_metadata" *ngIf="metadata?.archive_metadata.length > 0"></app-metadata-collapse>
-                                (click)="expandOriginalMetadata = !expandOriginalMetadata" aria-controls="collapseExample">
-                                <svg class="buttonicon" fill="currentColor" *ngIf="!expandOriginalMetadata">
-                                    <use xlink:href="assets/bootstrap-icons.svg#caret-down" />
-                                </svg>
-                                <svg class="buttonicon" fill="currentColor" *ngIf="expandOriginalMetadata">
-                                    <use xlink:href="assets/bootstrap-icons.svg#caret-up" />
-                                </svg>
-                            </button>
-                            Original document metadata
-                        </h6>
-
-                        <div #collapse="ngbCollapse" [(ngbCollapse)]="!expandOriginalMetadata">
-                            <table class="table table-borderless">
-                                <tbody>
-                                    <tr *ngFor="let m of metadata?.original_metadata">
-                                        <td>{{m.prefix}}:{{m.key}}</td>
-                                        <td>{{m.value}}</td>
-                                    </tr>
-                                </tbody>
-                            </table>
-                        </div>
-
-                        <h6 *ngIf="metadata?.has_archive_version && metadata?.archive_metadata.length > 0">
-                            <button type="button" class="btn btn-outline-secondary btn-sm mr-2"
-                                (click)="expandArchivedMetadata = !expandArchivedMetadata" aria-controls="collapseExample">
-                                <svg class="buttonicon" fill="currentColor" *ngIf="!expandArchivedMetadata">
-                                    <use xlink:href="assets/bootstrap-icons.svg#caret-down" />
-                                </svg>
-                                <svg class="buttonicon" fill="currentColor" *ngIf="expandArchivedMetadata">
-                                    <use xlink:href="assets/bootstrap-icons.svg#caret-up" />
-                                </svg>
-                            </button>
-                            Archived document metadata
-                        </h6>
-
-                        <div #collapse="ngbCollapse" [(ngbCollapse)]="!expandArchivedMetadata">
-                            <table class="table table-borderless">
-                                <tbody>
-                                    <tr *ngFor="let m of metadata?.archive_metadata">
-                                        <td>{{m.prefix}}:{{m.key}}</td>
-                                        <td>{{m.value}}</td>
-                                    </tr>
-                                </tbody>
-                            </table>
-                        </div>
 
                     </ng-template>
                 </li>
@@ -171,11 +126,9 @@
         </form>
     </div>
 
-    <div class="col-xl d-none d-xl-block document-preview">
+    <div class="col-md-6 col-xl-8 mb-3">
-        <object [data]="previewUrl | safe" type="application/pdf" width="100%" height="100%">
+      <div class="pdf-viewer-container" *ngIf="getContentType() == 'application/pdf'">
-            <p>Your browser does not support PDFs.
+        <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true"></pdf-viewer>
-                <a href="previewUrl">Download the PDF</a>.</p>
+      </div>
-        </object>
-
     </div>
 </div>

src-ui/src/app/components/document-detail/document-detail.component.scss

@@ -1,5 +1,6 @@
-.document-preview {
+.pdf-viewer-container {
-  height: calc(100vh - 180px);
+  height: calc(100vh - 160px);
   top: 70px;
   position: sticky;
+  background-color: gray;
 }

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -13,7 +13,7 @@ import { CorrespondentService } from 'src/app/services/rest/correspondent.servic
 import { DocumentTypeService } from 'src/app/services/rest/document-type.service';
 import { DocumentService } from 'src/app/services/rest/document.service';
 import { environment } from 'src/environments/environment';
-import { DeleteDialogComponent } from '../common/delete-dialog/delete-dialog.component';
+import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component';
 import { CorrespondentEditDialogComponent } from '../manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component';
 import { DocumentTypeEditDialogComponent } from '../manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
 
@@ -59,6 +59,10 @@ export class DocumentDetailComponent implements OnInit {
     private documentListViewService: DocumentListViewService,
     private titleService: Title) { }
 
+  getContentType() {
+    return this.metadata?.has_archive_version ? 'application/pdf' : this.metadata?.original_mime_type
+  }
+
   ngOnInit(): void {
     this.documentForm.valueChanges.subscribe(wow => {
       Object.assign(this.document, this.documentForm.value)
@@ -151,10 +155,13 @@ export class DocumentDetailComponent implements OnInit {
   }
 
   delete() {
-    let modal = this.modalService.open(DeleteDialogComponent, {backdrop: 'static'})
+    let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'})
-    modal.componentInstance.message = `Do you really want to delete document '${this.document.title}'?`
+    modal.componentInstance.title = "Confirm delete"
-    modal.componentInstance.message2 = `The files for this document will be deleted permanently. This operation cannot be undone.`
+    modal.componentInstance.messageBold = `Do you really want to delete document '${this.document.title}'?`
-    modal.componentInstance.deleteClicked.subscribe(() => {
+    modal.componentInstance.message = `The files for this document will be deleted permanently. This operation cannot be undone.`
+    modal.componentInstance.btnClass = "btn-danger"
+    modal.componentInstance.btnCaption = "Delete document"
+    modal.componentInstance.confirmClicked.subscribe(() => {
       this.documentsService.delete(this.document).subscribe(() => {
         modal.close()  
         this.close()

src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.html

@@ -0,0 +1,23 @@
+<h6>
+  <button type="button" class="btn btn-outline-secondary btn-sm mr-2"
+      (click)="expand = !expand">
+      <svg class="buttonicon" fill="currentColor" *ngIf="!expand">
+          <use xlink:href="assets/bootstrap-icons.svg#caret-down" />
+      </svg>
+      <svg class="buttonicon" fill="currentColor" *ngIf="expand">
+          <use xlink:href="assets/bootstrap-icons.svg#caret-up" />
+      </svg>
+  </button>
+  {{title}}
+</h6>
+
+<div #collapse="ngbCollapse" [(ngbCollapse)]="!expand">
+  <table class="table table-borderless">
+      <tbody>
+          <tr *ngFor="let m of metadata">
+              <td>{{m.prefix}}:{{m.key}}</td>
+              <td>{{m.value}}</td>
+          </tr>
+      </tbody>
+  </table>
+</div>

src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.spec.ts

@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { MetadataCollapseComponent } from './metadata-collapse.component';
+
+describe('MetadataCollapseComponent', () => {
+  let component: MetadataCollapseComponent;
+  let fixture: ComponentFixture<MetadataCollapseComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ MetadataCollapseComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(MetadataCollapseComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

src-ui/src/app/components/document-detail/metadata-collapse/metadata-collapse.component.ts

@@ -0,0 +1,23 @@
+import { Component, Input, OnInit } from '@angular/core';
+
+@Component({
+  selector: 'app-metadata-collapse',
+  templateUrl: './metadata-collapse.component.html',
+  styleUrls: ['./metadata-collapse.component.scss']
+})
+export class MetadataCollapseComponent implements OnInit {
+
+  constructor() { }
+
+  expand = false
+
+  @Input()
+  metadata
+
+  @Input()
+  title = "Metadata"
+
+  ngOnInit(): void {
+  }
+
+}

src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html

@@ -12,7 +12,7 @@
               <a *ngIf="clickCorrespondent.observers.length ; else nolink" [routerLink]="" title="Filter by correspondent" (click)="clickCorrespondent.emit(document.correspondent)" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>
               <ng-template #nolink>{{(document.correspondent$ | async)?.name}}</ng-template>:
             </ng-container>
-            {{document.title}}
+            {{document.title | documentTitle}}
             <app-tag [tag]="t" linkTitle="Filter by tag" *ngFor="let t of document.tags$ | async" class="ml-1" (click)="clickTag.emit(t.id)" [clickable]="clickTag.observers.length"></app-tag>
           </h5>
           <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archive_serial_number}}</h5>

src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html

@@ -1,6 +1,6 @@
 <div class="col p-2 h-100" style="width: 16rem;">
   <div class="card h-100 shadow-sm">
-    <div class=" border-bottom pr-1">
+    <div class="border-bottom">
       <img class="card-img doc-img" [src]="getThumbUrl()">
       <div style="top: 0; right: 0; font-size: large" class="text-right position-absolute mr-1">
         <div *ngFor="let t of getTagsLimited$() | async">
@@ -17,7 +17,7 @@
         <ng-container *ngIf="document.correspondent">
           <a [routerLink]="" title="Filter by correspondent" (click)="clickCorrespondent.emit(document.correspondent)" class="font-weight-bold">{{(document.correspondent$ | async)?.name}}</a>:
         </ng-container>
-        {{document.title}}
+        {{document.title | documentTitle}}
       </p>
     </div>
     <div class="card-footer">

src-ui/src/app/components/document-list/document-list.component.html

@@ -24,7 +24,7 @@
   <div class="btn-group btn-group-toggle ml-2" ngbRadioGroup [(ngModel)]="list.sortDirection">
     <div ngbDropdown class="btn-group">
       <button class="btn btn-outline-primary btn-sm" id="dropdownBasic1" ngbDropdownToggle>Sort by</button>
-      <div ngbDropdownMenu aria-labelledby="dropdownBasic1">
+      <div ngbDropdownMenu aria-labelledby="dropdownBasic1" class="shadow">
         <button *ngFor="let f of getSortFields()" ngbDropdownItem (click)="list.sortField = f.field"
           [class.active]="list.sortField == f.field">{{f.name}}</button>
       </div>
@@ -79,7 +79,7 @@
   </app-document-card-large>
 </div>
 
-<table class="table table-sm border shadow" *ngIf="displayMode == 'details'">
+<table class="table table-sm border shadow-sm" *ngIf="displayMode == 'details'">
   <thead>
     <th class="d-none d-lg-table-cell">ASN</th>
     <th class="d-none d-md-table-cell">Correspondent</th>
@@ -99,7 +99,7 @@
         </ng-container>
       </td>
       <td>
-        <a routerLink="/documents/{{d.id}}" title="Edit document" style="overflow-wrap: anywhere;">{{d.title}}</a>
+        <a routerLink="/documents/{{d.id}}" title="Edit document" style="overflow-wrap: anywhere;">{{d.title | documentTitle}}</a>
         <app-tag [tag]="t" *ngFor="let t of d.tags$ | async" class="ml-1" clickable="true" linkTitle="Filter by tag" (click)="clickTag(t)"></app-tag>
       </td>
       <td class="d-none d-xl-table-cell">

src-ui/src/app/components/manage/generic-list/generic-list.component.ts

@@ -4,7 +4,7 @@ import { MatchingModel, MATCHING_ALGORITHMS, MATCH_AUTO } from 'src/app/data/mat
 import { ObjectWithId } from 'src/app/data/object-with-id';
 import { SortableDirective, SortEvent } from 'src/app/directives/sortable.directive';
 import { AbstractPaperlessService } from 'src/app/services/rest/abstract-paperless-service';
-import { DeleteDialogComponent } from '../../common/delete-dialog/delete-dialog.component';
+import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component';
 
 @Directive()
 export abstract class GenericListComponent<T extends ObjectWithId> implements OnInit {
@@ -88,10 +88,13 @@ export abstract class GenericListComponent<T extends ObjectWithId> implements On
   }
 
   openDeleteDialog(object: T) {
-    var activeModal = this.modalService.open(DeleteDialogComponent, {backdrop: 'static'})
+    var activeModal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'})
-    activeModal.componentInstance.message = `Do you really want to delete ${this.getObjectName(object)}?`
+    activeModal.componentInstance.title = "Confirm delete"
-    activeModal.componentInstance.message2 = "Associated documents will not be deleted."
+    activeModal.componentInstance.messageBold = `Do you really want to delete ${this.getObjectName(object)}?`
-    activeModal.componentInstance.deleteClicked.subscribe(() => {
+    activeModal.componentInstance.message = "Associated documents will not be deleted."
+    activeModal.componentInstance.btnClass = "btn-danger"
+    activeModal.componentInstance.btnCaption = "Delete"
+    activeModal.componentInstance.confirmPressed.subscribe(() => {
       this.service.delete(object).subscribe(_ => {
         activeModal.close()
         this.reloadData()

src-ui/src/app/pipes/document-title.pipe.spec.ts

@@ -0,0 +1,8 @@
+import { DocumentTitlePipe } from './document-title.pipe';
+
+describe('DocumentTitlePipe', () => {
+  it('create an instance', () => {
+    const pipe = new DocumentTitlePipe();
+    expect(pipe).toBeTruthy();
+  });
+});

src-ui/src/app/pipes/document-title.pipe.ts

@@ -0,0 +1,16 @@
+import { Pipe, PipeTransform } from '@angular/core';
+
+@Pipe({
+  name: 'documentTitle'
+})
+export class DocumentTitlePipe implements PipeTransform {
+
+  transform(value: string): unknown {
+    if (value) {
+      return value
+    } else {
+      return "(no title)"
+    }
+  }
+
+}

src-ui/src/app/pipes/safe.pipe.spec.ts

@@ -1,8 +0,0 @@
-import { SafePipe } from './safe.pipe';
-
-describe('SafePipe', () => {
-  it('create an instance', () => {
-    const pipe = new SafePipe();
-    expect(pipe).toBeTruthy();
-  });
-});

src-ui/src/app/pipes/safe.pipe.ts

@@ -1,19 +0,0 @@
-import { Pipe, PipeTransform } from '@angular/core';
-import { DomSanitizer } from '@angular/platform-browser';
-
-@Pipe({
-  name: 'safe'
-})
-export class SafePipe implements PipeTransform {
-
-  constructor(private sanitizer: DomSanitizer) { }
-
-  transform(url) {
-    if (url == null) {
-      return this.sanitizer.bypassSecurityTrustResourceUrl("")
-    } else {
-      return this.sanitizer.bypassSecurityTrustResourceUrl(url);
-    }
-  }
-
-}

src/documents/admin.py

@@ -17,8 +17,6 @@ class CorrespondentAdmin(admin.ModelAdmin):
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
-    readonly_fields = ("slug",)
-
 
 class TagAdmin(admin.ModelAdmin):
 
@@ -31,8 +29,6 @@ class TagAdmin(admin.ModelAdmin):
     list_filter = ("colour", "matching_algorithm")
     list_editable = ("colour", "match", "matching_algorithm")
 
-    readonly_fields = ("slug", )
-
 
 class DocumentTypeAdmin(admin.ModelAdmin):
 
@@ -44,8 +40,6 @@ class DocumentTypeAdmin(admin.ModelAdmin):
     list_filter = ("matching_algorithm",)
     list_editable = ("match", "matching_algorithm")
 
-    readonly_fields = ("slug",)
-
 
 class DocumentAdmin(admin.ModelAdmin):
 
@@ -106,7 +100,7 @@ class DocumentAdmin(admin.ModelAdmin):
         for tag in obj.tags.all():
             r += self._html_tag(
                 "span",
-                tag.slug + ", "
+                tag.name + ", "
             )
         return r
 

src/documents/consumer.py

@@ -248,7 +248,7 @@ class Consumer(LoggingMixin):
         with open(self.path, "rb") as f:
             document = Document.objects.create(
                 correspondent=file_info.correspondent,
-                title=file_info.title,
+                title=(self.override_title or file_info.title)[:127],
                 content=text,
                 mime_type=mime_type,
                 checksum=hashlib.md5(f.read()).hexdigest(),
@@ -259,18 +259,17 @@ class Consumer(LoggingMixin):
 
         relevant_tags = set(file_info.tags)
         if relevant_tags:
-            tag_names = ", ".join([t.slug for t in relevant_tags])
+            tag_names = ", ".join([t.name for t in relevant_tags])
             self.log("debug", "Tagging with {}".format(tag_names))
             document.tags.add(*relevant_tags)
 
         self.apply_overrides(document)
 
+        document.save()
+
         return document
 
     def apply_overrides(self, document):
-        if self.override_title:
-            document.title = self.override_title
-
         if self.override_correspondent_id:
             document.correspondent = Correspondent.objects.get(
                 pk=self.override_correspondent_id)

src/documents/filters.py

@@ -51,7 +51,6 @@ class TagsFilter(Filter):
             return qs
 
         for tag_id in tag_ids:
-            print(self.exclude, tag_id)
             if self.exclude:
                 qs = qs.exclude(tags__id=tag_id)
             else:

src/documents/management/commands/decrypt_documents.py

@@ -82,7 +82,8 @@ class Command(BaseCommand):
             with open(document.thumbnail_path, "wb") as f:
                 f.write(raw_thumb)
 
-            document.save(update_fields=("storage_type", "filename"))
+            Document.objects.filter(id=document.id).update(
+                storage_type=document.storage_type, filename=document.filename)
 
             for path in old_paths:
                 os.unlink(path)

src/documents/management/commands/document_consumer.py

@@ -29,10 +29,9 @@ def _tags_from_path(filepath):
     path_parts = Path(filepath).relative_to(
                 settings.CONSUMPTION_DIR).parent.parts
     for part in path_parts:
-        tag_ids.add(Tag.objects.get_or_create(
+        tag_ids.add(Tag.objects.get_or_create(name__iexact=part, defaults={
-            slug=slugify(part),
+            "name": part
-            defaults={"name": part},
+        })[0].pk)
-        )[0].pk)
 
     return tag_ids
 

src/documents/management/commands/document_renamer.py

@@ -1,4 +1,8 @@
+import logging
+
+import tqdm
 from django.core.management.base import BaseCommand
+from django.db.models.signals import post_save
 
 from documents.models import Document
 from ...mixins import Renderable
@@ -18,6 +22,7 @@ class Command(Renderable, BaseCommand):
 
         self.verbosity = options["verbosity"]
 
-        for document in Document.objects.all():
+        logging.getLogger().handlers[0].level = logging.ERROR
-            # Saving the document again will generate a new filename and rename
+
-            document.save()
+        for document in tqdm.tqdm(Document.objects.all()):
+            post_save.send(Document, instance=document)

src/documents/migrations/1006_auto_20201208_2209.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.1.4 on 2020-12-08 22:09
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '1005_checksums'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='correspondent',
+            name='slug',
+        ),
+        migrations.RemoveField(
+            model_name='documenttype',
+            name='slug',
+        ),
+        migrations.RemoveField(
+            model_name='tag',
+            name='slug',
+        ),
+    ]

src/documents/models.py

@@ -36,7 +36,6 @@ class MatchingModel(models.Model):
     )
 
     name = models.CharField(max_length=128, unique=True)
-    slug = models.SlugField(blank=True, editable=False)
 
     match = models.CharField(max_length=256, blank=True)
     matching_algorithm = models.PositiveIntegerField(
@@ -69,7 +68,6 @@ class MatchingModel(models.Model):
     def save(self, *args, **kwargs):
 
         self.match = self.match.lower()
-        self.slug = slugify(self.name)
 
         models.Model.save(self, *args, **kwargs)
 
@@ -384,9 +382,7 @@ class FileInfo:
     def _get_correspondent(cls, name):
         if not name:
             return None
-        return Correspondent.objects.get_or_create(name=name, defaults={
+        return Correspondent.objects.get_or_create(name=name)[0]
-            "slug": slugify(name)
-        })[0]
 
     @classmethod
     def _get_title(cls, title):
@@ -396,10 +392,7 @@ class FileInfo:
     def _get_tags(cls, tags):
         r = []
         for t in tags.split(","):
-            r.append(Tag.objects.get_or_create(
+            r.append(Tag.objects.get_or_create(name=t)[0])
-                slug=slugify(t),
-                defaults={"name": t}
-            )[0])
         return tuple(r)
 
     @classmethod

src/documents/parsers.py

@@ -210,6 +210,7 @@ class DocumentParser(LoggingMixin):
     def __init__(self, logging_group):
         super().__init__()
         self.logging_group = logging_group
+        os.makedirs(settings.SCRATCH_DIR, exist_ok=True)
         self.tempdir = tempfile.mkdtemp(
             prefix="paperless-", dir=settings.SCRATCH_DIR)
 
@@ -217,6 +218,9 @@ class DocumentParser(LoggingMixin):
         self.text = None
         self.date = None
 
+    def extract_metadata(self, document_path, mime_type):
+        return []
+
     def parse(self, document_path, mime_type):
         raise NotImplementedError()
 

src/documents/sanity_checker.py

@@ -46,6 +46,10 @@ def check_sanity():
         for f in files:
             present_files.append(os.path.normpath(os.path.join(root, f)))
 
+    lockfile = os.path.normpath(settings.MEDIA_LOCK)
+    if lockfile in present_files:
+        present_files.remove(lockfile)
+
     for doc in Document.objects.all():
         # Check sanity of the thumbnail
         if not os.path.isfile(doc.thumbnail_path):

src/documents/serialisers.py

@@ -1,5 +1,5 @@
 import magic
-from pathvalidate import validate_filename, ValidationError
+from django.utils.text import slugify
 from rest_framework import serializers
 from rest_framework.fields import SerializerMethodField
 
@@ -7,12 +7,16 @@ from .models import Correspondent, Tag, Document, Log, DocumentType
 from .parsers import is_mime_type_supported
 
 
-class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
+class CorrespondentSerializer(serializers.ModelSerializer):
 
     document_count = serializers.IntegerField(read_only=True)
 
     last_correspondence = serializers.DateTimeField(read_only=True)
 
+    def get_slug(self, obj):
+        return slugify(obj.name)
+    slug = SerializerMethodField()
+
     class Meta:
         model = Correspondent
         fields = (
@@ -27,10 +31,14 @@ class CorrespondentSerializer(serializers.HyperlinkedModelSerializer):
         )
 
 
-class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
+class DocumentTypeSerializer(serializers.ModelSerializer):
 
     document_count = serializers.IntegerField(read_only=True)
 
+    def get_slug(self, obj):
+        return slugify(obj.name)
+    slug = SerializerMethodField()
+
     class Meta:
         model = DocumentType
         fields = (
@@ -44,10 +52,14 @@ class DocumentTypeSerializer(serializers.HyperlinkedModelSerializer):
         )
 
 
-class TagSerializer(serializers.HyperlinkedModelSerializer):
+class TagSerializer(serializers.ModelSerializer):
 
     document_count = serializers.IntegerField(read_only=True)
 
+    def get_slug(self, obj):
+        return slugify(obj.name)
+    slug = SerializerMethodField()
+
     class Meta:
         model = Tag
         fields = (
@@ -166,12 +178,6 @@ class PostDocumentSerializer(serializers.Serializer):
     )
 
     def validate_document(self, document):
-
-        try:
-            validate_filename(document.name)
-        except ValidationError:
-            raise serializers.ValidationError("Invalid filename.")
-
         document_data = document.file.read()
         mime_type = magic.from_buffer(document_data, mime=True)
 

src/documents/signals/handlers.py

@@ -7,6 +7,7 @@ from django.contrib.admin.models import ADDITION, LogEntry
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.db import models, DatabaseError
+from django.db.models import Q
 from django.dispatch import receiver
 from django.utils import timezone
 from filelock import FileLock
@@ -121,11 +122,14 @@ def set_tags(sender,
              classifier=None,
              replace=False,
              **kwargs):
+
     if replace:
-        document.tags.clear()
+        Document.tags.through.objects.filter(document=document).exclude(
-        current_tags = set([])
+            Q(tag__is_inbox_tag=True)).exclude(
-    else:
+            Q(tag__match="") & ~Q(tag__matching_algorithm=Tag.MATCH_AUTO)
-        current_tags = set(document.tags.all())
+        ).delete()
+
+    current_tags = set(document.tags.all())
 
     matched_tags = matching.match_tags(document.content, classifier)
 
@@ -136,7 +140,7 @@ def set_tags(sender,
 
     message = 'Tagging "{}" with "{}"'
     logger(
-        message.format(document, ", ".join([t.slug for t in relevant_tags])),
+        message.format(document, ", ".join([t.name for t in relevant_tags])),
         logging_group
     )
 
@@ -165,35 +169,36 @@ def run_post_consume_script(sender, document, **kwargs):
         reverse("document-download", kwargs={"pk": document.pk}),
         reverse("document-thumb", kwargs={"pk": document.pk}),
         str(document.correspondent),
-        str(",".join(document.tags.all().values_list("slug", flat=True)))
+        str(",".join(document.tags.all().values_list("name", flat=True)))
     )).wait()
 
 
 @receiver(models.signals.post_delete, sender=Document)
 def cleanup_document_deletion(sender, instance, using, **kwargs):
-    for f in (instance.source_path,
+    with FileLock(settings.MEDIA_LOCK):
-              instance.archive_path,
+        for f in (instance.source_path,
-              instance.thumbnail_path):
+                  instance.archive_path,
-        if os.path.isfile(f):
+                  instance.thumbnail_path):
-            try:
+            if os.path.isfile(f):
-                os.unlink(f)
+                try:
-                logging.getLogger(__name__).debug(
+                    os.unlink(f)
-                    f"Deleted file {f}.")
+                    logging.getLogger(__name__).debug(
-            except OSError as e:
+                        f"Deleted file {f}.")
-                logging.getLogger(__name__).warning(
+                except OSError as e:
-                    f"While deleting document {str(instance)}, the file "
+                    logging.getLogger(__name__).warning(
-                    f"{f} could not be deleted: {e}"
+                        f"While deleting document {str(instance)}, the file "
-                )
+                        f"{f} could not be deleted: {e}"
+                    )
 
-    delete_empty_directories(
+        delete_empty_directories(
-        os.path.dirname(instance.source_path),
+            os.path.dirname(instance.source_path),
-        root=settings.ORIGINALS_DIR
+            root=settings.ORIGINALS_DIR
-    )
+        )
 
-    delete_empty_directories(
+        delete_empty_directories(
-        os.path.dirname(instance.archive_path),
+            os.path.dirname(instance.archive_path),
-        root=settings.ARCHIVE_DIR
+            root=settings.ARCHIVE_DIR
-    )
+        )
 
 
 def validate_move(instance, old_path, new_path):

src/documents/tasks.py

@@ -1,5 +1,6 @@
 import logging
 
+import tqdm
 from django.conf import settings
 from whoosh.writing import AsyncWriter
 
@@ -23,7 +24,7 @@ def index_reindex():
     ix = index.open_index(recreate=True)
 
     with AsyncWriter(ix) as writer:
-        for document in documents:
+        for document in tqdm.tqdm(documents):
             index.update_document(writer, document)
 
 

src/documents/tests/test_api.py

@@ -403,16 +403,6 @@ class TestDocumentApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, 400)
         m.assert_not_called()
 
-    @mock.patch("documents.views.async_task")
-    @mock.patch("documents.serialisers.validate_filename")
-    def test_upload_invalid_filename(self, validate_filename, async_task):
-        validate_filename.side_effect = ValidationError()
-        with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f:
-            response = self.client.post("/api/documents/post_document/", {"document": f})
-        self.assertEqual(response.status_code, 400)
-
-        async_task.assert_not_called()
-
     @mock.patch("documents.views.async_task")
     def test_upload_with_title(self, async_task):
         with open(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), "rb") as f:

src/documents/tests/test_consumer.py

@@ -27,7 +27,7 @@ class TestAttributes(TestCase):
 
         self.assertEqual(file_info.title, title, filename)
 
-        self.assertEqual(tuple([t.slug for t in file_info.tags]), tags, filename)
+        self.assertEqual(tuple([t.name for t in file_info.tags]), tags, filename)
 
     def test_guess_attributes_from_name0(self):
         self._test_guess_attributes_from_name(
@@ -188,7 +188,7 @@ class TestFieldPermutations(TestCase):
             self.assertEqual(info.tags, (), filename)
         else:
             self.assertEqual(
-                [t.slug for t in info.tags], tags.split(','),
+                [t.name for t in info.tags], tags.split(','),
                 filename
             )
 
@@ -342,8 +342,8 @@ class TestFieldPermutations(TestCase):
             info = FileInfo.from_filename(filename)
             self.assertEqual(info.title, "0001")
             self.assertEqual(len(info.tags), 2)
-            self.assertEqual(info.tags[0].slug, "tag1")
+            self.assertEqual(info.tags[0].name, "tag1")
-            self.assertEqual(info.tags[1].slug, "tag2")
+            self.assertEqual(info.tags[1].name, "tag2")
             self.assertIsNone(info.created)
 
         # Complex transformation with date in replacement string
@@ -356,8 +356,8 @@ class TestFieldPermutations(TestCase):
             info = FileInfo.from_filename(filename)
             self.assertEqual(info.title, "0001")
             self.assertEqual(len(info.tags), 2)
-            self.assertEqual(info.tags[0].slug, "tag1")
+            self.assertEqual(info.tags[0].name, "tag1")
-            self.assertEqual(info.tags[1].slug, "tag2")
+            self.assertEqual(info.tags[1].name, "tag2")
             self.assertEqual(info.created.year, 2019)
             self.assertEqual(info.created.month, 9)
             self.assertEqual(info.created.day, 8)

src/documents/tests/test_file_handling.py

@@ -3,7 +3,6 @@ import hashlib
 import os
 import random
 import uuid
-from concurrent.futures.thread import ThreadPoolExecutor
 from pathlib import Path
 from unittest import mock
 
@@ -15,7 +14,6 @@ from .utils import DirectoriesMixin
 from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \
     generate_unique_filename
 from ..models import Document, Correspondent
-from ..sanity_checker import check_sanity
 
 
 class TestFileHandling(DirectoriesMixin, TestCase):
@@ -573,21 +571,3 @@ def run():
     for i in range(30):
         doc.title = str(random.randrange(1, 5))
         doc.save()
-
-
-class TestSuperMassive(DirectoriesMixin, TestCase):
-
-    @override_settings(PAPERLESS_FILENAME_FORMAT="{title}")
-    def test_super_massive(self):
-        # try to save as many documents in parallel as possible.
-        # try to make the system fail.
-
-        with ThreadPoolExecutor(max_workers=16) as executor:
-            results = [executor.submit(run) for i in range(16)]
-
-        for r in results:
-            if r.exception():
-                raise r.exception()
-
-        # nope, everything still good. Thank you, lockfiles.
-        self.assertEqual(len(check_sanity()), 0)

src/documents/tests/test_management_archiver.py

@@ -16,25 +16,23 @@ sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
 class TestArchiver(DirectoriesMixin, TestCase):
 
     def make_models(self):
-        self.d1 = Document.objects.create(checksum="A", title="A", content="first document", pk=1, mime_type="application/pdf")
+        return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf")
-        #self.d2 = Document.objects.create(checksum="B", title="B", content="second document")
-        #self.d3 = Document.objects.create(checksum="C", title="C", content="unrelated document")
 
     def test_archiver(self):
 
-        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
+        doc = self.make_models()
-        self.make_models()
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
 
         call_command('document_archiver')
 
     def test_handle_document(self):
 
-        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, "0000001.pdf"))
+        doc = self.make_models()
-        self.make_models()
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
 
-        handle_document(self.d1.pk)
+        handle_document(doc.pk)
 
-        doc = Document.objects.get(id=self.d1.id)
+        doc = Document.objects.get(id=doc.id)
 
         self.assertIsNotNone(doc.checksum)
         self.assertTrue(os.path.isfile(doc.archive_path))

src/documents/tests/test_management_consumer.py

@@ -230,7 +230,7 @@ class TestConsumerTags(DirectoriesMixin, ConsumerMixin, TransactionTestCase):
 
         tag_names = ("existingTag", "Space Tag")
         # Create a Tag prior to consuming a file using it in path
-        tag_ids = [Tag.objects.create(name=tag_names[0]).pk,]
+        tag_ids = [Tag.objects.create(name="existingtag").pk,]
 
         self.t_start()
 

src/documents/tests/test_management_decrypt.py

@@ -35,20 +35,20 @@ class TestDecryptDocuments(TestCase):
             PASSPHRASE="test"
         ).enable()
 
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
+        doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg",  mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", "0000002.png.gpg"), os.path.join(thumb_dir, "0000002.png.gpg"))
 
-        Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg"))
 
         call_command('decrypt_documents')
 
-        doc = Document.objects.get(id=2)
+        doc.refresh_from_db()
 
         self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
         self.assertEqual(doc.filename, "0000002.pdf")
         self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf")))
         self.assertTrue(os.path.isfile(doc.source_path))
-        self.assertTrue(os.path.isfile(os.path.join(thumb_dir, "0000002.png")))
+        self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png")))
         self.assertTrue(os.path.isfile(doc.thumbnail_path))
 
         with doc.source_file as f:

src/documents/tests/test_management_exporter.py

@@ -24,13 +24,14 @@ class TestExportImport(DirectoriesMixin, TestCase):
 
         file = os.path.join(self.dirs.originals_dir, "0000001.pdf")
 
-        Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", id=1, mime_type="application/pdf")
+        Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf")
-        Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", id=2, mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
+        Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
         Tag.objects.create(name="t")
         DocumentType.objects.create(name="dt")
         Correspondent.objects.create(name="c")
 
         target = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, target)
 
         call_command('document_exporter', target)
 
@@ -66,9 +67,6 @@ class TestExportImport(DirectoriesMixin, TestCase):
     def test_export_missing_files(self):
 
         target = tempfile.mkdtemp()
-        Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", id=3, mime_type="application/pdf")
+        self.addCleanup(shutil.rmtree, target)
+        Document.objects.create(checksum="AAAAAAAAAAAAAAAAA", title="wow", filename="0000004.pdf", mime_type="application/pdf")
         self.assertRaises(FileNotFoundError, call_command, 'document_exporter', target)
-
-    def test_duplicate_titles(self):
-        # TODO
-        pass

src/documents/tests/test_management_retagger.py

@@ -14,6 +14,12 @@ class TestRetagger(DirectoriesMixin, TestCase):
 
         self.tag_first = Tag.objects.create(name="tag1", match="first", matching_algorithm=Tag.MATCH_ANY)
         self.tag_second = Tag.objects.create(name="tag2", match="second", matching_algorithm=Tag.MATCH_ANY)
+        self.tag_inbox = Tag.objects.create(name="test", is_inbox_tag=True)
+        self.tag_no_match = Tag.objects.create(name="test2")
+
+        self.d3.tags.add(self.tag_inbox)
+        self.d3.tags.add(self.tag_no_match)
+
 
         self.correspondent_first = Correspondent.objects.create(
             name="c1", match="first", matching_algorithm=Correspondent.MATCH_ANY)
@@ -38,7 +44,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
 
         self.assertEqual(d_first.tags.count(), 1)
         self.assertEqual(d_second.tags.count(), 1)
-        self.assertEqual(d_unrelated.tags.count(), 0)
+        self.assertEqual(d_unrelated.tags.count(), 2)
 
         self.assertEqual(d_first.tags.first(), self.tag_first)
         self.assertEqual(d_second.tags.first(), self.tag_second)
@@ -56,3 +62,17 @@ class TestRetagger(DirectoriesMixin, TestCase):
 
         self.assertEqual(d_first.correspondent, self.correspondent_first)
         self.assertEqual(d_second.correspondent, self.correspondent_second)
+
+    def test_overwrite_preserve_inbox(self):
+        self.d1.tags.add(self.tag_second)
+
+        call_command('document_retagger', '--tags', '--overwrite')
+
+        d_first, d_second, d_unrelated = self.get_updated_docs()
+
+        self.assertIsNotNone(Tag.objects.get(id=self.tag_second.id))
+
+        self.assertCountEqual([tag.id for tag in d_first.tags.all()], [self.tag_first.id])
+        self.assertCountEqual([tag.id for tag in d_second.tags.all()], [self.tag_second.id])
+        self.assertCountEqual([tag.id for tag in d_unrelated.tags.all()], [self.tag_inbox.id, self.tag_no_match.id])
+

src/documents/views.py

@@ -1,11 +1,8 @@
-import logging
 import os
-import re
 import tempfile
 from datetime import datetime
 from time import mktime
 
-import pikepdf
 from django.conf import settings
 from django.db.models import Count, Max
 from django.http import HttpResponse, HttpResponseBadRequest, Http404
@@ -42,6 +39,7 @@ from .filters import (
     LogFilterSet
 )
 from .models import Correspondent, Document, Log, Tag, DocumentType
+from .parsers import get_parser_class_for_mime_type
 from .serialisers import (
     CorrespondentSerializer,
     DocumentSerializer,
@@ -163,34 +161,16 @@ class DocumentViewSet(RetrieveModelMixin,
             disposition, filename)
         return response
 
-    def get_metadata(self, file, type):
+    def get_metadata(self, file, mime_type):
         if not os.path.isfile(file):
             return None
 
-        namespace_pattern = re.compile(r"\{(.*)\}(.*)")
+        parser_class = get_parser_class_for_mime_type(mime_type)
-
+        if parser_class:
-        result = []
+            parser = parser_class(logging_group=None)
-        if type == 'application/pdf':
+            return parser.extract_metadata(file, mime_type)
-            pdf = pikepdf.open(file)
+        else:
-            meta = pdf.open_metadata()
+            return []
-            for key, value in meta.items():
-                if isinstance(value, list):
-                    value = " ".join([str(e) for e in value])
-                value = str(value)
-                try:
-                    m = namespace_pattern.match(key)
-                    result.append({
-                        "namespace": m.group(1),
-                        "prefix": meta.REVERSE_NS[m.group(1)],
-                        "key": m.group(2),
-                        "value": value
-                    })
-                except Exception as e:
-                    logging.getLogger(__name__).warning(
-                        f"Error while reading metadata {key}: {value}. Error: "
-                        f"{e}"
-                    )
-        return result
 
     @action(methods=['get'], detail=True)
     def metadata(self, request, pk=None):

src/paperless/settings.py

@@ -210,6 +210,12 @@ AUTH_PASSWORD_VALIDATORS = [
 
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None
 
+COOKIE_PREFIX = os.getenv("PAPERLESS_COOKIE_PREFIX", "")
+
+CSRF_COOKIE_NAME = f"{COOKIE_PREFIX}csrftoken"
+SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid"
+LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language"
+
 ###############################################################################
 # Database                                                                    #
 ###############################################################################

src/paperless/version.py

@@ -1 +1 @@
-__version__ = (0, 9, 5)
+__version__ = (0, 9, 6)

src/paperless_mail/mail.py

@@ -103,10 +103,7 @@ class MailAccountHandler(LoggingMixin):
 
     def _correspondent_from_name(self, name):
         try:
-            return Correspondent.objects.get_or_create(
+            return Correspondent.objects.get_or_create(name=name)[0]
-                name=name, defaults={
-                    "slug": slugify(name)
-                })[0]
         except DatabaseError as e:
             self.log(
                 "error",

src/paperless_tesseract/parsers.py

@@ -5,6 +5,7 @@ import subprocess
 
 import ocrmypdf
 import pdftotext
+import pikepdf
 from PIL import Image
 from django.conf import settings
 from ocrmypdf import InputFileError, EncryptedPdfError
@@ -18,6 +19,33 @@ class RasterisedDocumentParser(DocumentParser):
     image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.)
     """
 
+    def extract_metadata(self, document_path, mime_type):
+        namespace_pattern = re.compile(r"\{(.*)\}(.*)")
+
+        result = []
+        if mime_type == 'application/pdf':
+            pdf = pikepdf.open(document_path)
+            meta = pdf.open_metadata()
+            for key, value in meta.items():
+                if isinstance(value, list):
+                    value = " ".join([str(e) for e in value])
+                value = str(value)
+                try:
+                    m = namespace_pattern.match(key)
+                    result.append({
+                        "namespace": m.group(1),
+                        "prefix": meta.REVERSE_NS[m.group(1)],
+                        "key": m.group(2),
+                        "value": value
+                    })
+                except Exception as e:
+                    self.log(
+                        "warning",
+                        f"Error while reading metadata {key}: {value}. Error: "
+                        f"{e}"
+                    )
+        return result
+
     def get_thumbnail(self, document_path, mime_type):
         """
         The thumbnail of a PDF is just a 500px wide image of the first page.
@@ -82,6 +110,24 @@ class RasterisedDocumentParser(DocumentParser):
                 f"Error while getting DPI from image {image}: {e}")
             return None
 
+    def calculate_a4_dpi(self, image):
+        try:
+            with Image.open(image) as im:
+                width, height = im.size
+                # divide image width by A4 width (210mm) in inches.
+                dpi = int(width / (21 / 2.54))
+                self.log(
+                    'debug',
+                    f"Estimated DPI {dpi} based on image width {width}"
+                )
+                return dpi
+
+        except Exception as e:
+            self.log(
+                'warning',
+                f"Error while calculating DPI for image {image}: {e}")
+            return None
+
     def parse(self, document_path, mime_type):
         mode = settings.OCR_MODE
 
@@ -134,6 +180,7 @@ class RasterisedDocumentParser(DocumentParser):
 
         if self.is_image(mime_type):
             dpi = self.get_dpi(document_path)
+            a4_dpi = self.calculate_a4_dpi(document_path)
             if dpi:
                 self.log(
                     "debug",
@@ -142,6 +189,8 @@ class RasterisedDocumentParser(DocumentParser):
                 ocr_args['image_dpi'] = dpi
             elif settings.OCR_IMAGE_DPI:
                 ocr_args['image_dpi'] = settings.OCR_IMAGE_DPI
+            elif a4_dpi:
+                ocr_args['image_dpi'] = a4_dpi
             else:
                 raise ParseError(
                     f"Cannot produce archive PDF for image {document_path}, "
@@ -213,6 +262,9 @@ def strip_excess_whitespace(text):
 
 def get_text_from_pdf(pdf_file):
 
+    if not os.path.isfile(pdf_file):
+        return None
+
     with open(pdf_file, "rb") as f:
         try:
             pdf = pdftotext.PDF(f)

src/paperless_tesseract/tests/test_parser.py

@@ -164,8 +164,21 @@ class TestParser(DirectoriesMixin, TestCase):
 
         self.assertRaises(ParseError, f)
 
+    @mock.patch("paperless_tesseract.parsers.ocrmypdf.ocr")
+    def test_image_calc_a4_dpi(self, m):
+        parser = RasterisedDocumentParser(None)
 
-    def test_image_no_dpi_fail(self):
+        parser.parse(os.path.join(self.SAMPLE_FILES, "simple-no-dpi.png"), "image/png")
+
+        m.assert_called_once()
+
+        args, kwargs = m.call_args
+
+        self.assertEqual(kwargs['image_dpi'], 62)
+
+    @mock.patch("paperless_tesseract.parsers.RasterisedDocumentParser.calculate_a4_dpi")
+    def test_image_dpi_fail(self, m):
+        m.return_value = None
         parser = RasterisedDocumentParser(None)
 
         def f():