diff --git a/.dockerignore b/.dockerignore index 7534368f0..c00be4161 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,7 @@ -src-ui/node_modules -src-ui/dist +/src-ui/node_modules +/src-ui/dist .git +/export +/consume +/media +/data diff --git a/.gitignore b/.gitignore index 871a7bd08..25c7c421a 100644 --- a/.gitignore +++ b/.gitignore @@ -65,7 +65,6 @@ target/ .virtualenv virtualenv /venv -docker-compose.yml docker-compose.env # Used for development diff --git a/Dockerfile b/Dockerfile index 015d511d4..bb96305f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,6 @@ COPY Pipfile* ./ #Dependencies RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get -y --no-install-recommends install \ - anacron \ build-essential \ curl \ ghostscript \ @@ -60,20 +59,17 @@ RUN apt-get update \ COPY scripts/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml COPY scripts/gunicorn.conf.py ./ COPY scripts/supervisord.conf /etc/supervisord.conf -COPY scripts/paperless-cron /etc/cron.daily/ COPY scripts/docker-entrypoint.sh /sbin/docker-entrypoint.sh # copy app COPY src/ ./src/ -COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ ./src/documents/static/ +COPY --from=frontend /usr/src/paperless/src-ui/dist/paperless-ui/ ./src/documents/static/frontend/ # add users, setup scripts RUN addgroup --gid 1000 paperless \ && useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \ && chown -R paperless:paperless . \ - && chmod 755 /sbin/docker-entrypoint.sh \ - && chmod +x /etc/cron.daily/paperless-cron \ - && rm /etc/cron.daily/apt-compat /etc/cron.daily/dpkg + && chmod 755 /sbin/docker-entrypoint.sh WORKDIR /usr/src/paperless/src/ @@ -81,6 +77,6 @@ RUN sudo -HEu paperless python3 manage.py collectstatic --clear --no-input VOLUME ["/usr/src/paperless/data", "/usr/src/paperless/consume", "/usr/src/paperless/export"] ENTRYPOINT ["/sbin/docker-entrypoint.sh"] -CMD ["python3", "manage.py", "--help"] +CMD ["supervisord", "-c", "/etc/supervisord.conf"] LABEL maintainer="Jonas Winkler " diff --git a/Pipfile b/Pipfile index 09de36334..fdfe63595 100644 --- a/Pipfile +++ b/Pipfile @@ -24,8 +24,11 @@ gunicorn = "*" whitenoise = "*" fuzzywuzzy = "*" python-Levenshtein = "*" -django-extensions = "" +django-extensions = "*" watchdog = "*" +pathvalidate = "*" +django-q = "*" +redis = "*" channels = "~=3.0" channels-redis = "*" daphne = "~=3.0" diff --git a/README.md b/README.md index b925fa89b..6f013a913 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This is a list of changes that have been made to the original project. ## Added - **A new single page UI** built with bootstrap and Angular. Its much more responsive than the django admin pages. It features the follwing improvements over the old django admin interface: + - *Dashboard.* The landing page shows some useful information, such as statistics, recently scanned documents, file uploading, and possibly more in the future. - *Document uploading on the web page.* This is very crude right now, but gets the job done. It simply uploads the documents and stores them in the configured consumer directory. The API for that has always been in the project, there simply was no form on the UI to support it. - *Full text search* with a proper document indexer: The search feature sorts documents by relevance to the search query, highlights query terms in the found documents and provides autocomplete while typing the query. This is still very basic but will see extensions in the future. - *Saveable filters.* Save filter and sorting presets and optionally display a couple documents of saved filters (i.e., your inbox sorted descending by added date, or tagged TODO, oldest to newest) on the dash board. @@ -49,21 +50,18 @@ This is a list of changes that have been made to the original project. These features were removed each due to two reasons. First, I did not feel these features contributed all that much to the over project, and second, I don't want to maintain these features. - **(BREAKING) Reminders.** I have no idea what they were used for and thus removed them from the project. -- **Filename handling (I'm sorry).** The master branch of the paperless project has seen some changes regarding the filename handling of stored documents. These changes allow you to change the filename of stored documents from their default form ‘{id}.pdf’. These changes have not made it into this project, since the whole point of paperless is that you don't have to access your documents on the disk anymore. If you are using version 2.7.0, this does not affect you. If you are on the most recent push on the master branch, the provided migration will revert these changes and rename all your files to their original file name. - **Every customization made to the admin interface.** Since this is not the primary interface for the application anymore, there is no need to keep and maintain these. Besides, some changes were incompatible with the most recent versions of django. The interface is completely usable, though. ## Planned These features will make it into the application at some point, sorted by priority. -- **Better tag editor.** The tag editor on the document detail page is not very convenient. This was put in there to get the project working but will be replaced with something nicer eventually. - **More search.** The search backend is incredibly versatile and customizable. Searching is the most important feature of this project and thus, I want to implement things like: - Group and limit search results by correspondent, show “more from this” links in the results. - Ability to search for “Similar documents” in the search results - Provide corrections for mispelled queries - **More robust consumer** that shows its progress on the web page. - **Arbitrary tag colors**. Allow the selection of any color with a color picker. -- **Dashboard**. The landing page is a little bleak right now but will feature status updates about the consumer, previews of saved filters and database statistics in the future. ## On the chopping block. diff --git a/docker-compose.env.example b/docker-compose.env.example index cc2a1d3ec..fb529898a 100644 --- a/docker-compose.env.example +++ b/docker-compose.env.example @@ -1,7 +1,3 @@ -# Database settings for paperless -# If you want to use sqlite instead, remove this setting. -PAPERLESS_DBHOST="db" - # The UID and GID of the user used to run paperless in the container. Set this # to your UID and GID on the host so that you have write access to the # consumption directory. diff --git a/docker-compose.yml.example b/docker-compose.yml similarity index 83% rename from docker-compose.yml.example rename to docker-compose.yml index 3d8ed4719..f9b4d6c33 100644 --- a/docker-compose.yml.example +++ b/docker-compose.yml @@ -2,8 +2,7 @@ version: "3.4" services: broker: image: redis:latest - ports: - - 6379:6379 + #restart: always db: image: postgres:13 @@ -16,13 +15,12 @@ services: POSTGRES_PASSWORD: paperless webserver: - build: . - image: paperless-ng + image: paperless-ng:latest #restart: always depends_on: - db ports: - - "8000:8000" + - 8000:8000 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000"] interval: 30s @@ -34,6 +32,9 @@ services: - ./export:/usr/src/paperless/export - ./consume:/usr/src/paperless/consume env_file: docker-compose.env + environment: + PAPERLESS_REDIS: redis://broker:6379 + PAPERLESS_DBHOST: db command: ["supervisord", "-c", "/etc/supervisord.conf"] diff --git a/paperless.conf.example b/paperless.conf.example index 41ac778e7..48df40ab2 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -3,6 +3,16 @@ # As this file contains passwords it should only be readable by the user # running paperless. +############################################################################### +#### Message Broker #### +############################################################################### + +# This is required for processing scheduled tasks such as email fetching, index +# optimization and for training the automatic document matcher. +# Defaults to localhost:6379. +#PAPERLESS_REDIS="redis://localhost:6379" + + ############################################################################### #### Database Settings #### ############################################################################### @@ -63,7 +73,19 @@ PAPERLESS_CONSUMPTION_DIR="../consume" # Any email sent to the target account that does not contain this text will be # ignored. -#PAPERLESS_EMAIL_SECRET="" +PAPERLESS_EMAIL_SECRET="" + +# Specify a filename format for the document (directories are supported) +# Use the following placeholders: +# * {correspondent} +# * {title} +# * {created} +# * {added} +# * {tags[KEY]} If your tags conform to key_value or key-value +# * {tags[INDEX]} If your tags are strings, select the tag by index +# Uniqueness of filenames is ensured, as an incrementing counter is attached +# to each filename. +#PAPERLESS_FILENAME_FORMAT="" ############################################################################### #### Security #### @@ -117,11 +139,6 @@ PAPERLESS_CONSUMPTION_DIR="../consume" # https://docs.djangoproject.com/en/1.11/ref/settings/#force-script-name #PAPERLESS_FORCE_SCRIPT_NAME="" -# If you are using alternative authentication means or are just using paperless -# as a single user on a small private network, this option allows you to disable -# user authentication if you set it to "true" -#PAPERLESS_DISABLE_LOGIN="false" - ############################################################################### #### Software Tweaks #### ############################################################################### diff --git a/scripts/paperless-cron b/scripts/paperless-cron deleted file mode 100644 index 238857227..000000000 --- a/scripts/paperless-cron +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -cd /usr/src/paperless/src - -sudo -HEu paperless python3 manage.py document_create_classifier diff --git a/scripts/supervisord.conf b/scripts/supervisord.conf index cb6fd1650..ae2a77513 100644 --- a/scripts/supervisord.conf +++ b/scripts/supervisord.conf @@ -24,8 +24,9 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 -[program:anacron] -command=anacron -d +[program:scheduler] +command=python3 manage.py qcluster +user=paperless stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 diff --git a/src-ui/angular.json b/src-ui/angular.json index 6135ffa91..aca54b8e0 100644 --- a/src-ui/angular.json +++ b/src-ui/angular.json @@ -14,6 +14,7 @@ "builder": "@angular-devkit/build-angular:browser", "options": { "outputPath": "dist/paperless-ui", + "outputHashing": "none", "index": "src/index.html", "main": "src/main.ts", "polyfills": "src/polyfills.ts", @@ -38,7 +39,7 @@ } ], "optimization": true, - "outputHashing": "all", + "outputHashing": "none", "sourceMap": false, "extractCss": true, "namedChunks": false, diff --git a/src-ui/package-lock.json b/src-ui/package-lock.json index 1d73a856f..45b1d2d6d 100644 --- a/src-ui/package-lock.json +++ b/src-ui/package-lock.json @@ -2049,9 +2049,9 @@ } }, "@ng-bootstrap/ng-bootstrap": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/@ng-bootstrap/ng-bootstrap/-/ng-bootstrap-7.0.0.tgz", - "integrity": "sha512-SxUaptGWJmCxM0d2Zy1mx7K7p/YBwGZ69NmmBQVY4BE6p5av0hWrVmv9rzzfBz0rhxU7RPZLor2Jpaoq8Xyl4w==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@ng-bootstrap/ng-bootstrap/-/ng-bootstrap-8.0.0.tgz", + "integrity": "sha512-v77Gfd8xHH+exq0WqIqVRlxbUEHdA/2+RUJenUP2IDTQN9E1rWl7O461/kosr+0XPuxPArHQJxhh/WsCYckcNg==", "requires": { "tslib": "^2.0.0" } diff --git a/src-ui/package.json b/src-ui/package.json index b2da7eabe..a9e909155 100644 --- a/src-ui/package.json +++ b/src-ui/package.json @@ -20,7 +20,7 @@ "@angular/platform-browser": "~10.1.5", "@angular/platform-browser-dynamic": "~10.1.5", "@angular/router": "~10.1.5", - "@ng-bootstrap/ng-bootstrap": "^7.0.0", + "@ng-bootstrap/ng-bootstrap": "^8.0.0", "bootstrap": "^4.5.0", "ng-bootstrap": "^1.6.3", "ngx-file-drop": "^10.0.0", diff --git a/src-ui/src/app/app-routing.module.ts b/src-ui/src/app/app-routing.module.ts index fde8fd31f..27f0629b4 100644 --- a/src-ui/src/app/app-routing.module.ts +++ b/src-ui/src/app/app-routing.module.ts @@ -4,7 +4,6 @@ import { AppFrameComponent } from './components/app-frame/app-frame.component'; import { DashboardComponent } from './components/dashboard/dashboard.component'; import { DocumentDetailComponent } from './components/document-detail/document-detail.component'; import { DocumentListComponent } from './components/document-list/document-list.component'; -import { LoginComponent } from './components/login/login.component'; import { CorrespondentListComponent } from './components/manage/correspondent-list/correspondent-list.component'; import { DocumentTypeListComponent } from './components/manage/document-type-list/document-type-list.component'; import { LogsComponent } from './components/manage/logs/logs.component'; @@ -12,25 +11,23 @@ import { SettingsComponent } from './components/manage/settings/settings.compone import { TagListComponent } from './components/manage/tag-list/tag-list.component'; import { NotFoundComponent } from './components/not-found/not-found.component'; import { SearchComponent } from './components/search/search.component'; -import { AuthGuardService } from './services/auth-guard.service'; const routes: Routes = [ {path: '', redirectTo: 'dashboard', pathMatch: 'full'}, {path: '', component: AppFrameComponent, children: [ - {path: 'dashboard', component: DashboardComponent, canActivate: [AuthGuardService] }, - {path: 'documents', component: DocumentListComponent, canActivate: [AuthGuardService] }, - {path: 'view/:id', component: DocumentListComponent, canActivate: [AuthGuardService] }, - {path: 'search', component: SearchComponent, canActivate: [AuthGuardService] }, - {path: 'documents/:id', component: DocumentDetailComponent, canActivate: [AuthGuardService] }, + {path: 'dashboard', component: DashboardComponent }, + {path: 'documents', component: DocumentListComponent }, + {path: 'view/:id', component: DocumentListComponent }, + {path: 'search', component: SearchComponent }, + {path: 'documents/:id', component: DocumentDetailComponent }, - {path: 'tags', component: TagListComponent, canActivate: [AuthGuardService] }, - {path: 'documenttypes', component: DocumentTypeListComponent, canActivate: [AuthGuardService] }, - {path: 'correspondents', component: CorrespondentListComponent, canActivate: [AuthGuardService] }, - {path: 'logs', component: LogsComponent, canActivate: [AuthGuardService] }, - {path: 'settings', component: SettingsComponent, canActivate: [AuthGuardService] }, + {path: 'tags', component: TagListComponent }, + {path: 'documenttypes', component: DocumentTypeListComponent }, + {path: 'correspondents', component: CorrespondentListComponent }, + {path: 'logs', component: LogsComponent }, + {path: 'settings', component: SettingsComponent }, ]}, - {path: 'login', component: LoginComponent }, {path: '404', component: NotFoundComponent}, {path: '**', redirectTo: '/404', pathMatch: 'full'} ]; diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index 5cc59d567..5a3cfde19 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -12,7 +12,6 @@ import { TagListComponent } from './components/manage/tag-list/tag-list.componen import { DocumentTypeListComponent } from './components/manage/document-type-list/document-type-list.component'; import { LogsComponent } from './components/manage/logs/logs.component'; import { SettingsComponent } from './components/manage/settings/settings.component'; -import { LoginComponent } from './components/login/login.component'; import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { DatePipe } from '@angular/common'; import { SafePipe } from './pipes/safe.pipe'; @@ -29,7 +28,6 @@ import { PageHeaderComponent } from './components/common/page-header/page-header import { AppFrameComponent } from './components/app-frame/app-frame.component'; import { ToastsComponent } from './components/common/toasts/toasts.component'; import { FilterEditorComponent } from './components/filter-editor/filter-editor.component'; -import { AuthInterceptor } from './services/auth.interceptor'; import { DocumentCardLargeComponent } from './components/document-list/document-card-large/document-card-large.component'; import { DocumentCardSmallComponent } from './components/document-list/document-card-small/document-card-small.component'; import { NgxFileDropModule } from 'ngx-file-drop'; @@ -40,6 +38,7 @@ import { SaveViewConfigDialogComponent } from './components/document-list/save-v import { InfiniteScrollModule } from 'ngx-infinite-scroll'; import { DateTimeComponent } from './components/common/input/date-time/date-time.component'; import { TagsComponent } from './components/common/input/tags/tags.component'; +import { SortableDirective } from './directives/sortable.directive'; import { SavedViewWidgetComponent } from './components/dashboard/widgets/saved-view-widget/saved-view-widget.component'; import { ConsumerStatusWidgetComponent } from './components/dashboard/widgets/consumer-status-widget/consumer-status-widget.component'; import { StatisticsWidgetComponent } from './components/dashboard/widgets/statistics-widget/statistics-widget.component'; @@ -56,7 +55,6 @@ import { FileUploadWidgetComponent } from './components/dashboard/widgets/file-u DocumentTypeListComponent, LogsComponent, SettingsComponent, - LoginComponent, SafePipe, NotFoundComponent, CorrespondentEditDialogComponent, @@ -78,6 +76,7 @@ import { FileUploadWidgetComponent } from './components/dashboard/widgets/file-u SaveViewConfigDialogComponent, DateTimeComponent, TagsComponent, + SortableDirective, ConsumerStatusWidgetComponent, SavedViewWidgetComponent, StatisticsWidgetComponent, @@ -94,12 +93,7 @@ import { FileUploadWidgetComponent } from './components/dashboard/widgets/file-u InfiniteScrollModule ], providers: [ - DatePipe, - { - provide: HTTP_INTERCEPTORS, - useClass: AuthInterceptor, - multi: true - } + DatePipe ], bootstrap: [AppComponent] }) diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index ad12a9d43..0b18777ef 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -10,7 +10,7 @@