+ Update checking works by pinging the the public Github API for the latest release to determine whether a new version is available.
+ Actual updating of the app must still be performed manually.
+
+
+ No tracking data is collected by the app in any way.
+
+
+
+
+
Bulk editing
@@ -205,5 +220,5 @@
-
+
diff --git a/src-ui/src/app/components/manage/settings/settings.component.ts b/src-ui/src/app/components/manage/settings/settings.component.ts
index dcdb6bc64..e033bdc6a 100644
--- a/src-ui/src/app/components/manage/settings/settings.component.ts
+++ b/src-ui/src/app/components/manage/settings/settings.component.ts
@@ -1,4 +1,11 @@
-import { Component, Inject, LOCALE_ID, OnInit, OnDestroy } from '@angular/core'
+import {
+ Component,
+ Inject,
+ LOCALE_ID,
+ OnInit,
+ OnDestroy,
+ AfterViewInit,
+} from '@angular/core'
import { FormControl, FormGroup } from '@angular/forms'
import { PaperlessSavedView } from 'src/app/data/paperless-saved-view'
import { DocumentListViewService } from 'src/app/services/document-list-view.service'
@@ -9,8 +16,18 @@ import {
} from 'src/app/services/settings.service'
import { Toast, ToastService } from 'src/app/services/toast.service'
import { dirtyCheck, DirtyComponent } from '@ngneat/dirty-check-forms'
-import { Observable, Subscription, BehaviorSubject, first } from 'rxjs'
+import {
+ Observable,
+ Subscription,
+ BehaviorSubject,
+ first,
+ tap,
+ takeUntil,
+ Subject,
+} from 'rxjs'
import { SETTINGS_KEYS } from 'src/app/data/paperless-uisettings'
+import { ActivatedRoute } from '@angular/router'
+import { ViewportScroller } from '@angular/common'
import { ForwardRefHandling } from '@angular/compiler'
@Component({
@@ -18,7 +35,9 @@ import { ForwardRefHandling } from '@angular/compiler'
templateUrl: './settings.component.html',
styleUrls: ['./settings.component.scss'],
})
-export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
+export class SettingsComponent
+ implements OnInit, AfterViewInit, OnDestroy, DirtyComponent
+{
savedViewGroup = new FormGroup({})
settingsForm = new FormGroup({
@@ -40,6 +59,7 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
notificationsConsumerFailed: new FormControl(null),
notificationsConsumerSuppressOnDashboard: new FormControl(null),
commentsEnabled: new FormControl(null),
+ updateCheckingEnabled: new FormControl(null),
})
savedViews: PaperlessSavedView[]
@@ -47,7 +67,9 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
store: BehaviorSubject
storeSub: Subscription
isDirty$: Observable
- isDirty: Boolean = false
+ isDirty: boolean = false
+ unsubscribeNotifier: Subject = new Subject()
+ savePending: boolean = false
get computedDateLocale(): string {
return (
@@ -57,29 +79,28 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
)
}
- get displayLanguageIsDirty(): boolean {
- return (
- this.settingsForm.get('displayLanguage').value !=
- this.store?.getValue()['displayLanguage']
- )
- }
-
constructor(
public savedViewService: SavedViewService,
private documentListViewService: DocumentListViewService,
private toastService: ToastService,
private settings: SettingsService,
- @Inject(LOCALE_ID) public currentLocale: string
+ @Inject(LOCALE_ID) public currentLocale: string,
+ private viewportScroller: ViewportScroller,
+ private activatedRoute: ActivatedRoute
) {
- this.settings.changed.subscribe({
- next: () => {
- this.settingsForm.patchValue(this.getCurrentSettings(), {
- emitEvent: false,
- })
- },
+ this.settings.settingsSaved.subscribe(() => {
+ if (!this.savePending) this.initialize()
})
}
+ ngAfterViewInit(): void {
+ if (this.activatedRoute.snapshot.fragment) {
+ this.viewportScroller.scrollToAnchor(
+ this.activatedRoute.snapshot.fragment
+ )
+ }
+ }
+
private getCurrentSettings() {
return {
bulkEditConfirmationDialogs: this.settings.get(
@@ -91,7 +112,6 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
documentListItemPerPage: this.settings.get(
SETTINGS_KEYS.DOCUMENT_LIST_SIZE
),
- slimSidebarEnabled: this.settings.get(SETTINGS_KEYS.SLIM_SIDEBAR),
darkModeUseSystem: this.settings.get(SETTINGS_KEYS.DARK_MODE_USE_SYSTEM),
darkModeEnabled: this.settings.get(SETTINGS_KEYS.DARK_MODE_ENABLED),
darkModeInvertThumbs: this.settings.get(
@@ -118,55 +138,68 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
SETTINGS_KEYS.NOTIFICATIONS_CONSUMER_SUPPRESS_ON_DASHBOARD
),
commentsEnabled: this.settings.get(SETTINGS_KEYS.COMMENTS_ENABLED),
+ updateCheckingEnabled: this.settings.get(
+ SETTINGS_KEYS.UPDATE_CHECKING_ENABLED
+ ),
}
}
ngOnInit() {
this.savedViewService.listAll().subscribe((r) => {
this.savedViews = r.results
- let storeData = this.getCurrentSettings()
+ this.initialize()
+ })
+ }
- for (let view of this.savedViews) {
- storeData.savedViews[view.id.toString()] = {
- id: view.id,
- name: view.name,
- show_on_dashboard: view.show_on_dashboard,
- show_in_sidebar: view.show_in_sidebar,
- }
- this.savedViewGroup.addControl(
- view.id.toString(),
- new FormGroup({
- id: new FormControl(null),
- name: new FormControl(null),
- show_on_dashboard: new FormControl(null),
- show_in_sidebar: new FormControl(null),
- })
- )
+ initialize() {
+ this.unsubscribeNotifier.next(true)
+
+ let storeData = this.getCurrentSettings()
+
+ for (let view of this.savedViews) {
+ storeData.savedViews[view.id.toString()] = {
+ id: view.id,
+ name: view.name,
+ show_on_dashboard: view.show_on_dashboard,
+ show_in_sidebar: view.show_in_sidebar,
}
+ this.savedViewGroup.addControl(
+ view.id.toString(),
+ new FormGroup({
+ id: new FormControl(null),
+ name: new FormControl(null),
+ show_on_dashboard: new FormControl(null),
+ show_in_sidebar: new FormControl(null),
+ })
+ )
+ }
- this.store = new BehaviorSubject(storeData)
+ this.store = new BehaviorSubject(storeData)
- this.storeSub = this.store.asObservable().subscribe((state) => {
- this.settingsForm.patchValue(state, { emitEvent: false })
- })
+ this.storeSub = this.store.asObservable().subscribe((state) => {
+ this.settingsForm.patchValue(state, { emitEvent: false })
+ })
- // Initialize dirtyCheck
- this.isDirty$ = dirtyCheck(this.settingsForm, this.store.asObservable())
+ // Initialize dirtyCheck
+ this.isDirty$ = dirtyCheck(this.settingsForm, this.store.asObservable())
- // Record dirty in case we need to 'undo' appearance settings if not saved on close
- this.isDirty$.subscribe((dirty) => {
+ // Record dirty in case we need to 'undo' appearance settings if not saved on close
+ this.isDirty$
+ .pipe(takeUntil(this.unsubscribeNotifier))
+ .subscribe((dirty) => {
this.isDirty = dirty
})
- // "Live" visual changes prior to save
- this.settingsForm.valueChanges.subscribe(() => {
+ // "Live" visual changes prior to save
+ this.settingsForm.valueChanges
+ .pipe(takeUntil(this.unsubscribeNotifier))
+ .subscribe(() => {
this.settings.updateAppearanceSettings(
this.settingsForm.get('darkModeUseSystem').value,
this.settingsForm.get('darkModeEnabled').value,
this.settingsForm.get('themeColor').value
)
})
- })
}
ngOnDestroy() {
@@ -185,7 +218,14 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
}
private saveLocalSettings() {
- const reloadRequired = this.displayLanguageIsDirty // just this one, for now
+ this.savePending = true
+ const reloadRequired =
+ this.settingsForm.value.displayLanguage !=
+ this.store?.getValue()['displayLanguage'] || // displayLanguage is dirty
+ (this.settingsForm.value.updateCheckingEnabled !=
+ this.store?.getValue()['updateCheckingEnabled'] &&
+ this.settingsForm.value.updateCheckingEnabled) // update checking was turned on
+
this.settings.set(
SETTINGS_KEYS.BULK_EDIT_APPLY_ON_CLOSE,
this.settingsForm.value.bulkEditApplyOnClose
@@ -250,10 +290,15 @@ export class SettingsComponent implements OnInit, OnDestroy, DirtyComponent {
SETTINGS_KEYS.COMMENTS_ENABLED,
this.settingsForm.value.commentsEnabled
)
+ this.settings.set(
+ SETTINGS_KEYS.UPDATE_CHECKING_ENABLED,
+ this.settingsForm.value.updateCheckingEnabled
+ )
this.settings.setLanguage(this.settingsForm.value.displayLanguage)
this.settings
.storeSettings()
.pipe(first())
+ .pipe(tap(() => (this.savePending = false)))
.subscribe({
next: () => {
this.store.next(this.settingsForm.value)
diff --git a/src-ui/src/app/data/matching-model.ts b/src-ui/src/app/data/matching-model.ts
index e228893d6..8ce05528e 100644
--- a/src-ui/src/app/data/matching-model.ts
+++ b/src-ui/src/app/data/matching-model.ts
@@ -6,6 +6,7 @@ export const MATCH_LITERAL = 3
export const MATCH_REGEX = 4
export const MATCH_FUZZY = 5
export const MATCH_AUTO = 6
+export const DEFAULT_MATCHING_ALGORITHM = MATCH_AUTO
export const MATCHING_ALGORITHMS = [
{
diff --git a/src-ui/src/app/data/paperless-document.ts b/src-ui/src/app/data/paperless-document.ts
index f5f83868c..8b038d79e 100644
--- a/src-ui/src/app/data/paperless-document.ts
+++ b/src-ui/src/app/data/paperless-document.ts
@@ -29,8 +29,6 @@ export interface PaperlessDocument extends ObjectWithId {
content?: string
- file_type?: string
-
tags$?: Observable
tags?: number[]
@@ -47,7 +45,7 @@ export interface PaperlessDocument extends ObjectWithId {
added?: Date
- file_name?: string
+ original_file_name?: string
download_url?: string
diff --git a/src-ui/src/app/data/paperless-uisettings.ts b/src-ui/src/app/data/paperless-uisettings.ts
index a5fdef51f..403d11f08 100644
--- a/src-ui/src/app/data/paperless-uisettings.ts
+++ b/src-ui/src/app/data/paperless-uisettings.ts
@@ -38,6 +38,9 @@ export const SETTINGS_KEYS = {
'general-settings:notifications:consumer-suppress-on-dashboard',
COMMENTS_ENABLED: 'general-settings:comments-enabled',
SLIM_SIDEBAR: 'general-settings:slim-sidebar',
+ UPDATE_CHECKING_ENABLED: 'general-settings:update-checking:enabled',
+ UPDATE_CHECKING_BACKEND_SETTING:
+ 'general-settings:update-checking:backend-setting',
}
export const SETTINGS: PaperlessUiSetting[] = [
@@ -126,4 +129,14 @@ export const SETTINGS: PaperlessUiSetting[] = [
type: 'boolean',
default: true,
},
+ {
+ key: SETTINGS_KEYS.UPDATE_CHECKING_ENABLED,
+ type: 'boolean',
+ default: false,
+ },
+ {
+ key: SETTINGS_KEYS.UPDATE_CHECKING_BACKEND_SETTING,
+ type: 'string',
+ default: '',
+ },
]
diff --git a/src-ui/src/app/services/rest/remote-version.service.ts b/src-ui/src/app/services/rest/remote-version.service.ts
index ab1b5a66b..9b1def363 100644
--- a/src-ui/src/app/services/rest/remote-version.service.ts
+++ b/src-ui/src/app/services/rest/remote-version.service.ts
@@ -6,7 +6,6 @@ import { environment } from 'src/environments/environment'
export interface AppRemoteVersion {
version: string
update_available: boolean
- feature_is_set: boolean
}
@Injectable({
diff --git a/src-ui/src/app/services/settings.service.ts b/src-ui/src/app/services/settings.service.ts
index ae48e373d..10cc93ce0 100644
--- a/src-ui/src/app/services/settings.service.ts
+++ b/src-ui/src/app/services/settings.service.ts
@@ -47,7 +47,7 @@ export class SettingsService {
public displayName: string
- public changed = new EventEmitter()
+ public settingsSaved: EventEmitter = new EventEmitter()
constructor(
rendererFactory: RendererFactory2,
@@ -316,13 +316,7 @@ export class SettingsService {
)
}
- get(key: string): any {
- let setting = SETTINGS.find((s) => s.key == key)
-
- if (!setting) {
- return null
- }
-
+ private getSettingRawValue(key: string): any {
let value = null
// parse key:key:key into nested object
const keys = key.replace('general-settings:', '').split(':')
@@ -333,6 +327,17 @@ export class SettingsService {
if (index == keys.length - 1) value = settingObj[keyPart]
else settingObj = settingObj[keyPart]
})
+ return value
+ }
+
+ get(key: string): any {
+ let setting = SETTINGS.find((s) => s.key == key)
+
+ if (!setting) {
+ return null
+ }
+
+ let value = this.getSettingRawValue(key)
if (value != null) {
switch (setting.type) {
@@ -362,10 +367,19 @@ export class SettingsService {
})
}
+ private settingIsSet(key: string): boolean {
+ let value = this.getSettingRawValue(key)
+ return value != null
+ }
+
storeSettings(): Observable {
- return this.http
- .post(this.baseUrl, { settings: this.settings })
- .pipe(tap((result) => this.changed.emit(!!result.success)))
+ return this.http.post(this.baseUrl, { settings: this.settings }).pipe(
+ tap((results) => {
+ if (results.success) {
+ this.settingsSaved.emit()
+ }
+ })
+ )
}
maybeMigrateSettings() {
@@ -405,5 +419,31 @@ export class SettingsService {
},
})
}
+
+ if (
+ !this.settingIsSet(SETTINGS_KEYS.UPDATE_CHECKING_ENABLED) &&
+ this.get(SETTINGS_KEYS.UPDATE_CHECKING_BACKEND_SETTING) != 'default'
+ ) {
+ this.set(
+ SETTINGS_KEYS.UPDATE_CHECKING_ENABLED,
+ this.get(SETTINGS_KEYS.UPDATE_CHECKING_BACKEND_SETTING).toString() ===
+ 'true'
+ )
+
+ this.storeSettings()
+ .pipe(first())
+ .subscribe({
+ error: (e) => {
+ this.toastService.showError(
+ 'Error migrating update checking setting'
+ )
+ console.log(e)
+ },
+ })
+ }
+ }
+
+ get updateCheckingIsSet(): boolean {
+ return this.settingIsSet(SETTINGS_KEYS.UPDATE_CHECKING_ENABLED)
}
}
diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts
index ecfed31c7..a2c3b2bc9 100644
--- a/src-ui/src/environments/environment.prod.ts
+++ b/src-ui/src/environments/environment.prod.ts
@@ -5,7 +5,7 @@ export const environment = {
apiBaseUrl: document.baseURI + 'api/',
apiVersion: '2',
appTitle: 'Paperless-ngx',
- version: '1.9.0-dev',
+ version: '1.9.2-dev',
webSocketHost: window.location.host,
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
webSocketBaseUrl: base_url.pathname + 'ws/',
diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss
index 0ffb12afb..b66710dcd 100644
--- a/src-ui/src/styles.scss
+++ b/src-ui/src/styles.scss
@@ -540,6 +540,25 @@ a.badge {
border-color: var(--bs-primary);
}
+.btn-group-xs {
+ > .btn {
+ padding: 0.2rem 0.25rem;
+ font-size: 0.675rem;
+ line-height: 1.2;
+ border-radius: 0.15rem;
+ }
+
+ > .btn:not(:first-child) {
+ border-top-left-radius: 0;
+ border-bottom-left-radius: 0;
+ }
+
+ > .btn:not(:last-child) {
+ border-top-right-radius: 0;
+ border-bottom-right-radius: 0;
+ }
+}
+
code {
color: var(--pngx-body-color-accent)
}
diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index d1b01290a..609ebed83 100644
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -111,14 +111,16 @@ class Consumer(LoggingMixin):
def pre_check_duplicate(self):
with open(self.path, "rb") as f:
checksum = hashlib.md5(f.read()).hexdigest()
- if Document.objects.filter(
+ existing_doc = Document.objects.filter(
Q(checksum=checksum) | Q(archive_checksum=checksum),
- ).exists():
+ )
+ if existing_doc.exists():
if settings.CONSUMER_DELETE_DUPLICATES:
os.unlink(self.path)
self._fail(
MESSAGE_DOCUMENT_ALREADY_EXISTS,
- f"Not consuming {self.filename}: It is a duplicate.",
+ f"Not consuming {self.filename}: It is a duplicate of"
+ f" {existing_doc.get().title} (#{existing_doc.get().pk})",
)
def pre_check_directories(self):
diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py
index a1db44791..ca28240b0 100644
--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@@ -608,6 +608,15 @@ class UiSettingsViewSerializer(serializers.ModelSerializer):
"settings",
]
+ def validate_settings(self, settings):
+ # we never save update checking backend setting
+ if "update_checking" in settings:
+ try:
+ settings["update_checking"].pop("backend_setting")
+ except KeyError:
+ pass
+ return settings
+
def create(self, validated_data):
ui_settings = UiSettings.objects.update_or_create(
user=validated_data.get("user"),
diff --git a/src/documents/tasks.py b/src/documents/tasks.py
index 94b849456..72b8b5244 100644
--- a/src/documents/tasks.py
+++ b/src/documents/tasks.py
@@ -112,10 +112,22 @@ def consume_file(
newname = f"{str(n)}_" + override_filename
else:
newname = None
+
+ # If the file is an upload, it's in the scratch directory
+ # Move it to consume directory to be picked up
+ # Otherwise, use the current parent to keep possible tags
+ # from subdirectories
+ try:
+ # is_relative_to would be nicer, but new in 3.9
+ _ = path.relative_to(settings.SCRATCH_DIR)
+ save_to_dir = settings.CONSUMPTION_DIR
+ except ValueError:
+ save_to_dir = path.parent
+
barcodes.save_to_dir(
document,
newname=newname,
- target_dir=path.parent,
+ target_dir=save_to_dir,
)
# Delete the PDF file which was split
diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py
index 4fc90b72e..9ad8dd118 100644
--- a/src/documents/tests/test_api.py
+++ b/src/documents/tests/test_api.py
@@ -1581,7 +1581,11 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, 200)
self.assertDictEqual(
response.data["settings"],
- {},
+ {
+ "update_checking": {
+ "backend_setting": "default",
+ },
+ },
)
def test_api_set_ui_settings(self):
@@ -2542,38 +2546,6 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase):
def setUp(self):
super().setUp()
- def test_remote_version_default(self):
- response = self.client.get(self.ENDPOINT)
-
- self.assertEqual(response.status_code, 200)
- self.assertDictEqual(
- response.data,
- {
- "version": "0.0.0",
- "update_available": False,
- "feature_is_set": False,
- },
- )
-
- @override_settings(
- ENABLE_UPDATE_CHECK=False,
- )
- def test_remote_version_disabled(self):
- response = self.client.get(self.ENDPOINT)
-
- self.assertEqual(response.status_code, 200)
- self.assertDictEqual(
- response.data,
- {
- "version": "0.0.0",
- "update_available": False,
- "feature_is_set": True,
- },
- )
-
- @override_settings(
- ENABLE_UPDATE_CHECK=True,
- )
@mock.patch("urllib.request.urlopen")
def test_remote_version_enabled_no_update_prefix(self, urlopen_mock):
@@ -2591,13 +2563,9 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase):
{
"version": "1.6.0",
"update_available": False,
- "feature_is_set": True,
},
)
- @override_settings(
- ENABLE_UPDATE_CHECK=True,
- )
@mock.patch("urllib.request.urlopen")
def test_remote_version_enabled_no_update_no_prefix(self, urlopen_mock):
@@ -2617,13 +2585,9 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase):
{
"version": version.__full_version_str__,
"update_available": False,
- "feature_is_set": True,
},
)
- @override_settings(
- ENABLE_UPDATE_CHECK=True,
- )
@mock.patch("urllib.request.urlopen")
def test_remote_version_enabled_update(self, urlopen_mock):
@@ -2650,13 +2614,9 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase):
{
"version": new_version_str,
"update_available": True,
- "feature_is_set": True,
},
)
- @override_settings(
- ENABLE_UPDATE_CHECK=True,
- )
@mock.patch("urllib.request.urlopen")
def test_remote_version_bad_json(self, urlopen_mock):
@@ -2674,13 +2634,9 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase):
{
"version": "0.0.0",
"update_available": False,
- "feature_is_set": True,
},
)
- @override_settings(
- ENABLE_UPDATE_CHECK=True,
- )
@mock.patch("urllib.request.urlopen")
def test_remote_version_exception(self, urlopen_mock):
@@ -2698,7 +2654,6 @@ class TestApiRemoteVersion(DirectoriesMixin, APITestCase):
{
"version": "0.0.0",
"update_available": False,
- "feature_is_set": True,
},
)
diff --git a/src/documents/views.py b/src/documents/views.py
index e301ab5f6..e32a93725 100644
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -261,6 +261,9 @@ class DocumentViewSet(
file_handle = doc.source_file
filename = doc.get_public_filename()
mime_type = doc.mime_type
+ # Support browser previewing csv files by using text mime type
+ if mime_type in {"application/csv", "text/csv"} and disposition == "inline":
+ mime_type = "text/plain"
if doc.storage_type == Document.STORAGE_TYPE_GPG:
file_handle = GnuPG.decrypted(file_handle)
@@ -780,42 +783,38 @@ class RemoteVersionView(GenericAPIView):
remote_version = "0.0.0"
is_greater_than_current = False
current_version = packaging_version.parse(version.__full_version_str__)
- # TODO: this can likely be removed when frontend settings are saved to DB
- feature_is_set = settings.ENABLE_UPDATE_CHECK != "default"
- if feature_is_set and settings.ENABLE_UPDATE_CHECK:
- try:
- req = urllib.request.Request(
- "https://api.github.com/repos/paperless-ngx/"
- "paperless-ngx/releases/latest",
- )
- # Ensure a JSON response
- req.add_header("Accept", "application/json")
-
- with urllib.request.urlopen(req) as response:
- remote = response.read().decode("utf-8")
- try:
- remote_json = json.loads(remote)
- remote_version = remote_json["tag_name"]
- # Basically PEP 616 but that only went in 3.9
- if remote_version.startswith("ngx-"):
- remote_version = remote_version[len("ngx-") :]
- except ValueError:
- logger.debug("An error occurred parsing remote version json")
- except urllib.error.URLError:
- logger.debug("An error occurred checking for available updates")
-
- is_greater_than_current = (
- packaging_version.parse(
- remote_version,
- )
- > current_version
+ try:
+ req = urllib.request.Request(
+ "https://api.github.com/repos/paperless-ngx/"
+ "paperless-ngx/releases/latest",
)
+ # Ensure a JSON response
+ req.add_header("Accept", "application/json")
+
+ with urllib.request.urlopen(req) as response:
+ remote = response.read().decode("utf-8")
+ try:
+ remote_json = json.loads(remote)
+ remote_version = remote_json["tag_name"]
+ # Basically PEP 616 but that only went in 3.9
+ if remote_version.startswith("ngx-"):
+ remote_version = remote_version[len("ngx-") :]
+ except ValueError:
+ logger.debug("An error occurred parsing remote version json")
+ except urllib.error.URLError:
+ logger.debug("An error occurred checking for available updates")
+
+ is_greater_than_current = (
+ packaging_version.parse(
+ remote_version,
+ )
+ > current_version
+ )
return Response(
{
"version": remote_version,
"update_available": is_greater_than_current,
- "feature_is_set": feature_is_set,
},
)
@@ -848,15 +847,23 @@ class UiSettingsView(GenericAPIView):
displayname = user.username
if user.first_name or user.last_name:
displayname = " ".join([user.first_name, user.last_name])
- settings = {}
+ ui_settings = {}
if hasattr(user, "ui_settings"):
- settings = user.ui_settings.settings
+ ui_settings = user.ui_settings.settings
+ if "update_checking" in ui_settings:
+ ui_settings["update_checking"][
+ "backend_setting"
+ ] = settings.ENABLE_UPDATE_CHECK
+ else:
+ ui_settings["update_checking"] = {
+ "backend_setting": settings.ENABLE_UPDATE_CHECK,
+ }
return Response(
{
"user_id": user.id,
"username": user.username,
"display_name": displayname,
- "settings": settings,
+ "settings": ui_settings,
},
)
diff --git a/src/paperless/checks.py b/src/paperless/checks.py
index e592a5bf2..845ff2d0b 100644
--- a/src/paperless/checks.py
+++ b/src/paperless/checks.py
@@ -127,10 +127,10 @@ def settings_values_check(app_configs, **kwargs):
Error(f'OCR output type "{settings.OCR_OUTPUT_TYPE}" is not valid'),
)
- if settings.OCR_MODE not in {"force", "skip", "redo_ocr"}:
+ if settings.OCR_MODE not in {"force", "skip", "redo", "skip_noarchive"}:
msgs.append(Error(f'OCR output mode "{settings.OCR_MODE}" is not valid'))
- if settings.OCR_CLEAN not in {"clean", "clean_final"}:
+ if settings.OCR_CLEAN not in {"clean", "clean-final", "none"}:
msgs.append(Error(f'OCR clean mode "{settings.OCR_CLEAN}" is not valid'))
return msgs
diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index ff9d350ce..e092b3f3e 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -319,6 +319,7 @@ DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": os.path.join(DATA_DIR, "db.sqlite3"),
+ "OPTIONS": {},
},
}
@@ -340,21 +341,18 @@ if os.getenv("PAPERLESS_DBHOST"):
# Leave room for future extensibility
if os.getenv("PAPERLESS_DBENGINE") == "mariadb":
engine = "django.db.backends.mysql"
- options = {"read_default_file": "/etc/mysql/my.cnf"}
+ options = {"read_default_file": "/etc/mysql/my.cnf", "charset": "utf8mb4"}
else: # Default to PostgresDB
engine = "django.db.backends.postgresql_psycopg2"
options = {"sslmode": os.getenv("PAPERLESS_DBSSLMODE", "prefer")}
DATABASES["default"]["ENGINE"] = engine
- for key, value in options.items():
- DATABASES["default"]["OPTIONS"][key] = value
+ DATABASES["default"]["OPTIONS"].update(options)
if os.getenv("PAPERLESS_DB_TIMEOUT") is not None:
- _new_opts = {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))}
- if "OPTIONS" in DATABASES["default"]:
- DATABASES["default"]["OPTIONS"].update(_new_opts)
- else:
- DATABASES["default"]["OPTIONS"] = _new_opts
+ DATABASES["default"]["OPTIONS"].update(
+ {"timeout": float(os.getenv("PAPERLESS_DB_TIMEOUT"))},
+ )
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
diff --git a/src/paperless/version.py b/src/paperless/version.py
index 1642e3f89..d196c358d 100644
--- a/src/paperless/version.py
+++ b/src/paperless/version.py
@@ -1,7 +1,7 @@
from typing import Final
from typing import Tuple
-__version__: Final[Tuple[int, int, int]] = (1, 9, 0)
+__version__: Final[Tuple[int, int, int]] = (1, 9, 2)
# Version string like X.Y.Z
__full_version_str__: Final[str] = ".".join(map(str, __version__))
# Version string like X.Y
diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py
index ebab59a88..f28586a2a 100644
--- a/src/paperless_mail/mail.py
+++ b/src/paperless_mail/mail.py
@@ -4,6 +4,7 @@ import tempfile
from datetime import date
from datetime import timedelta
from fnmatch import fnmatch
+from typing import Dict
import magic
import pathvalidate
@@ -30,7 +31,7 @@ class MailError(Exception):
class BaseMailAction:
- def get_criteria(self):
+ def get_criteria(self) -> Dict:
return {}
def post_consume(self, M, message_uids, parameter):
@@ -78,7 +79,7 @@ class TagMailAction(BaseMailAction):
M.flag(message_uids, [self.keyword], True)
-def get_rule_action(rule):
+def get_rule_action(rule) -> BaseMailAction:
if rule.action == MailRule.MailAction.FLAG:
return FlagMailAction()
elif rule.action == MailRule.MailAction.DELETE:
@@ -108,7 +109,7 @@ def make_criterias(rule):
return {**criterias, **get_rule_action(rule).get_criteria()}
-def get_mailbox(server, port, security):
+def get_mailbox(server, port, security) -> MailBox:
if security == MailAccount.ImapSecurity.NONE:
mailbox = MailBoxUnencrypted(server, port)
elif security == MailAccount.ImapSecurity.STARTTLS:
@@ -167,7 +168,7 @@ class MailAccountHandler(LoggingMixin):
"Unknown correspondent selector",
) # pragma: nocover
- def handle_mail_account(self, account):
+ def handle_mail_account(self, account: MailAccount):
self.renew_logging_group()
@@ -181,7 +182,14 @@ class MailAccountHandler(LoggingMixin):
account.imap_security,
) as M:
+ supports_gmail_labels = "X-GM-EXT-1" in M.client.capabilities
+ supports_auth_plain = "AUTH=PLAIN" in M.client.capabilities
+
+ self.log("debug", f"GMAIL Label Support: {supports_gmail_labels}")
+ self.log("debug", f"AUTH=PLAIN Support: {supports_auth_plain}")
+
try:
+
M.login(account.username, account.password)
except UnicodeEncodeError:
@@ -215,7 +223,11 @@ class MailAccountHandler(LoggingMixin):
for rule in account.rules.order_by("order"):
try:
- total_processed_files += self.handle_mail_rule(M, rule)
+ total_processed_files += self.handle_mail_rule(
+ M,
+ rule,
+ supports_gmail_labels,
+ )
except Exception as e:
self.log(
"error",
@@ -233,7 +245,12 @@ class MailAccountHandler(LoggingMixin):
return total_processed_files
- def handle_mail_rule(self, M: MailBox, rule):
+ def handle_mail_rule(
+ self,
+ M: MailBox,
+ rule: MailRule,
+ supports_gmail_labels: bool = False,
+ ):
self.log("debug", f"Rule {rule}: Selecting folder {rule.folder}")
@@ -261,11 +278,19 @@ class MailAccountHandler(LoggingMixin):
) from err
criterias = make_criterias(rule)
- criterias_imap = AND(**criterias)
+
+ # Deal with the Gmail label extension
if "gmail_label" in criterias:
+
gmail_label = criterias["gmail_label"]
del criterias["gmail_label"]
- criterias_imap = AND(NOT(gmail_label=gmail_label), **criterias)
+
+ if not supports_gmail_labels:
+ criterias_imap = AND(**criterias)
+ else:
+ criterias_imap = AND(NOT(gmail_label=gmail_label), **criterias)
+ else:
+ criterias_imap = AND(**criterias)
self.log(
"debug",
diff --git a/src/paperless_mail/tests/test_live_mail.py b/src/paperless_mail/tests/test_live_mail.py
new file mode 100644
index 000000000..1af870156
--- /dev/null
+++ b/src/paperless_mail/tests/test_live_mail.py
@@ -0,0 +1,70 @@
+import os
+
+import pytest
+from django.test import TestCase
+from paperless_mail.mail import MailAccountHandler
+from paperless_mail.mail import MailError
+from paperless_mail.models import MailAccount
+from paperless_mail.models import MailRule
+
+# Only run if the environment is setup
+# And the environment is not empty (forks, I think)
+@pytest.mark.skipif(
+ "PAPERLESS_MAIL_TEST_HOST" not in os.environ
+ or not len(os.environ["PAPERLESS_MAIL_TEST_HOST"]),
+ reason="Live server testing not enabled",
+)
+class TestMailLiveServer(TestCase):
+ def setUp(self) -> None:
+
+ self.mail_account_handler = MailAccountHandler()
+ self.account = MailAccount.objects.create(
+ name="test",
+ imap_server=os.environ["PAPERLESS_MAIL_TEST_HOST"],
+ username=os.environ["PAPERLESS_MAIL_TEST_USER"],
+ password=os.environ["PAPERLESS_MAIL_TEST_PASSWD"],
+ imap_port=993,
+ )
+
+ return super().setUp()
+
+ def tearDown(self) -> None:
+ self.account.delete()
+ return super().tearDown()
+
+ def test_process_non_gmail_server_flag(self):
+
+ try:
+ rule1 = MailRule.objects.create(
+ name="testrule",
+ account=self.account,
+ action=MailRule.MailAction.FLAG,
+ )
+
+ self.mail_account_handler.handle_mail_account(self.account)
+
+ rule1.delete()
+
+ except MailError as e:
+ self.fail(f"Failure: {e}")
+ except Exception as e:
+ pass
+
+ def test_process_non_gmail_server_tag(self):
+
+ try:
+
+ rule2 = MailRule.objects.create(
+ name="testrule",
+ account=self.account,
+ action=MailRule.MailAction.TAG,
+ )
+
+ self.mail_account_handler.handle_mail_account(self.account)
+
+ rule2.delete()
+
+ except MailError as e:
+ self.fail(f"Failure: {e}")
+ except Exception as e:
+ pass
diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py
index be016a79a..997184fd2 100644
--- a/src/paperless_mail/tests/test_mail.py
+++ b/src/paperless_mail/tests/test_mail.py
@@ -47,15 +47,16 @@ class BogusFolderManager:
class BogusClient:
+ def __init__(self, messages):
+ self.messages: List[MailMessage] = messages
+ self.capabilities: List[str] = []
+
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
pass
- def __init__(self, messages):
- self.messages: List[MailMessage] = messages
-
def authenticate(self, mechanism, authobject):
# authobject must be a callable object
auth_bytes = authobject(None)
@@ -80,12 +81,6 @@ class BogusMailBox(ContextManager):
# Note the non-ascii characters here
UTF_PASSWORD: str = "w57äöüw4b6huwb6nhu"
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- pass
-
def __init__(self):
self.messages: List[MailMessage] = []
self.messages_spam: List[MailMessage] = []
@@ -93,6 +88,12 @@ class BogusMailBox(ContextManager):
self.client = BogusClient(self.messages)
self._host = ""
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
+
def updateClient(self):
self.client = BogusClient(self.messages)
@@ -648,6 +649,7 @@ class TestMail(DirectoriesMixin, TestCase):
def test_handle_mail_account_tag_gmail(self):
self.bogus_mailbox._host = "imap.gmail.com"
+ self.bogus_mailbox.client.capabilities = ["X-GM-EXT-1"]
account = MailAccount.objects.create(
name="test",
diff --git a/src/paperless_text/signals.py b/src/paperless_text/signals.py
index 3d025c14e..94d1ccc2e 100644
--- a/src/paperless_text/signals.py
+++ b/src/paperless_text/signals.py
@@ -11,5 +11,6 @@ def text_consumer_declaration(sender, **kwargs):
"mime_types": {
"text/plain": ".txt",
"text/csv": ".csv",
+ "application/csv": ".csv",
},
}
diff --git a/src/paperless_tika/tests/samples/sample.docx b/src/paperless_tika/tests/samples/sample.docx
new file mode 100644
index 000000000..be6f33313
Binary files /dev/null and b/src/paperless_tika/tests/samples/sample.docx differ
diff --git a/src/paperless_tika/tests/samples/sample.odt b/src/paperless_tika/tests/samples/sample.odt
new file mode 100644
index 000000000..f0c291aa4
Binary files /dev/null and b/src/paperless_tika/tests/samples/sample.odt differ
diff --git a/src/paperless_tika/tests/test_live_tika.py b/src/paperless_tika/tests/test_live_tika.py
new file mode 100644
index 000000000..e1af7cf86
--- /dev/null
+++ b/src/paperless_tika/tests/test_live_tika.py
@@ -0,0 +1,78 @@
+import datetime
+import os
+from pathlib import Path
+from typing import Final
+
+import pytest
+from django.test import TestCase
+from paperless_tika.parsers import TikaDocumentParser
+
+
+@pytest.mark.skipif("TIKA_LIVE" not in os.environ, reason="No tika server")
+class TestTikaParserAgainstServer(TestCase):
+ """
+ This test case tests the Tika parsing against a live tika server,
+ if the environment contains the correct value indicating such a server
+ is available.
+ """
+
+ SAMPLE_DIR: Final[Path] = (Path(__file__).parent / Path("samples")).resolve()
+
+ def setUp(self) -> None:
+ self.parser = TikaDocumentParser(logging_group=None)
+
+ def tearDown(self) -> None:
+ self.parser.cleanup()
+
+ def test_basic_parse_odt(self):
+ """
+ GIVEN:
+ - An input ODT format document
+ WHEN:
+ - The document is parsed
+ THEN:
+ - Document content is correct
+ - Document date is correct
+ """
+ test_file = self.SAMPLE_DIR / Path("sample.odt")
+
+ self.parser.parse(test_file, "application/vnd.oasis.opendocument.text")
+
+ self.assertEqual(
+ self.parser.text,
+ "This is an ODT test document, created September 14, 2022",
+ )
+ self.assertIsNotNone(self.parser.archive_path)
+ with open(self.parser.archive_path, "rb") as f:
+ # PDFs begin with the bytes PDF-x.y
+ self.assertTrue(b"PDF-" in f.read()[:10])
+
+ # TODO: Unsure what can set the Creation-Date field in a document, enable when possible
+ # self.assertEqual(self.parser.date, datetime.datetime(2022, 9, 14))
+
+ def test_basic_parse_docx(self):
+ """
+ GIVEN:
+ - An input DOCX format document
+ WHEN:
+ - The document is parsed
+ THEN:
+ - Document content is correct
+ - Document date is correct
+ """
+ test_file = self.SAMPLE_DIR / Path("sample.docx")
+
+ self.parser.parse(
+ test_file,
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ )
+
+ self.assertEqual(
+ self.parser.text,
+ "This is an DOCX test document, also made September 14, 2022",
+ )
+ self.assertIsNotNone(self.parser.archive_path)
+ with open(self.parser.archive_path, "rb") as f:
+ self.assertTrue(b"PDF-" in f.read()[:10])
+
+ # self.assertEqual(self.parser.date, datetime.datetime(2022, 9, 14))