diff --git a/docs/api.md b/docs/api.md index bf9e88659..e5da43a5c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -278,39 +278,39 @@ attribute with various information about the search results: ### Filtering by custom fields You can filter documents by their custom field values by specifying the -`custom_field_lookup` query parameter. Here are some recipes for common +`custom_field_query` query parameter. Here are some recipes for common use cases: 1. Documents with a custom field "due" (date) between Aug 1, 2024 and Sept 1, 2024 (inclusive): - `?custom_field_lookup=["due", "range", ["2024-08-01", "2024-09-01"]]` + `?custom_field_query=["due", "range", ["2024-08-01", "2024-09-01"]]` 2. Documents with a custom field "customer" (text) that equals "bob" (case sensitive): - `?custom_field_lookup=["customer", "exact", "bob"]` + `?custom_field_query=["customer", "exact", "bob"]` 3. Documents with a custom field "answered" (boolean) set to `true`: - `?custom_field_lookup=["answered", "exact", true]` + `?custom_field_query=["answered", "exact", true]` 4. Documents with a custom field "favorite animal" (select) set to either "cat" or "dog": - `?custom_field_lookup=["favorite animal", "in", ["cat", "dog"]]` + `?custom_field_query=["favorite animal", "in", ["cat", "dog"]]` 5. Documents with a custom field "address" (text) that is empty: - `?custom_field_lookup=["OR", ["address", "isnull", true], ["address", "exact", ""]]` + `?custom_field_query=["OR", ["address", "isnull", true], ["address", "exact", ""]]` 6. Documents that don't have a field called "foo": - `?custom_field_lookup=["foo", "exists", false]` + `?custom_field_query=["foo", "exists", false]` 7. Documents that have document links "references" to both document 3 and 7: - `?custom_field_lookup=["references", "contains", [3, 7]]` + `?custom_field_query=["references", "contains", [3, 7]]` All field types support basic operations including `exact`, `in`, `isnull`, and `exists`. String, URL, and monetary fields support case-insensitive @@ -320,22 +320,6 @@ including `gt` (>), `gte` (>=), `lt` (<), `lte` (<=), and `range`. Lastly, document link fields support a `contains` operator that behaves like a "is superset of" check. -!!! warning - - It is possible to do case-insensitive exact match (i.e., `iexact`) and - case-sensitive substring match (i.e., `contains`, `startswith`, - `endswith`) for string, URL, and monetary fields, but - [they may not work as expected on some database backends](https://docs.djangoproject.com/en/5.1/ref/databases/#substring-matching-and-case-sensitivity). - - It is also possible to use regular expressions to match string, URL, and - monetary fields, but the syntax is database-dependent, and accepting - regular expressions from untrusted sources could make your instance - vulnerable to regular expression denial of service attacks. - - For these reasons the above expressions are disabled by default. - If you understand the implications, you may enable them by uncommenting - `PAPERLESS_CUSTOM_FIELD_LOOKUP_OPT_IN` in your configuration file. - ### `/api/search/autocomplete/` Get auto completions for a partial search term. diff --git a/paperless.conf.example b/paperless.conf.example index 5fabbf390..63ee7be22 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -81,7 +81,6 @@ #PAPERLESS_THUMBNAIL_FONT_NAME= #PAPERLESS_IGNORE_DATES= #PAPERLESS_ENABLE_UPDATE_CHECK= -#PAPERLESS_ALLOW_CUSTOM_FIELD_LOOKUP=iexact,contains,startswith,endswith,regex,iregex # Tika settings diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 9b588ac6b..3fffe4f6e 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -698,7 +698,7 @@ src/app/components/common/input/document-link/document-link.component.html - 38 + 51 src/app/components/common/permissions-dialog/permissions-dialog.component.html @@ -1031,7 +1031,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 143 + 152 @@ -1088,7 +1088,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.html - 110 + 105 src/app/components/manage/mail/mail.component.html @@ -3300,6 +3300,102 @@ 63 + + True + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 40 + + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 73 + + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 79 + + + + False + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 41 + + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 74 + + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 80 + + + + Search docs... + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 96 + + + + Any + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 126 + + + src/app/components/common/filterable-dropdown/filterable-dropdown.component.html + 17 + + + + All + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 128 + + + src/app/components/common/filterable-dropdown/filterable-dropdown.component.html + 15 + + + src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.html + 16 + + + src/app/components/common/permissions-select/permissions-select.component.html + 16 + + + src/app/components/common/permissions-select/permissions-select.component.html + 27 + + + src/app/components/document-list/bulk-editor/bulk-editor.component.html + 14 + + + + Not + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 131 + + + + Add query + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 150 + + + + Add expression + + src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html + 153 + + now @@ -4549,36 +4645,6 @@ 146 - - All - - src/app/components/common/filterable-dropdown/filterable-dropdown.component.html - 15 - - - src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.html - 16 - - - src/app/components/common/permissions-select/permissions-select.component.html - 16 - - - src/app/components/common/permissions-select/permissions-select.component.html - 27 - - - src/app/components/document-list/bulk-editor/bulk-editor.component.html - 14 - - - - Any - - src/app/components/common/filterable-dropdown/filterable-dropdown.component.html - 17 - - Include @@ -4668,7 +4734,7 @@ src/app/components/common/input/document-link/document-link.component.html - 9 + 12 src/app/components/common/input/file/file.component.html @@ -4740,14 +4806,14 @@ Remove link src/app/components/common/input/document-link/document-link.component.html - 30 + 43 Open link src/app/components/common/input/document-link/document-link.component.html - 31 + 44 src/app/components/common/input/url/url.component.html @@ -4761,6 +4827,13 @@ 44 + + Search for documents + + src/app/components/common/input/document-link/document-link.component.ts + 53 + + Selected items @@ -5834,7 +5907,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 131 + 140 src/app/data/document.ts @@ -6416,7 +6489,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 139 + 148 @@ -6425,10 +6498,6 @@ src/app/components/document-list/bulk-editor/bulk-editor.component.html 83 - - src/app/components/document-list/filter-editor/filter-editor.component.html - 90 - Merge @@ -6925,7 +6994,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.html - 116 + 111 @@ -6950,7 +7019,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 136 + 145 src/app/data/document.ts @@ -7126,161 +7195,154 @@ Dates src/app/components/document-list/filter-editor/filter-editor.component.html - 100 + 95 Title & content src/app/components/document-list/filter-editor/filter-editor.component.ts - 134 + 143 More like src/app/components/document-list/filter-editor/filter-editor.component.ts - 149 + 158 equals src/app/components/document-list/filter-editor/filter-editor.component.ts - 155 + 164 is empty src/app/components/document-list/filter-editor/filter-editor.component.ts - 159 + 168 is not empty src/app/components/document-list/filter-editor/filter-editor.component.ts - 163 + 172 greater than src/app/components/document-list/filter-editor/filter-editor.component.ts - 167 + 176 less than src/app/components/document-list/filter-editor/filter-editor.component.ts - 171 + 180 Correspondent: src/app/components/document-list/filter-editor/filter-editor.component.ts - 191,193 + 200,202 Without correspondent src/app/components/document-list/filter-editor/filter-editor.component.ts - 195 + 204 Document type: src/app/components/document-list/filter-editor/filter-editor.component.ts - 201,203 + 210,212 Without document type src/app/components/document-list/filter-editor/filter-editor.component.ts - 205 + 214 Storage path: src/app/components/document-list/filter-editor/filter-editor.component.ts - 211,213 + 220,222 Without storage path src/app/components/document-list/filter-editor/filter-editor.component.ts - 215 + 224 Tag: src/app/components/document-list/filter-editor/filter-editor.component.ts - 219,221 + 228,230 Without any tag src/app/components/document-list/filter-editor/filter-editor.component.ts - 225 + 234 - - Custom fields: + + Custom fields query src/app/components/document-list/filter-editor/filter-editor.component.ts - 229,231 - - - - Without any custom field - - src/app/components/document-list/filter-editor/filter-editor.component.ts - 235 + 238 Title: src/app/components/document-list/filter-editor/filter-editor.component.ts - 239 + 241 ASN: src/app/components/document-list/filter-editor/filter-editor.component.ts - 242 + 244 Owner: src/app/components/document-list/filter-editor/filter-editor.component.ts - 245 + 247 Owner not in: src/app/components/document-list/filter-editor/filter-editor.component.ts - 248 + 250 Without an owner src/app/components/document-list/filter-editor/filter-editor.component.ts - 251 + 253 @@ -8007,6 +8069,83 @@ 9 + + Equal to + + src/app/data/custom-field-query.ts + 24 + + + + In + + src/app/data/custom-field-query.ts + 25 + + + + Is null + + src/app/data/custom-field-query.ts + 26 + + + + Exists + + src/app/data/custom-field-query.ts + 27 + + + + Contains + + src/app/data/custom-field-query.ts + 28 + + + + Contains (case-insensitive) + + src/app/data/custom-field-query.ts + 29 + + + + Greater than + + src/app/data/custom-field-query.ts + 30 + + + + Greater than or equal to + + src/app/data/custom-field-query.ts + 31 + + + + Less than + + src/app/data/custom-field-query.ts + 32 + + + + Less than or equal to + + src/app/data/custom-field-query.ts + 33 + + + + Range + + src/app/data/custom-field-query.ts + 34 + + Boolean diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index 005de5369..93c458ae0 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -108,6 +108,7 @@ import { FileDropComponent } from './components/file-drop/file-drop.component' import { CustomFieldsComponent } from './components/manage/custom-fields/custom-fields.component' import { CustomFieldEditDialogComponent } from './components/common/edit-dialog/custom-field-edit-dialog/custom-field-edit-dialog.component' import { CustomFieldsDropdownComponent } from './components/common/custom-fields-dropdown/custom-fields-dropdown.component' +import { CustomFieldsQueryDropdownComponent } from './components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component' import { ProfileEditDialogComponent } from './components/common/profile-edit-dialog/profile-edit-dialog.component' import { PdfViewerModule } from 'ng2-pdf-viewer' import { DocumentLinkComponent } from './components/common/input/document-link/document-link.component' @@ -141,6 +142,7 @@ import { arrowRightShort, arrowUpRight, asterisk, + braces, bodyText, boxArrowUp, boxArrowUpRight, @@ -198,6 +200,7 @@ import { link, listTask, listUl, + nodePlus, pencil, people, peopleFill, @@ -227,6 +230,7 @@ import { uiRadios, upcScan, x, + xCircle, xLg, } from 'ngx-bootstrap-icons' @@ -242,6 +246,7 @@ const icons = { arrowRightShort, arrowUpRight, asterisk, + braces, bodyText, boxArrowUp, boxArrowUpRight, @@ -299,6 +304,7 @@ const icons = { link, listTask, listUl, + nodePlus, pencil, people, peopleFill, @@ -328,6 +334,7 @@ const icons = { uiRadios, upcScan, x, + xCircle, xLg, } @@ -485,6 +492,7 @@ function initializeApp(settings: SettingsService) { CustomFieldsComponent, CustomFieldEditDialogComponent, CustomFieldsDropdownComponent, + CustomFieldsQueryDropdownComponent, ProfileEditDialogComponent, DocumentLinkComponent, PreviewPopupComponent, diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html new file mode 100644 index 000000000..9da2886f4 --- /dev/null +++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.html @@ -0,0 +1,163 @@ +
+ +
+
+ @for (element of selectionModel.queries; track element.id; let i = $index) { +
+ @switch (element.type) { + @case (CustomFieldQueryComponentType.Atom) { + + } + @case (CustomFieldQueryComponentType.Expression) { + + } + } +
+ } +
+
+
+ + + @if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Date) { + + + } @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Float || getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Integer) { + + } @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Boolean) { + + } @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Select) { + + } @else { + + } + + + +
+ + + @switch (atom.operator) { + @case (CustomFieldQueryOperator.Exists) { + + } + @case (CustomFieldQueryOperator.IsNull) { + + } + @case (CustomFieldQueryOperator.GreaterThanOrEqual) { + + } + @case (CustomFieldQueryOperator.LessThanOrEqual) { + + } + @case (CustomFieldQueryOperator.GreaterThan) { + + } + @case (CustomFieldQueryOperator.LessThan) { + + } + @case (CustomFieldQueryOperator.Contains) { + + } + @case (CustomFieldQueryOperator.In) { + + } + @case (CustomFieldQueryOperator.Exact) { + + } + @default { + + } + } + +
+ + + +
+
+
+ + + + + @if (expression.negatable) { + + + } +
+
+ @for (element of expression.value; track element.id; let i = $index) { +
+ @switch (element.type) { + @case (CustomFieldQueryComponentType.Atom) { + + } + @case (CustomFieldQueryComponentType.Expression) { + + } + } +
+ } +
+
+
+ + + @if (expression.depth > 0) { + + } +
+
+ diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.scss b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.scss new file mode 100644 index 000000000..a10c4658d --- /dev/null +++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.scss @@ -0,0 +1,43 @@ +.dropdown-menu { + width: 370px; + @media(min-width: 768px) { + width: 600px; + } +} + +::ng-deep .ng-select-container { + border-top-right-radius: 0 !important; + border-bottom-right-radius: 0 !important; + height: 100% !important; +} + +::ng-deep .rounded-end .ng-select-container { + border-top-right-radius: var(--bs-border-radius) !important; + border-bottom-right-radius: var(--bs-border-radius) !important; + border-top-left-radius: 0 !important; + border-bottom-left-radius: 0 !important; +} + +::ng-deep .ng-select { + max-width: 100px; + min-width: 35%; + font-size: 14px; +} + +::ng-deep .doc-link-select { + padding-top: 0 !important; + border-top-right-radius: var(--bs-border-radius) !important; + border-bottom-right-radius: var(--bs-border-radius) !important; + background-image: none !important; + + .ng-select-container, + .ng-select.ng-select-opened > .ng-select-container { + border: none !important; + min-height: 34px !important; + background: none !important; + } + .ng-select { + max-width: 200px; + min-width: 140px; + } +} diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.spec.ts b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.spec.ts new file mode 100644 index 000000000..e6199c696 --- /dev/null +++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.spec.ts @@ -0,0 +1,320 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' +import { + CustomFieldQueriesModel, + CustomFieldsQueryDropdownComponent, +} from './custom-fields-query-dropdown.component' +import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service' +import { of } from 'rxjs' +import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field' +import { + CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP, + CustomFieldQueryLogicalOperator, + CustomFieldQueryOperatorGroups, +} from 'src/app/data/custom-field-query' +import { provideHttpClientTesting } from '@angular/common/http/testing' +import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http' +import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap' +import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons' +import { + CustomFieldQueryExpression, + CustomFieldQueryAtom, + CustomFieldQueryElement, +} from 'src/app/utils/custom-field-query-element' + +const customFields = [ + { + id: 1, + name: 'Test Field', + data_type: CustomFieldDataType.String, + extra_data: {}, + }, + { + id: 2, + name: 'Test Select Field', + data_type: CustomFieldDataType.Select, + extra_data: { select_options: ['Option 1', 'Option 2'] }, + }, +] + +describe('CustomFieldsQueryDropdownComponent', () => { + let component: CustomFieldsQueryDropdownComponent + let fixture: ComponentFixture + let customFieldsService: CustomFieldsService + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [CustomFieldsQueryDropdownComponent], + imports: [NgbDropdownModule, NgxBootstrapIconsModule.pick(allIcons)], + providers: [ + provideHttpClient(withInterceptorsFromDi()), + provideHttpClientTesting(), + ], + }).compileComponents() + + customFieldsService = TestBed.inject(CustomFieldsService) + jest.spyOn(customFieldsService, 'listAll').mockReturnValue( + of({ + count: customFields.length, + all: customFields.map((f) => f.id), + results: customFields, + }) + ) + fixture = TestBed.createComponent(CustomFieldsQueryDropdownComponent) + component = fixture.componentInstance + component.icon = 'ui-radios' + fixture.detectChanges() + }) + + it('should initialize custom fields on creation', () => { + expect(component.customFields).toEqual(customFields) + }) + + it('should add an expression when opened if queries are empty', () => { + component.selectionModel.clear() + component.onOpenChange(true) + expect(component.selectionModel.queries.length).toBe(1) + }) + + it('should support reset the selection model', () => { + component.selectionModel.addExpression() + component.reset() + expect(component.selectionModel.isEmpty()).toBeTruthy() + }) + + it('should get operators for a field', () => { + const field: CustomField = { + id: 1, + name: 'Test Field', + data_type: CustomFieldDataType.String, + extra_data: {}, + } + component.customFields = [field] + const operators = component.getOperatorsForField(1) + expect(operators.length).toEqual( + [ + ...CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP[ + CustomFieldQueryOperatorGroups.Basic + ], + ...CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP[ + CustomFieldQueryOperatorGroups.String + ], + ].length + ) + + // Fallback to basic operators if field is not found + const operators2 = component.getOperatorsForField(2) + expect(operators2.length).toEqual( + CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP[ + CustomFieldQueryOperatorGroups.Basic + ].length + ) + }) + + it('should get select options for a field', () => { + const field: CustomField = { + id: 1, + name: 'Test Field', + data_type: CustomFieldDataType.Select, + extra_data: { select_options: ['Option 1', 'Option 2'] }, + } + component.customFields = [field] + const options = component.getSelectOptionsForField(1) + expect(options).toEqual(['Option 1', 'Option 2']) + + // Fallback to empty array if field is not found + const options2 = component.getSelectOptionsForField(2) + expect(options2).toEqual([]) + }) + + it('should remove an element from the selection model', () => { + const expression = new CustomFieldQueryExpression() + const atom = new CustomFieldQueryAtom() + ;(expression.value as CustomFieldQueryElement[]).push(atom) + component.selectionModel.addExpression(expression) + component.removeElement(atom) + expect(component.selectionModel.isEmpty()).toBeTruthy() + const expression2 = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [ + [1, 'icontains', 'test'], + [2, 'icontains', 'test'], + ], + ]) + component.selectionModel.addExpression(expression2) + component.removeElement(expression2) + expect(component.selectionModel.isEmpty()).toBeTruthy() + }) + + it('should emit selectionModelChange when model changes', () => { + const nextSpy = jest.spyOn(component.selectionModelChange, 'next') + const atom = new CustomFieldQueryAtom([1, 'icontains', 'test']) + component.selectionModel.addAtom(atom) + atom.changed.next(atom) + expect(nextSpy).toHaveBeenCalled() + }) + + it('should complete selection model subscription when new selection model is set', () => { + const completeSpy = jest.spyOn(component.selectionModel.changed, 'complete') + const selectionModel = new CustomFieldQueriesModel() + component.selectionModel = selectionModel + expect(completeSpy).toHaveBeenCalled() + }) + + it('should support adding an atom', () => { + const expression = new CustomFieldQueryExpression() + component.addAtom(expression) + expect(expression.value.length).toBe(1) + }) + + it('should support adding an expression', () => { + const expression = new CustomFieldQueryExpression() + component.addExpression(expression) + expect(expression.value.length).toBe(1) + }) + + it('should support getting a custom field by ID', () => { + expect(component.getCustomFieldByID(1)).toEqual(customFields[0]) + }) + + it('should sanitize name from title', () => { + component.title = 'Test Title' + expect(component.name).toBe('test_title') + }) + + describe('CustomFieldQueriesModel', () => { + let model: CustomFieldQueriesModel + + beforeEach(() => { + model = new CustomFieldQueriesModel() + }) + + it('should initialize with empty queries', () => { + expect(model.queries).toEqual([]) + }) + + it('should clear queries and fire event', () => { + const nextSpy = jest.spyOn(model.changed, 'next') + model.addExpression() + model.clear() + expect(model.queries).toEqual([]) + expect(nextSpy).toHaveBeenCalledWith(model) + }) + + it('should clear queries without firing event', () => { + const nextSpy = jest.spyOn(model.changed, 'next') + model.addExpression() + model.clear(false) + expect(model.queries).toEqual([]) + expect(nextSpy).not.toHaveBeenCalled() + }) + + it('should validate an empty model as invalid', () => { + expect(model.isValid()).toBeFalsy() + }) + + it('should validate a model with valid expression as valid', () => { + const expression = new CustomFieldQueryExpression() + const atom = new CustomFieldQueryAtom([1, 'icontains', 'test']) + const atom2 = new CustomFieldQueryAtom([2, 'icontains', 'test']) + const expression2 = new CustomFieldQueryExpression() + expression2.addAtom(atom) + expression2.addAtom(atom2) + expression.addExpression(expression2) + model.addExpression(expression) + expect(model.isValid()).toBeTruthy() + }) + + it('should validate a model with invalid expression as invalid', () => { + const expression = new CustomFieldQueryExpression() + model.addExpression(expression) + expect(model.isValid()).toBeFalsy() + }) + + it('should validate an atom with in or contains operator', () => { + const atom = new CustomFieldQueryAtom([1, 'in', '[1,2,3]']) + expect(model['validateAtom'].apply(null, [atom])).toBeTruthy() + atom.operator = 'contains' + atom.value = [1, 2, 3] + expect(model['validateAtom'].apply(null, [atom])).toBeTruthy() + atom.value = null + expect(model['validateAtom'].apply(null, [atom])).toBeFalsy() + }) + + it('should check if model is empty', () => { + expect(model.isEmpty()).toBeTruthy() + model.addExpression() + expect(model.isEmpty()).toBeTruthy() + const atom = new CustomFieldQueryAtom([1, 'icontains', 'test']) + model.addAtom(atom) + expect(model.isEmpty()).toBeFalsy() + }) + + it('should add an atom to the model', () => { + const atom = new CustomFieldQueryAtom([1, 'icontains', 'test']) + model.addAtom(atom) + expect(model.queries.length).toBe(1) + expect( + (model.queries[0] as CustomFieldQueryExpression).value.length + ).toBe(1) + }) + + it('should add an expression to the model, propagate changes', () => { + const expression = new CustomFieldQueryExpression() + model.addExpression(expression) + expect(model.queries.length).toBe(1) + const expression2 = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [ + [1, 'icontains', 'test'], + [2, 'icontains', 'test'], + ], + ]) + model.addExpression(expression2) + const nextSpy = jest.spyOn(model.changed, 'next') + expression2.changed.next(expression2) + expect(nextSpy).toHaveBeenCalled() + }) + + it('should remove an element from the model', () => { + const expression = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [ + [1, 'icontains', 'test'], + [2, 'icontains', 'test'], + ], + ]) + const atom = new CustomFieldQueryAtom([1, 'icontains', 'test']) + const expression2 = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [ + [3, 'icontains', 'test'], + [4, 'icontains', 'test'], + ], + ]) + expression.addAtom(atom) + expression2.addExpression(expression) + model.addExpression(expression2) + model.removeElement(atom) + expect(model.queries.length).toBe(1) + model.removeElement(expression2) + }) + + it('should fire changed event when an atom changes', () => { + const nextSpy = jest.spyOn(model.changed, 'next') + const atom = new CustomFieldQueryAtom([1, 'icontains', 'test']) + model.addAtom(atom) + atom.changed.next(atom) + expect(nextSpy).toHaveBeenCalledWith(model) + }) + + it('should complete changed subject when element is removed', () => { + const expression = new CustomFieldQueryExpression() + const atom = new CustomFieldQueryAtom([1, 'icontains', 'test']) + ;(expression.value as CustomFieldQueryElement[]).push(atom) + model.addExpression(expression) + const completeSpy = jest.spyOn(atom.changed, 'complete') + model.removeElement(atom) + expect(completeSpy).toHaveBeenCalled() + }) + }) +}) diff --git a/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts new file mode 100644 index 000000000..923907158 --- /dev/null +++ b/src-ui/src/app/components/common/custom-fields-query-dropdown/custom-fields-query-dropdown.component.ts @@ -0,0 +1,294 @@ +import { + Component, + EventEmitter, + Input, + Output, + ViewChild, +} from '@angular/core' +import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap' +import { Subject, first, takeUntil } from 'rxjs' +import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field' +import { + CustomFieldQueryElementType, + CustomFieldQueryOperator, + CUSTOM_FIELD_QUERY_OPERATOR_GROUPS_BY_TYPE, + CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP, + CustomFieldQueryOperatorGroups, + CUSTOM_FIELD_QUERY_OPERATOR_LABELS, + CUSTOM_FIELD_QUERY_MAX_DEPTH, + CUSTOM_FIELD_QUERY_MAX_ATOMS, +} from 'src/app/data/custom-field-query' +import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service' +import { + CustomFieldQueryElement, + CustomFieldQueryExpression, + CustomFieldQueryAtom, +} from 'src/app/utils/custom-field-query-element' +import { popperOptionsReenablePreventOverflow } from 'src/app/utils/popper-options' + +export class CustomFieldQueriesModel { + public queries: CustomFieldQueryElement[] = [] + + public readonly changed = new Subject() + + public clear(fireEvent = true) { + this.queries = [] + if (fireEvent) { + this.changed.next(this) + } + } + + public isValid(): boolean { + return ( + this.queries.length > 0 && + this.validateExpression(this.queries[0] as CustomFieldQueryExpression) + ) + } + + public isEmpty(): boolean { + return ( + this.queries.length === 0 || + (this.queries.length === 1 && this.queries[0].value.length === 0) + ) + } + + private validateAtom(atom: CustomFieldQueryAtom) { + let valid = !!(atom.field && atom.operator && atom.value !== null) + if ( + [ + CustomFieldQueryOperator.In.valueOf(), + CustomFieldQueryOperator.Contains.valueOf(), + ].includes(atom.operator) && + atom.value + ) { + valid = valid && atom.value.length > 0 + } + return valid + } + + private validateExpression(expression: CustomFieldQueryExpression) { + return ( + expression.operator && + expression.value.length > 0 && + (expression.value as CustomFieldQueryElement[]).every((e) => + e.type === CustomFieldQueryElementType.Atom + ? this.validateAtom(e as CustomFieldQueryAtom) + : this.validateExpression(e as CustomFieldQueryExpression) + ) + ) + } + + public addAtom(atom: CustomFieldQueryAtom) { + if (this.queries.length === 0) { + this.addExpression() + } + ;(this.queries[0].value as CustomFieldQueryElement[]).push(atom) + atom.changed.subscribe(() => { + if (atom.field && atom.operator && atom.value) { + this.changed.next(this) + } + }) + } + + public addExpression( + expression: CustomFieldQueryExpression = new CustomFieldQueryExpression() + ) { + if (this.queries.length > 0) { + ;( + (this.queries[0] as CustomFieldQueryExpression) + .value as CustomFieldQueryElement[] + ).push(expression) + } else { + this.queries.push(expression) + } + expression.changed.subscribe(() => { + this.changed.next(this) + }) + } + + private findElement( + queryElement: CustomFieldQueryElement, + elements: any[] + ): CustomFieldQueryElement { + for (let i = 0; i < elements.length; i++) { + if (elements[i] === queryElement) { + return elements.splice(i, 1)[0] + } else if (elements[i].type === CustomFieldQueryElementType.Expression) { + return this.findElement( + queryElement, + elements[i].value as CustomFieldQueryElement[] + ) + } + } + } + + public removeElement(queryElement: CustomFieldQueryElement) { + let foundComponent + for (let i = 0; i < this.queries.length; i++) { + let query = this.queries[i] + if (query === queryElement) { + foundComponent = this.queries.splice(i, 1)[0] + break + } else if (query.type === CustomFieldQueryElementType.Expression) { + foundComponent = this.findElement(queryElement, query.value as any[]) + } + } + if (foundComponent) { + foundComponent.changed.complete() + if (this.isEmpty()) { + this.clear() + } + this.changed.next(this) + } + } +} + +@Component({ + selector: 'pngx-custom-fields-query-dropdown', + templateUrl: './custom-fields-query-dropdown.component.html', + styleUrls: ['./custom-fields-query-dropdown.component.scss'], +}) +export class CustomFieldsQueryDropdownComponent { + public CustomFieldQueryComponentType = CustomFieldQueryElementType + public CustomFieldQueryOperator = CustomFieldQueryOperator + public CustomFieldDataType = CustomFieldDataType + public CUSTOM_FIELD_QUERY_MAX_DEPTH = CUSTOM_FIELD_QUERY_MAX_DEPTH + public CUSTOM_FIELD_QUERY_MAX_ATOMS = CUSTOM_FIELD_QUERY_MAX_ATOMS + public popperOptions = popperOptionsReenablePreventOverflow + + @Input() + title: string + + @Input() + filterPlaceholder: string = '' + + @Input() + icon: string + + @Input() + allowSelectNone: boolean = false + + @Input() + editing = false + + @Input() + applyOnClose = false + + get name(): string { + return this.title ? this.title.replace(/\s/g, '_').toLowerCase() : null + } + + @Input() + disabled: boolean = false + + @ViewChild('dropdown') dropdown: NgbDropdown + + private _selectionModel: CustomFieldQueriesModel + + @Input() + set selectionModel(model: CustomFieldQueriesModel) { + if (this._selectionModel) { + this._selectionModel.changed.complete() + } + model.changed.subscribe(() => { + this.onModelChange() + }) + this._selectionModel = model + } + + get selectionModel(): CustomFieldQueriesModel { + return this._selectionModel + } + + private onModelChange() { + if (this.selectionModel.isEmpty() || this.selectionModel.isValid()) { + this.selectionModelChange.next(this.selectionModel) + this.selectionModel.isEmpty() && this.dropdown?.close() + } + } + + @Output() + selectionModelChange = new EventEmitter() + + customFields: CustomField[] = [] + + private unsubscribeNotifier: Subject = new Subject() + + constructor(protected customFieldsService: CustomFieldsService) { + this.selectionModel = new CustomFieldQueriesModel() + this.getFields() + this.reset() + } + + ngOnDestroy(): void { + this.unsubscribeNotifier.next(this) + this.unsubscribeNotifier.complete() + } + + public onOpenChange(open: boolean) { + if (open && this.selectionModel.queries.length === 0) { + this.selectionModel.addExpression() + } + } + + public get isActive(): boolean { + return ( + (this.selectionModel.queries[0] as CustomFieldQueryExpression)?.value + ?.length > 0 + ) + } + + private getFields() { + this.customFieldsService + .listAll() + .pipe(first(), takeUntil(this.unsubscribeNotifier)) + .subscribe((result) => { + this.customFields = result.results + }) + } + + public getCustomFieldByID(id: number): CustomField { + return this.customFields.find((field) => field.id === id) + } + + public addAtom(expression: CustomFieldQueryExpression) { + expression.addAtom() + } + + public addExpression(expression: CustomFieldQueryExpression) { + expression.addExpression() + } + + public removeElement(element: CustomFieldQueryElement) { + this.selectionModel.removeElement(element) + } + + public reset() { + this.selectionModel.clear(false) + this.selectionModel.changed.next(this.selectionModel) + } + + getOperatorsForField( + fieldID: number + ): Array<{ value: string; label: string }> { + const field = this.customFields.find((field) => field.id === fieldID) + const groups: CustomFieldQueryOperatorGroups[] = field + ? CUSTOM_FIELD_QUERY_OPERATOR_GROUPS_BY_TYPE[field.data_type] + : [CustomFieldQueryOperatorGroups.Basic] + const operators = groups.flatMap( + (group) => CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP[group] + ) + return operators.map((operator) => ({ + value: operator, + label: CUSTOM_FIELD_QUERY_OPERATOR_LABELS[operator], + })) + } + + getSelectOptionsForField(fieldID: number): string[] { + const field = this.customFields.find((field) => field.id === fieldID) + if (field) { + return field.extra_data['select_options'] + } + return [] + } +} diff --git a/src-ui/src/app/components/common/input/document-link/document-link.component.html b/src-ui/src/app/components/common/input/document-link/document-link.component.html index a8ecce4e6..94f4f21b4 100644 --- a/src-ui/src/app/components/common/input/document-link/document-link.component.html +++ b/src-ui/src/app/components/common/input/document-link/document-link.component.html @@ -1,50 +1,57 @@ -
-
-
- @if (title) { - - } - @if (removable) { - - } -
-
-
- - -
- - -  {{document.title}} - -
- - - - - - -
{{document.title}} ({{document.created | customDate:'shortDate'}})
- - +@if (minimal) { + +} @else { +
+
+
+ @if (title) { + + } + @if (removable) { + + } +
+
+ + @if (hint) { + {{hint}} + }
- @if (hint) { - {{hint}} - }
-
+} + + + + +
+ + +  {{document.title}} + +
+ + + + + + +
{{document.title}} ({{document.created | customDate:'shortDate'}})
+ + + diff --git a/src-ui/src/app/components/common/input/document-link/document-link.component.ts b/src-ui/src/app/components/common/input/document-link/document-link.component.ts index 83a6a742e..882aacad5 100644 --- a/src-ui/src/app/components/common/input/document-link/document-link.component.ts +++ b/src-ui/src/app/components/common/input/document-link/document-link.component.ts @@ -46,6 +46,12 @@ export class DocumentLinkComponent @Input() parentDocumentID: number + @Input() + minimal: boolean = false + + @Input() + placeholder: string = $localize`Search for documents` + constructor(private documentsService: DocumentService) { super() } diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html index e70f4c710..4eb9d179e 100644 --- a/src-ui/src/app/components/document-list/document-list.component.html +++ b/src-ui/src/app/components/document-list/document-list.component.html @@ -140,7 +140,7 @@ } @else { @if (list.displayMode === DisplayMode.LARGE_CARDS) {
- @for (d of list.documents; track trackByDocumentId($index, d)) { + @for (d of list.documents; track d.id) { - @for (d of list.documents; track trackByDocumentId($index, d)) { + @for (d of list.documents; track d.id) {
@@ -364,7 +364,7 @@ } @if (list.displayMode === DisplayMode.SMALL_CARDS) {
- @for (d of list.documents; track trackByDocumentId($index, d)) { + @for (d of list.documents; track d.id) { 0) { - + > } { ToggleableDropdownButtonComponent, DatesDropdownComponent, CustomDatePipe, + CustomFieldsQueryDropdownComponent, ], imports: [ RouterModule, @@ -190,6 +198,7 @@ describe('FilterEditorComponent', () => { NgbDatepickerModule, NgxBootstrapIconsModule.pick(allIcons), NgbTypeaheadModule, + NgSelectModule, ], providers: [ FilterPipe, @@ -838,108 +847,79 @@ describe('FilterEditorComponent', () => { ] })) - it('should ingest filter rules for has all custom fields', fakeAsync(() => { - expect(component.customFieldSelectionModel.getSelectedItems()).toHaveLength( - 0 - ) + it('should ingest filter rules for custom fields all', fakeAsync(() => { + expect(component.customFieldQueriesModel.isEmpty()).toBeTruthy() component.filterRules = [ { rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, - value: '42', - }, - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, - value: '43', + value: '42,43', }, ] - expect(component.customFieldSelectionModel.logicalOperator).toEqual( - LogicalOperator.And + expect(component.customFieldQueriesModel.queries[0].operator).toEqual( + CustomFieldQueryLogicalOperator.And ) - expect(component.customFieldSelectionModel.getSelectedItems()).toEqual( - custom_fields - ) - // coverage - component.filterRules = [ - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, - value: null, - }, - ] - component.toggleTag(2) // coverage + expect(component.customFieldQueriesModel.queries[0].value.length).toEqual(2) + expect( + ( + component.customFieldQueriesModel.queries[0] + .value[0] as CustomFieldQueryAtom + ).serialize() + ).toEqual(['42', CustomFieldQueryOperator.Exists, 'true']) })) it('should ingest filter rules for has any custom fields', fakeAsync(() => { - expect(component.customFieldSelectionModel.getSelectedItems()).toHaveLength( - 0 - ) + expect(component.customFieldQueriesModel.isEmpty()).toBeTruthy() component.filterRules = [ { rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY, - value: '42', - }, - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY, - value: '43', + value: '42,43', }, ] - expect(component.customFieldSelectionModel.logicalOperator).toEqual( - LogicalOperator.Or + expect(component.customFieldQueriesModel.queries[0].operator).toEqual( + CustomFieldQueryLogicalOperator.Or ) - expect(component.customFieldSelectionModel.getSelectedItems()).toEqual( - custom_fields - ) - // coverage - component.filterRules = [ - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY, - value: null, - }, - ] + expect(component.customFieldQueriesModel.queries[0].value.length).toEqual(2) + expect( + ( + component.customFieldQueriesModel.queries[0] + .value[0] as CustomFieldQueryAtom + ).serialize() + ).toEqual(['42', CustomFieldQueryOperator.Exists, 'true']) })) - it('should ingest filter rules for has any custom field', fakeAsync(() => { - expect(component.customFieldSelectionModel.getSelectedItems()).toHaveLength( - 0 - ) + it('should ingest filter rules for custom field queries', fakeAsync(() => { + expect(component.customFieldQueriesModel.isEmpty()).toBeTruthy() component.filterRules = [ { - rule_type: FILTER_HAS_ANY_CUSTOM_FIELDS, - value: '1', + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: '["AND", [[42, "exists", "true"],[43, "exists", "true"]]]', }, ] - expect(component.customFieldSelectionModel.getSelectedItems()).toHaveLength( - 1 + expect(component.customFieldQueriesModel.queries[0].operator).toEqual( + CustomFieldQueryLogicalOperator.And ) - expect(component.customFieldSelectionModel.get(null)).toBeTruthy() - })) + expect(component.customFieldQueriesModel.queries[0].value.length).toEqual(2) + expect( + ( + component.customFieldQueriesModel.queries[0] + .value[0] as CustomFieldQueryAtom + ).serialize() + ).toEqual([42, CustomFieldQueryOperator.Exists, 'true']) - it('should ingest filter rules for exclude tag(s)', fakeAsync(() => { - expect(component.customFieldSelectionModel.getExcludedItems()).toHaveLength( - 0 - ) + // atom component.filterRules = [ { - rule_type: FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS, - value: '42', - }, - { - rule_type: FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS, - value: '43', - }, - ] - expect(component.customFieldSelectionModel.logicalOperator).toEqual( - LogicalOperator.And - ) - expect(component.customFieldSelectionModel.getExcludedItems()).toEqual( - custom_fields - ) - // coverage - component.filterRules = [ - { - rule_type: FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS, - value: null, + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: '[42, "exists", "true"]', }, ] + expect(component.customFieldQueriesModel.queries[0].value.length).toEqual(1) + expect( + ( + component.customFieldQueriesModel.queries[0] + .value[0] as CustomFieldQueryAtom + ).serialize() + ).toEqual([42, CustomFieldQueryOperator.Exists, 'true']) })) it('should ingest filter rules for owner', fakeAsync(() => { @@ -1453,71 +1433,37 @@ describe('FilterEditorComponent', () => { ]) })) - it('should convert user input to correct filter rules on custom field select not assigned', fakeAsync(() => { - const customFieldsFilterableDropdown = fixture.debugElement.queryAll( - By.directive(FilterableDropdownComponent) - )[4] - customFieldsFilterableDropdown.triggerEventHandler('opened') - const customFieldButton = customFieldsFilterableDropdown.queryAll( - By.directive(ToggleableDropdownButtonComponent) - )[0] - customFieldButton.triggerEventHandler('toggle') - fixture.detectChanges() - expect(component.filterRules).toEqual([ - { - rule_type: FILTER_HAS_ANY_CUSTOM_FIELDS, - value: 'false', - }, - ]) - })) - it('should convert user input to correct filter rules on custom field selections', fakeAsync(() => { - const customFieldsFilterableDropdown = fixture.debugElement.queryAll( - By.directive(FilterableDropdownComponent) - )[4] // CF dropdown - customFieldsFilterableDropdown.triggerEventHandler('opened') - const customFieldButtons = customFieldsFilterableDropdown.queryAll( - By.directive(ToggleableDropdownButtonComponent) + const customFieldsQueryDropdown = fixture.debugElement.queryAll( + By.directive(CustomFieldsQueryDropdownComponent) + )[0] + const customFieldToggleButton = customFieldsQueryDropdown.query( + By.css('button') ) - customFieldButtons[1].triggerEventHandler('toggle') - customFieldButtons[2].triggerEventHandler('toggle') + customFieldToggleButton.triggerEventHandler('click') fixture.detectChanges() - expect(component.filterRules).toEqual([ - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, - value: custom_fields[0].id.toString(), - }, - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, - value: custom_fields[1].id.toString(), - }, - ]) - const toggleOperatorButtons = customFieldsFilterableDropdown.queryAll( - By.css('input[type=radio]') + const customFieldButtons = customFieldsQueryDropdown.queryAll( + By.css('button') ) - toggleOperatorButtons[1].nativeElement.checked = true - toggleOperatorButtons[1].triggerEventHandler('change') + customFieldButtons[1].triggerEventHandler('click') fixture.detectChanges() + const query = component.customFieldQueriesModel + .queries[0] as CustomFieldQueryAtom + query.field = custom_fields[0].id + const fieldSelect: NgSelectComponent = customFieldsQueryDropdown.queryAll( + By.directive(NgSelectComponent) + )[0].componentInstance + fieldSelect.open() + const options = customFieldsQueryDropdown.queryAll(By.css('.ng-option')) + options[0].nativeElement.click() + expect(component.customFieldQueriesModel.queries[0].value.length).toEqual(1) expect(component.filterRules).toEqual([ { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY, - value: custom_fields[0].id.toString(), - }, - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY, - value: custom_fields[1].id.toString(), - }, - ]) - customFieldButtons[2].triggerEventHandler('exclude') - fixture.detectChanges() - expect(component.filterRules).toEqual([ - { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, - value: custom_fields[0].id.toString(), - }, - { - rule_type: FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS, - value: custom_fields[1].id.toString(), + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: JSON.stringify([ + CustomFieldQueryLogicalOperator.Or, + [[custom_fields[0].id, 'exists', 'true']], + ]), }, ]) })) @@ -1930,21 +1876,11 @@ describe('FilterEditorComponent', () => { component.filterRules = [ { - rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, - value: '42', + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: '["AND",[["42","exists","true"],["43","exists","true"]]]', }, ] - expect(component.generateFilterName()).toEqual( - `Custom fields: ${custom_fields[0].name}` - ) - - component.filterRules = [ - { - rule_type: FILTER_HAS_ANY_CUSTOM_FIELDS, - value: 'false', - }, - ] - expect(component.generateFilterName()).toEqual('Without any custom field') + expect(component.generateFilterName()).toEqual(`Custom fields query`) component.filterRules = [ { diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index fe1f6cc8c..24ef1b347 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -12,7 +12,7 @@ import { import { Tag } from 'src/app/data/tag' import { Correspondent } from 'src/app/data/correspondent' import { DocumentType } from 'src/app/data/document-type' -import { Observable, Subject, Subscription, from } from 'rxjs' +import { Observable, Subject, from } from 'rxjs' import { catchError, debounceTime, @@ -62,7 +62,7 @@ import { FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_ANY_CUSTOM_FIELDS, - FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS, + FILTER_CUSTOM_FIELDS_QUERY, } from 'src/app/data/filter-rule-type' import { FilterableDropdownSelectionModel, @@ -92,6 +92,15 @@ import { ComponentWithPermissions } from '../../with-permissions/with-permission import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service' import { CustomField } from 'src/app/data/custom-field' import { SearchService } from 'src/app/services/rest/search.service' +import { + CustomFieldQueryLogicalOperator, + CustomFieldQueryOperator, +} from 'src/app/data/custom-field-query' +import { CustomFieldQueriesModel } from '../../common/custom-fields-query-dropdown/custom-fields-query-dropdown.component' +import { + CustomFieldQueryExpression, + CustomFieldQueryAtom, +} from 'src/app/utils/custom-field-query-element' const TEXT_FILTER_TARGET_TITLE = 'title' const TEXT_FILTER_TARGET_TITLE_CONTENT = 'title-content' @@ -225,15 +234,8 @@ export class FilterEditorComponent return $localize`Without any tag` } - case FILTER_HAS_CUSTOM_FIELDS_ALL: - return $localize`Custom fields: ${ - this.customFields.find((f) => f.id == +rule.value)?.name - }` - - case FILTER_HAS_ANY_CUSTOM_FIELDS: - if (rule.value == 'false') { - return $localize`Without any custom field` - } + case FILTER_CUSTOM_FIELDS_QUERY: + return $localize`Custom fields query` case FILTER_TITLE: return $localize`Title: ${rule.value}` @@ -321,7 +323,7 @@ export class FilterEditorComponent correspondentSelectionModel = new FilterableDropdownSelectionModel() documentTypeSelectionModel = new FilterableDropdownSelectionModel() storagePathSelectionModel = new FilterableDropdownSelectionModel() - customFieldSelectionModel = new FilterableDropdownSelectionModel() + customFieldQueriesModel = new CustomFieldQueriesModel() dateCreatedBefore: string dateCreatedAfter: string @@ -356,7 +358,7 @@ export class FilterEditorComponent this.storagePathSelectionModel.clear(false) this.tagSelectionModel.clear(false) this.correspondentSelectionModel.clear(false) - this.customFieldSelectionModel.clear(false) + this.customFieldQueriesModel.clear(false) this._textFilter = null this._moreLikeId = null this.dateAddedBefore = null @@ -523,34 +525,45 @@ export class FilterEditorComponent false ) break + case FILTER_CUSTOM_FIELDS_QUERY: + try { + const query = JSON.parse(rule.value) + if (Array.isArray(query)) { + if (query.length === 2) { + // expression + this.customFieldQueriesModel.addExpression( + new CustomFieldQueryExpression(query as any) + ) + } else if (query.length === 3) { + // atom + this.customFieldQueriesModel.addAtom( + new CustomFieldQueryAtom(query as any) + ) + } + } + } catch (e) { + // error handled by list view service + } + break + // Legacy custom field filters case FILTER_HAS_CUSTOM_FIELDS_ALL: - this.customFieldSelectionModel.logicalOperator = LogicalOperator.And - this.customFieldSelectionModel.set( - rule.value ? +rule.value : null, - ToggleableItemState.Selected, - false + this.customFieldQueriesModel.addExpression( + new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + rule.value + .split(',') + .map((id) => [id, CustomFieldQueryOperator.Exists, 'true']), + ]) ) break case FILTER_HAS_CUSTOM_FIELDS_ANY: - this.customFieldSelectionModel.logicalOperator = LogicalOperator.Or - this.customFieldSelectionModel.set( - rule.value ? +rule.value : null, - ToggleableItemState.Selected, - false - ) - break - case FILTER_HAS_ANY_CUSTOM_FIELDS: - this.customFieldSelectionModel.set( - null, - ToggleableItemState.Selected, - false - ) - break - case FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS: - this.customFieldSelectionModel.set( - rule.value ? +rule.value : null, - ToggleableItemState.Excluded, - false + this.customFieldQueriesModel.addExpression( + new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.Or, + rule.value + .split(',') + .map((id) => [id, CustomFieldQueryOperator.Exists, 'true']), + ]) ) break case FILTER_ASN_ISNULL: @@ -768,34 +781,14 @@ export class FilterEditorComponent }) }) } - if (this.customFieldSelectionModel.isNoneSelected()) { + let queries = this.customFieldQueriesModel.queries.map((query) => + query.serialize() + ) + if (queries.length > 0) { filterRules.push({ - rule_type: FILTER_HAS_ANY_CUSTOM_FIELDS, - value: 'false', + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: JSON.stringify(queries[0]), }) - } else { - const customFieldFilterType = - this.customFieldSelectionModel.logicalOperator == LogicalOperator.And - ? FILTER_HAS_CUSTOM_FIELDS_ALL - : FILTER_HAS_CUSTOM_FIELDS_ANY - this.customFieldSelectionModel - .getSelectedItems() - .filter((field) => field.id) - .forEach((field) => { - filterRules.push({ - rule_type: customFieldFilterType, - value: field.id?.toString(), - }) - }) - this.customFieldSelectionModel - .getExcludedItems() - .filter((field) => field.id) - .forEach((field) => { - filterRules.push({ - rule_type: FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS, - value: field.id?.toString(), - }) - }) } if (this.dateCreatedBefore) { filterRules.push({ @@ -1079,10 +1072,6 @@ export class FilterEditorComponent this.storagePathSelectionModel.apply() } - onCustomFieldsDropdownOpen() { - this.customFieldSelectionModel.apply() - } - updateTextFilter(text, updateRules = true) { this._textFilter = text if (updateRules) { diff --git a/src-ui/src/app/data/custom-field-query.ts b/src-ui/src/app/data/custom-field-query.ts new file mode 100644 index 000000000..226a10605 --- /dev/null +++ b/src-ui/src/app/data/custom-field-query.ts @@ -0,0 +1,127 @@ +import { CustomFieldDataType } from './custom-field' + +export enum CustomFieldQueryLogicalOperator { + And = 'AND', + Or = 'OR', + Not = 'NOT', +} + +export enum CustomFieldQueryOperator { + Exact = 'exact', + In = 'in', + IsNull = 'isnull', + Exists = 'exists', + Contains = 'contains', + IContains = 'icontains', + GreaterThan = 'gt', + GreaterThanOrEqual = 'gte', + LessThan = 'lt', + LessThanOrEqual = 'lte', + Range = 'range', +} + +export const CUSTOM_FIELD_QUERY_OPERATOR_LABELS = { + [CustomFieldQueryOperator.Exact]: $localize`Equal to`, + [CustomFieldQueryOperator.In]: $localize`In`, + [CustomFieldQueryOperator.IsNull]: $localize`Is null`, + [CustomFieldQueryOperator.Exists]: $localize`Exists`, + [CustomFieldQueryOperator.Contains]: $localize`Contains`, + [CustomFieldQueryOperator.IContains]: $localize`Contains (case-insensitive)`, + [CustomFieldQueryOperator.GreaterThan]: $localize`Greater than`, + [CustomFieldQueryOperator.GreaterThanOrEqual]: $localize`Greater than or equal to`, + [CustomFieldQueryOperator.LessThan]: $localize`Less than`, + [CustomFieldQueryOperator.LessThanOrEqual]: $localize`Less than or equal to`, + [CustomFieldQueryOperator.Range]: $localize`Range`, +} + +export enum CustomFieldQueryOperatorGroups { + Basic = 'basic', + String = 'string', + Arithmetic = 'arithmetic', + Containment = 'containment', + Subset = 'subset', + Date = 'date', +} + +// Modified from filters.py > SUPPORTED_EXPR_OPERATORS +export const CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP = { + [CustomFieldQueryOperatorGroups.Basic]: [ + CustomFieldQueryOperator.Exists, + CustomFieldQueryOperator.IsNull, + CustomFieldQueryOperator.Exact, + ], + [CustomFieldQueryOperatorGroups.String]: [CustomFieldQueryOperator.IContains], + [CustomFieldQueryOperatorGroups.Arithmetic]: [ + CustomFieldQueryOperator.GreaterThan, + CustomFieldQueryOperator.GreaterThanOrEqual, + CustomFieldQueryOperator.LessThan, + CustomFieldQueryOperator.LessThanOrEqual, + ], + [CustomFieldQueryOperatorGroups.Containment]: [ + CustomFieldQueryOperator.Contains, + ], + [CustomFieldQueryOperatorGroups.Subset]: [CustomFieldQueryOperator.In], + [CustomFieldQueryOperatorGroups.Date]: [ + CustomFieldQueryOperator.GreaterThanOrEqual, + CustomFieldQueryOperator.LessThanOrEqual, + ], +} + +// filters.py > SUPPORTED_EXPR_CATEGORIES +export const CUSTOM_FIELD_QUERY_OPERATOR_GROUPS_BY_TYPE = { + [CustomFieldDataType.String]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.String, + ], + [CustomFieldDataType.Url]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.String, + ], + [CustomFieldDataType.Date]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.Date, + ], + [CustomFieldDataType.Boolean]: [CustomFieldQueryOperatorGroups.Basic], + [CustomFieldDataType.Integer]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.Arithmetic, + ], + [CustomFieldDataType.Float]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.Arithmetic, + ], + [CustomFieldDataType.Monetary]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.String, + CustomFieldQueryOperatorGroups.Arithmetic, + ], + [CustomFieldDataType.DocumentLink]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.Containment, + ], + [CustomFieldDataType.Select]: [ + CustomFieldQueryOperatorGroups.Basic, + CustomFieldQueryOperatorGroups.Subset, + ], +} + +export const CUSTOM_FIELD_QUERY_VALUE_TYPES_BY_OPERATOR = { + [CustomFieldQueryOperator.Exact]: 'string|boolean', + [CustomFieldQueryOperator.IsNull]: 'boolean', + [CustomFieldQueryOperator.Exists]: 'boolean', + [CustomFieldQueryOperator.IContains]: 'string', + [CustomFieldQueryOperator.GreaterThanOrEqual]: 'string|number', + [CustomFieldQueryOperator.LessThanOrEqual]: 'string|number', + [CustomFieldQueryOperator.GreaterThan]: 'number', + [CustomFieldQueryOperator.LessThan]: 'number', + [CustomFieldQueryOperator.Contains]: 'array', + [CustomFieldQueryOperator.In]: 'array', +} + +export const CUSTOM_FIELD_QUERY_MAX_DEPTH = 4 +export const CUSTOM_FIELD_QUERY_MAX_ATOMS = 5 + +export enum CustomFieldQueryElementType { + Atom = 'Atom', + Expression = 'Expression', +} diff --git a/src-ui/src/app/data/filter-rule-type.ts b/src-ui/src/app/data/filter-rule-type.ts index 9a87a421c..1c6b1cdf8 100644 --- a/src-ui/src/app/data/filter-rule-type.ts +++ b/src-ui/src/app/data/filter-rule-type.ts @@ -55,6 +55,8 @@ export const FILTER_HAS_CUSTOM_FIELDS_ANY = 39 export const FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS = 40 export const FILTER_HAS_ANY_CUSTOM_FIELDS = 41 +export const FILTER_CUSTOM_FIELDS_QUERY = 42 + export const FILTER_RULE_TYPES: FilterRuleType[] = [ { id: FILTER_TITLE, @@ -317,6 +319,12 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [ multi: false, default: true, }, + { + id: FILTER_CUSTOM_FIELDS_QUERY, + filtervar: 'custom_field_query', + datatype: 'string', + multi: false, + }, ] export interface FilterRuleType { diff --git a/src-ui/src/app/utils/custom-field-query-element.spec.ts b/src-ui/src/app/utils/custom-field-query-element.spec.ts new file mode 100644 index 000000000..65be3738a --- /dev/null +++ b/src-ui/src/app/utils/custom-field-query-element.spec.ts @@ -0,0 +1,245 @@ +import { + CustomFieldQueryElement, + CustomFieldQueryAtom, + CustomFieldQueryExpression, +} from './custom-field-query-element' +import { + CustomFieldQueryElementType, + CustomFieldQueryLogicalOperator, + CustomFieldQueryOperator, +} from '../data/custom-field-query' +import { fakeAsync, tick } from '@angular/core/testing' + +describe('CustomFieldQueryElement', () => { + it('should initialize with correct type and id', () => { + const element = new CustomFieldQueryElement( + CustomFieldQueryElementType.Atom + ) + expect(element.type).toBe(CustomFieldQueryElementType.Atom) + expect(element.id).toBeDefined() + }) + + it('should trigger changed on operator change', () => { + const element = new CustomFieldQueryElement( + CustomFieldQueryElementType.Atom + ) + element.changed.subscribe((changedElement) => { + expect(changedElement).toBe(element) + }) + element.operator = CustomFieldQueryOperator.Exists + }) + + it('should trigger changed subject on value change', () => { + const element = new CustomFieldQueryElement( + CustomFieldQueryElementType.Atom + ) + element.changed.subscribe((changedElement) => { + expect(changedElement).toBe(element) + }) + element.value = 'new value' + }) + + it('should throw error on serialize call', () => { + const element = new CustomFieldQueryElement( + CustomFieldQueryElementType.Atom + ) + expect(() => element.serialize()).toThrow('Implemented in subclass') + }) +}) + +describe('CustomFieldQueryAtom', () => { + it('should initialize with correct field, operator, and value', () => { + const atom = new CustomFieldQueryAtom([1, 'operator', 'value']) + expect(atom.field).toBe(1) + expect(atom.operator).toBe('operator') + expect(atom.value).toBe('value') + }) + + it('should trigger changed subject on field change', () => { + const atom = new CustomFieldQueryAtom() + atom.changed.subscribe((changedAtom) => { + expect(changedAtom).toBe(atom) + }) + atom.field = 2 + }) + + it('should set value to null if operator is not found in CUSTOM_FIELD_QUERY_VALUE_TYPES_BY_OPERATOR', () => { + const atom = new CustomFieldQueryAtom() + atom.operator = 'nonexistent_operator' + expect(atom.value).toBeNull() + }) + + it('should set value to empty string if new type is string', () => { + const atom = new CustomFieldQueryAtom() + atom.operator = CustomFieldQueryOperator.IContains + expect(atom.value).toBe('') + }) + + it('should set value to "true" if new type is boolean', () => { + const atom = new CustomFieldQueryAtom() + atom.operator = CustomFieldQueryOperator.Exists + expect(atom.value).toBe('true') + }) + + it('should set value to empty array if new type is array', () => { + const atom = new CustomFieldQueryAtom() + atom.operator = CustomFieldQueryOperator.In + expect(atom.value).toEqual([]) + }) + + it('should try to set existing value to number if new type is number', () => { + const atom = new CustomFieldQueryAtom() + atom.value = '42' + atom.operator = CustomFieldQueryOperator.GreaterThan + expect(atom.value).toBe('42') + + // fallback to null if value is not parseable + atom.value = 'not_a_number' + atom.operator = CustomFieldQueryOperator.GreaterThan + expect(atom.value).toBeNull() + }) + + it('should change boolean values to empty string if operator is not boolean', () => { + const atom = new CustomFieldQueryAtom() + atom.value = 'true' + atom.operator = CustomFieldQueryOperator.Exact + expect(atom.value).toBe('') + }) + + it('should serialize correctly', () => { + const atom = new CustomFieldQueryAtom([1, 'operator', 'value']) + expect(atom.serialize()).toEqual([1, 'operator', 'value']) + }) + + it('should emit changed on value change after debounce', fakeAsync(() => { + const atom = new CustomFieldQueryAtom() + const changeSpy = jest.spyOn(atom.changed, 'next') + atom.value = 'new value' + tick(1000) + expect(changeSpy).toHaveBeenCalled() + })) +}) + +describe('CustomFieldQueryExpression', () => { + it('should initialize with default operator and empty value', () => { + const expression = new CustomFieldQueryExpression() + expect(expression.operator).toBe(CustomFieldQueryLogicalOperator.Or) + expect(expression.value).toEqual([]) + }) + + it('should initialize with correct operator and value, propagate changes', () => { + const expression = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [ + [1, 'exists', 'true'], + [2, 'exists', 'true'], + ], + ]) + expect(expression.operator).toBe(CustomFieldQueryLogicalOperator.And) + expect(expression.value.length).toBe(2) + + // propagate changes + const expressionChangeSpy = jest.spyOn(expression.changed, 'next') + ;(expression.value[0] as CustomFieldQueryAtom).changed.next( + expression.value[0] as any + ) + expect(expressionChangeSpy).toHaveBeenCalled() + + const expression2 = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.Not, + [[CustomFieldQueryLogicalOperator.Or, []]], + ]) + const expressionChangeSpy2 = jest.spyOn(expression2.changed, 'next') + ;(expression2.value[0] as CustomFieldQueryExpression).changed.next( + expression2.value[0] as any + ) + expect(expressionChangeSpy2).toHaveBeenCalled() + }) + + it('should initialize with a sub-expression i.e. NOT', () => { + const expression = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.Not, + [ + 'AND', + [ + [1, 'exists', 'true'], + [2, 'exists', 'true'], + ], + ], + ]) + expect(expression.value).toHaveLength(1) + const changedSpy = jest.spyOn(expression.changed, 'next') + ;(expression.value[0] as CustomFieldQueryExpression).changed.next( + expression.value[0] as any + ) + expect(changedSpy).toHaveBeenCalled() + }) + + it('should add atom correctly, propagate changes', () => { + const expression = new CustomFieldQueryExpression() + const atom = new CustomFieldQueryAtom([ + 1, + CustomFieldQueryOperator.Exists, + 'true', + ]) + expression.addAtom(atom) + expect(expression.value).toContain(atom) + const changeSpy = jest.spyOn(expression.changed, 'next') + atom.changed.next(atom) + expect(changeSpy).toHaveBeenCalled() + // coverage + expression.addAtom() + }) + + it('should add expression correctly, propagate changes', () => { + const expression = new CustomFieldQueryExpression() + const subExpression = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.Or, + [], + ]) + expression.addExpression(subExpression) + expect(expression.value).toContain(subExpression) + const changeSpy = jest.spyOn(expression.changed, 'next') + subExpression.changed.next(subExpression) + expect(changeSpy).toHaveBeenCalled() + // coverage + expression.addExpression() + }) + + it('should serialize correctly', () => { + const expression = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [[1, 'exists', 'true']], + ]) + expect(expression.serialize()).toEqual([ + CustomFieldQueryLogicalOperator.And, + [[1, 'exists', 'true']], + ]) + }) + + it('should serialize NOT expressions correctly', () => { + const expression = new CustomFieldQueryExpression() + expression.addExpression( + new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.And, + [ + [1, 'exists', 'true'], + [2, 'exists', 'true'], + ], + ]) + ) + expression.operator = CustomFieldQueryLogicalOperator.Not + const serialized = expression.serialize() + expect(serialized[0]).toBe(CustomFieldQueryLogicalOperator.Not) + expect(serialized[1][0]).toBe(CustomFieldQueryLogicalOperator.And) + expect(serialized[1][1].length).toBe(2) + }) + + it('should be negatable if it has one child which is an expression', () => { + const expression = new CustomFieldQueryExpression([ + CustomFieldQueryLogicalOperator.Not, + [[CustomFieldQueryLogicalOperator.Or, []]], + ]) + expect(expression.negatable).toBe(true) + }) +}) diff --git a/src-ui/src/app/utils/custom-field-query-element.ts b/src-ui/src/app/utils/custom-field-query-element.ts new file mode 100644 index 000000000..696853f12 --- /dev/null +++ b/src-ui/src/app/utils/custom-field-query-element.ts @@ -0,0 +1,210 @@ +import { Subject, debounceTime, distinctUntilChanged } from 'rxjs' +import { v4 as uuidv4 } from 'uuid' +import { + CustomFieldQueryElementType, + CUSTOM_FIELD_QUERY_VALUE_TYPES_BY_OPERATOR, + CustomFieldQueryLogicalOperator, + CustomFieldQueryOperator, +} from '../data/custom-field-query' + +export class CustomFieldQueryElement { + public readonly type: CustomFieldQueryElementType + public changed: Subject + protected valueModelChanged: Subject< + string | string[] | number[] | CustomFieldQueryElement[] + > + public depth: number = 0 + public id: string = uuidv4() + + constructor(type: CustomFieldQueryElementType) { + this.type = type + this.changed = new Subject() + this.valueModelChanged = new Subject() + this.connectValueModelChanged() + } + + protected connectValueModelChanged() { + // Allows overriding in subclasses + this.valueModelChanged.subscribe(() => { + this.changed.next(this) + }) + } + + public serialize() { + throw new Error('Implemented in subclass') + } + + protected _operator: string = null + public set operator(value: string) { + this._operator = value + this.changed.next(this) + } + public get operator(): string { + return this._operator + } + + protected _value: string | string[] | number[] | CustomFieldQueryElement[] = + null + public set value( + value: string | string[] | number[] | CustomFieldQueryElement[] + ) { + this._value = value + this.valueModelChanged.next(value) + } + public get value(): string | string[] | number[] | CustomFieldQueryElement[] { + return this._value + } +} + +export class CustomFieldQueryAtom extends CustomFieldQueryElement { + protected _field: number + set field(field: any) { + this._field = parseInt(field, 10) + this.changed.next(this) + } + get field(): number { + return this._field + } + + override set operator(operator: string) { + const newTypes: string[] = + CUSTOM_FIELD_QUERY_VALUE_TYPES_BY_OPERATOR[operator]?.split('|') + if (!newTypes) { + this.value = null + } else { + if (!newTypes.includes(typeof this.value)) { + switch (newTypes[0]) { + case 'string': + this.value = '' + break + case 'boolean': + this.value = 'true' + break + case 'array': + this.value = [] + break + case 'number': + const num = parseFloat(this.value as string) + this.value = isNaN(num) ? null : num.toString() + break + } + } else if ( + ['true', 'false'].includes(this.value as string) && + newTypes.includes('string') + ) { + this.value = '' + } + } + super.operator = operator + } + + override get operator(): string { + // why? + return super.operator + } + + constructor(queryArray: [number, string, string] = [null, null, null]) { + super(CustomFieldQueryElementType.Atom) + ;[this._field, this._operator, this._value] = queryArray + } + + protected override connectValueModelChanged(): void { + this.valueModelChanged + .pipe(debounceTime(1000), distinctUntilChanged()) + .subscribe(() => { + this.changed.next(this) + }) + } + + public override serialize() { + return [this._field, this._operator, this._value] + } +} + +export class CustomFieldQueryExpression extends CustomFieldQueryElement { + protected _value: string[] | number[] | CustomFieldQueryElement[] + + constructor( + expressionArray: [CustomFieldQueryLogicalOperator, any[]] = [ + CustomFieldQueryLogicalOperator.Or, + null, + ] + ) { + super(CustomFieldQueryElementType.Expression) + let values + ;[this._operator, values] = expressionArray + if (!values || values.length === 0) { + this._value = [] + } else if (values?.length > 0 && values[0] instanceof Array) { + this._value = values.map((value) => { + if (value.length === 3) { + const atom = new CustomFieldQueryAtom(value) + atom.depth = this.depth + 1 + atom.changed.subscribe(() => { + this.changed.next(this) + }) + return atom + } else { + const expression = new CustomFieldQueryExpression(value) + expression.depth = this.depth + 1 + expression.changed.subscribe(() => { + this.changed.next(this) + }) + return expression + } + }) + } else { + const expression = new CustomFieldQueryExpression(values as any) + expression.depth = this.depth + 1 + expression.changed.subscribe(() => { + this.changed.next(this) + }) + this._value = [expression] + } + } + + public override serialize() { + let value + value = this._value.map((element) => element.serialize()) + // If the expression is negated it should have only one child which is an expression + if ( + this._operator === CustomFieldQueryLogicalOperator.Not && + value.length === 1 + ) { + value = value[0] + } + return [this._operator, value] + } + + public addAtom( + atom: CustomFieldQueryAtom = new CustomFieldQueryAtom([ + null, + CustomFieldQueryOperator.Exists, + 'true', + ]) + ) { + atom.depth = this.depth + 1 + ;(this._value as CustomFieldQueryElement[]).push(atom) + atom.changed.subscribe(() => { + this.changed.next(this) + }) + } + + public addExpression( + expression: CustomFieldQueryExpression = new CustomFieldQueryExpression() + ) { + expression.depth = this.depth + 1 + ;(this._value as CustomFieldQueryElement[]).push(expression) + expression.changed.subscribe(() => { + this.changed.next(this) + }) + } + + public get negatable(): boolean { + return ( + this.value.length === 1 && + (this.value[0] as CustomFieldQueryElement).type === + CustomFieldQueryElementType.Expression + ) + } +} diff --git a/src-ui/src/app/utils/query-params.spec.ts b/src-ui/src/app/utils/query-params.spec.ts index a1bc0cdcd..64a89efec 100644 --- a/src-ui/src/app/utils/query-params.spec.ts +++ b/src-ui/src/app/utils/query-params.spec.ts @@ -2,13 +2,17 @@ import { convertToParamMap } from '@angular/router' import { FilterRule } from '../data/filter-rule' import { FILTER_CORRESPONDENT, + FILTER_CUSTOM_FIELDS_QUERY, FILTER_HAS_ANY_TAG, + FILTER_HAS_CUSTOM_FIELDS_ALL, + FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_HAS_TAGS_ALL, } from '../data/filter-rule-type' -import { paramsToViewState } from './query-params' +import { paramsToViewState, transformLegacyFilterRules } from './query-params' import { paramsFromViewState } from './query-params' import { queryParamsFromFilterRules } from './query-params' import { filterRulesFromQueryParams } from './query-params' +import { CustomFieldQueryLogicalOperator } from '../data/custom-field-query' const tags__id__all = '9' const filterRules: FilterRule[] = [ @@ -193,4 +197,58 @@ describe('QueryParams Utils', () => { }, ]) }) + + it('should transform legacy filter rules', () => { + let filterRules: FilterRule[] = [ + { + rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY, + value: '1', + }, + { + rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY, + value: '2', + }, + ] + + let transformedFilterRules = transformLegacyFilterRules(filterRules) + + expect(transformedFilterRules).toEqual([ + { + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: JSON.stringify([ + CustomFieldQueryLogicalOperator.Or, + [ + [1, 'exists', true], + [2, 'exists', true], + ], + ]), + }, + ]) + + filterRules = [ + { + rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, + value: '3', + }, + { + rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL, + value: '4', + }, + ] + + transformedFilterRules = transformLegacyFilterRules(filterRules) + + expect(transformedFilterRules).toEqual([ + { + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: JSON.stringify([ + CustomFieldQueryLogicalOperator.And, + [ + [3, 'exists', true], + [4, 'exists', true], + ], + ]), + }, + ]) + }) }) diff --git a/src-ui/src/app/utils/query-params.ts b/src-ui/src/app/utils/query-params.ts index 1121bd6a3..608d4edfb 100644 --- a/src-ui/src/app/utils/query-params.ts +++ b/src-ui/src/app/utils/query-params.ts @@ -1,7 +1,17 @@ import { ParamMap, Params } from '@angular/router' import { FilterRule } from '../data/filter-rule' -import { FilterRuleType, FILTER_RULE_TYPES } from '../data/filter-rule-type' +import { + FilterRuleType, + FILTER_RULE_TYPES, + FILTER_HAS_CUSTOM_FIELDS_ANY, + FILTER_CUSTOM_FIELDS_QUERY, + FILTER_HAS_CUSTOM_FIELDS_ALL, +} from '../data/filter-rule-type' import { ListViewState } from '../services/document-list-view.service' +import { + CustomFieldQueryLogicalOperator, + CustomFieldQueryOperator, +} from '../data/custom-field-query' const SORT_FIELD_PARAMETER = 'sort' const SORT_REVERSE_PARAMETER = 'reverse' @@ -40,6 +50,49 @@ export function paramsToViewState(queryParams: ParamMap): ListViewState { } } +export function transformLegacyFilterRules( + filterRules: FilterRule[] +): FilterRule[] { + const LEGACY_CUSTOM_FIELD_FILTER_RULE_TYPES = [ + FILTER_HAS_CUSTOM_FIELDS_ANY, + FILTER_HAS_CUSTOM_FIELDS_ALL, + ] + if ( + filterRules.filter((rule) => + LEGACY_CUSTOM_FIELD_FILTER_RULE_TYPES.includes(rule.rule_type) + ).length + ) { + const anyRules = filterRules.filter( + (rule) => rule.rule_type === FILTER_HAS_CUSTOM_FIELDS_ANY + ) + const allRules = filterRules.filter( + (rule) => rule.rule_type === FILTER_HAS_CUSTOM_FIELDS_ALL + ) + const customFieldQueryLogicalOperator = allRules.length + ? CustomFieldQueryLogicalOperator.And + : CustomFieldQueryLogicalOperator.Or + const valueRules = allRules.length ? allRules : anyRules + const customFieldQueryExpression = [ + customFieldQueryLogicalOperator, + [ + ...valueRules.map((rule) => [ + parseInt(rule.value), + CustomFieldQueryOperator.Exists, + true, + ]), + ], + ] + filterRules.push({ + rule_type: FILTER_CUSTOM_FIELDS_QUERY, + value: JSON.stringify(customFieldQueryExpression), + }) + } + // TODO: can we support FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS or FILTER_HAS_ANY_CUSTOM_FIELDS? + return filterRules.filter( + (rule) => !LEGACY_CUSTOM_FIELD_FILTER_RULE_TYPES.includes(rule.rule_type) + ) +} + export function filterRulesFromQueryParams( queryParams: ParamMap ): FilterRule[] { @@ -77,7 +130,9 @@ export function filterRulesFromQueryParams( }) ) }) - + filterRulesFromQueryParams = transformLegacyFilterRules( + filterRulesFromQueryParams + ) return filterRulesFromQueryParams } diff --git a/src/documents/filters.py b/src/documents/filters.py index 25e840141..f0a9a55b3 100644 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -29,13 +29,15 @@ from documents.models import Log from documents.models import ShareLink from documents.models import StoragePath from documents.models import Tag -from paperless import settings CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"] ID_KWARGS = ["in", "exact"] INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"] DATE_KWARGS = ["year", "month", "day", "date__gt", "gt", "date__lt", "lt"] +CUSTOM_FIELD_QUERY_MAX_DEPTH = 10 +CUSTOM_FIELD_QUERY_MAX_ATOMS = 20 + class CorrespondentFilterSet(FilterSet): class Meta: @@ -234,19 +236,13 @@ def handle_validation_prefix(func: Callable): return wrapper -class CustomFieldLookupParser: +class CustomFieldQueryParser: EXPR_BY_CATEGORY = { "basic": ["exact", "in", "isnull", "exists"], "string": [ - "iexact", - "contains", "icontains", - "startswith", "istartswith", - "endswith", "iendswith", - "regex", - "iregex", ], "arithmetic": [ "gt", @@ -258,23 +254,6 @@ class CustomFieldLookupParser: "containment": ["contains"], } - # These string lookup expressions are problematic. We shall disable - # them by default unless the user explicitly opts in. - STR_EXPR_DISABLED_BY_DEFAULT = [ - # SQLite: is case-sensitive outside the ASCII range - "iexact", - # SQLite: behaves the same as icontains - "contains", - # SQLite: behaves the same as istartswith - "startswith", - # SQLite: behaves the same as iendswith - "endswith", - # Syntax depends on database backends, can be exploited for ReDoS - "regex", - # Syntax depends on database backends, can be exploited for ReDoS - "iregex", - ] - SUPPORTED_EXPR_CATEGORIES = { CustomField.FieldDataType.STRING: ("basic", "string"), CustomField.FieldDataType.URL: ("basic", "string"), @@ -282,7 +261,7 @@ class CustomFieldLookupParser: CustomField.FieldDataType.BOOL: ("basic",), CustomField.FieldDataType.INT: ("basic", "arithmetic"), CustomField.FieldDataType.FLOAT: ("basic", "arithmetic"), - CustomField.FieldDataType.MONETARY: ("basic", "string"), + CustomField.FieldDataType.MONETARY: ("basic", "string", "arithmetic"), CustomField.FieldDataType.DOCUMENTLINK: ("basic", "containment"), CustomField.FieldDataType.SELECT: ("basic",), } @@ -371,7 +350,7 @@ class CustomFieldLookupParser: elif len(expr) == 3: return self._parse_atom(*expr) raise serializers.ValidationError( - [_("Invalid custom field lookup expression")], + [_("Invalid custom field query expression")], ) @handle_validation_prefix @@ -416,13 +395,7 @@ class CustomFieldLookupParser: self._atom_count += 1 if self._atom_count > self._max_atom_count: raise serializers.ValidationError( - [ - _( - "Maximum number of query conditions exceeded. You can raise " - "the limit by setting PAPERLESS_CUSTOM_FIELD_LOOKUP_MAX_ATOMS " - "in your configuration file.", - ), - ], + [_("Maximum number of query conditions exceeded.")], ) custom_field = self._get_custom_field(id_or_name, validation_prefix="0") @@ -444,6 +417,11 @@ class CustomFieldLookupParser: value_field_name = CustomFieldInstance.get_value_field_name( custom_field.data_type, ) + if ( + custom_field.data_type == CustomField.FieldDataType.MONETARY + and op in self.EXPR_BY_CATEGORY["arithmetic"] + ): + value_field_name = "value_monetary_amount" has_field = Q(custom_fields__field=custom_field) # Our special exists operator. @@ -494,22 +472,6 @@ class CustomFieldLookupParser: # Check if the operator is supported for the current data_type. supported = False for category in self.SUPPORTED_EXPR_CATEGORIES[custom_field.data_type]: - if ( - category == "string" - and op in self.STR_EXPR_DISABLED_BY_DEFAULT - and op not in settings.CUSTOM_FIELD_LOOKUP_OPT_IN - ): - raise serializers.ValidationError( - [ - _( - "{expr!r} is disabled by default because it does not " - "behave consistently across database backends, or can " - "cause security risks. If you understand the implications " - "you may enabled it by adding it to " - "`PAPERLESS_CUSTOM_FIELD_LOOKUP_OPT_IN`.", - ).format(expr=op), - ], - ) if op in self.EXPR_BY_CATEGORY[category]: supported = True break @@ -527,7 +489,7 @@ class CustomFieldLookupParser: if not supported: raise serializers.ValidationError( [ - _("{data_type} does not support lookup expr {expr!r}.").format( + _("{data_type} does not support query expr {expr!r}.").format( data_type=custom_field.data_type, expr=raw_op, ), @@ -548,7 +510,7 @@ class CustomFieldLookupParser: custom_field.data_type == CustomField.FieldDataType.DATE and prefix in self.DATE_COMPONENTS ): - # DateField admits lookups in the form of `year__exact`, etc. These take integers. + # DateField admits queries in the form of `year__exact`, etc. These take integers. field = serializers.IntegerField() elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK: # We can be more specific here and make sure the value is a list. @@ -610,7 +572,7 @@ class CustomFieldLookupParser: custom_fields__value_document_ids__isnull=False, ) - # First we lookup reverse links from the requested documents. + # First we look up reverse links from the requested documents. links = CustomFieldInstance.objects.filter( document_id__in=value, field__data_type=CustomField.FieldDataType.DOCUMENTLINK, @@ -635,22 +597,14 @@ class CustomFieldLookupParser: # guard against queries that are too deeply nested self._current_depth += 1 if self._current_depth > self._max_query_depth: - raise serializers.ValidationError( - [ - _( - "Maximum nesting depth exceeded. You can raise the limit " - "by setting PAPERLESS_CUSTOM_FIELD_LOOKUP_MAX_DEPTH in " - "your configuration file.", - ), - ], - ) + raise serializers.ValidationError([_("Maximum nesting depth exceeded.")]) try: yield finally: self._current_depth -= 1 -class CustomFieldLookupFilter(Filter): +class CustomFieldQueryFilter(Filter): def __init__(self, validation_prefix): """ A filter that filters documents based on custom field name and value. @@ -665,10 +619,10 @@ class CustomFieldLookupFilter(Filter): if not value: return qs - parser = CustomFieldLookupParser( + parser = CustomFieldQueryParser( self._validation_prefix, - max_query_depth=settings.CUSTOM_FIELD_LOOKUP_MAX_DEPTH, - max_atom_count=settings.CUSTOM_FIELD_LOOKUP_MAX_ATOMS, + max_query_depth=CUSTOM_FIELD_QUERY_MAX_DEPTH, + max_atom_count=CUSTOM_FIELD_QUERY_MAX_ATOMS, ) q, annotations = parser.parse(value) @@ -722,7 +676,7 @@ class DocumentFilterSet(FilterSet): exclude=True, ) - custom_field_lookup = CustomFieldLookupFilter("custom_field_lookup") + custom_field_query = CustomFieldQueryFilter("custom_field_query") shared_by__id = SharedByUser() diff --git a/src/documents/migrations/1054_customfieldinstance_value_monetary_amount_and_more.py b/src/documents/migrations/1054_customfieldinstance_value_monetary_amount_and_more.py new file mode 100644 index 000000000..92d45de33 --- /dev/null +++ b/src/documents/migrations/1054_customfieldinstance_value_monetary_amount_and_more.py @@ -0,0 +1,95 @@ +# Generated by Django 5.1.1 on 2024-09-29 16:26 + +import django.db.models.functions.comparison +import django.db.models.functions.text +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "1053_document_page_count"), + ] + + operations = [ + migrations.AddField( + model_name="customfieldinstance", + name="value_monetary_amount", + field=models.GeneratedField( + db_persist=True, + expression=models.Case( + models.When( + then=django.db.models.functions.comparison.Cast( + django.db.models.functions.text.Substr("value_monetary", 1), + output_field=models.DecimalField( + decimal_places=2, + max_digits=65, + ), + ), + value_monetary__regex="^\\d+", + ), + default=django.db.models.functions.comparison.Cast( + django.db.models.functions.text.Substr("value_monetary", 4), + output_field=models.DecimalField( + decimal_places=2, + max_digits=65, + ), + ), + output_field=models.DecimalField(decimal_places=2, max_digits=65), + ), + output_field=models.DecimalField(decimal_places=2, max_digits=65), + ), + ), + migrations.AlterField( + model_name="savedviewfilterrule", + name="rule_type", + field=models.PositiveIntegerField( + choices=[ + (0, "title contains"), + (1, "content contains"), + (2, "ASN is"), + (3, "correspondent is"), + (4, "document type is"), + (5, "is in inbox"), + (6, "has tag"), + (7, "has any tag"), + (8, "created before"), + (9, "created after"), + (10, "created year is"), + (11, "created month is"), + (12, "created day is"), + (13, "added before"), + (14, "added after"), + (15, "modified before"), + (16, "modified after"), + (17, "does not have tag"), + (18, "does not have ASN"), + (19, "title or content contains"), + (20, "fulltext query"), + (21, "more like this"), + (22, "has tags in"), + (23, "ASN greater than"), + (24, "ASN less than"), + (25, "storage path is"), + (26, "has correspondent in"), + (27, "does not have correspondent in"), + (28, "has document type in"), + (29, "does not have document type in"), + (30, "has storage path in"), + (31, "does not have storage path in"), + (32, "owner is"), + (33, "has owner in"), + (34, "does not have owner"), + (35, "does not have owner in"), + (36, "has custom field value"), + (37, "is shared by me"), + (38, "has custom fields"), + (39, "has custom field in"), + (40, "does not have custom field in"), + (41, "does not have custom field"), + (42, "custom fields query"), + ], + verbose_name="rule type", + ), + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 6dae8ba65..80476bffa 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -22,6 +22,9 @@ from multiselectfield import MultiSelectField if settings.AUDIT_LOG_ENABLED: from auditlog.registry import auditlog +from django.db.models import Case +from django.db.models.functions import Cast +from django.db.models.functions import Substr from django_softdelete.models import SoftDeleteModel from documents.data_models import DocumentSource @@ -519,6 +522,7 @@ class SavedViewFilterRule(models.Model): (39, _("has custom field in")), (40, _("does not have custom field in")), (41, _("does not have custom field")), + (42, _("custom fields query")), ] saved_view = models.ForeignKey( @@ -921,6 +925,27 @@ class CustomFieldInstance(models.Model): value_monetary = models.CharField(null=True, max_length=128) + value_monetary_amount = models.GeneratedField( + expression=Case( + # If the value starts with a number and no currency symbol, use the whole string + models.When( + value_monetary__regex=r"^\d+", + then=Cast( + Substr("value_monetary", 1), + output_field=models.DecimalField(decimal_places=2, max_digits=65), + ), + ), + # If the value starts with a 3-char currency symbol, use the rest of the string + default=Cast( + Substr("value_monetary", 4), + output_field=models.DecimalField(decimal_places=2, max_digits=65), + ), + output_field=models.DecimalField(decimal_places=2, max_digits=65), + ), + output_field=models.DecimalField(decimal_places=2, max_digits=65), + db_persist=True, + ) + value_document_ids = models.JSONField(null=True) value_select = models.PositiveSmallIntegerField(null=True) diff --git a/src/documents/tests/test_api_filter_by_custom_fields.py b/src/documents/tests/test_api_filter_by_custom_fields.py index 0f7da0a61..c9a0cdcfc 100644 --- a/src/documents/tests/test_api_filter_by_custom_fields.py +++ b/src/documents/tests/test_api_filter_by_custom_fields.py @@ -1,11 +1,9 @@ import json -import re from collections.abc import Callable from datetime import date from unittest.mock import Mock from urllib.parse import quote -import pytest from django.contrib.auth.models import User from rest_framework.test import APITestCase @@ -13,7 +11,6 @@ from documents.models import CustomField from documents.models import Document from documents.serialisers import DocumentSerializer from documents.tests.utils import DirectoriesMixin -from paperless import settings class DocumentWrapper: @@ -31,11 +28,7 @@ class DocumentWrapper: return self._document.custom_fields.get(field__name=custom_field).value -def string_expr_opted_in(op): - return op in settings.CUSTOM_FIELD_LOOKUP_OPT_IN - - -class TestDocumentSearchApi(DirectoriesMixin, APITestCase): +class TestCustomFieldsSearch(DirectoriesMixin, APITestCase): def setUp(self): super().setUp() @@ -111,6 +104,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self._create_document(monetary_field="USD100.00") self._create_document(monetary_field="USD1.00") self._create_document(monetary_field="EUR50.00") + self._create_document(monetary_field="101.00") # CustomField.FieldDataType.DOCUMENTLINK self._create_document(documentlink_field=None) @@ -188,7 +182,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): "/api/documents/?" + "&".join( ( - f"custom_field_lookup={query_string}", + f"custom_field_query={query_string}", "ordering=archive_serial_number", "page=1", f"page_size={len(self.documents)}", @@ -212,7 +206,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): "/api/documents/?" + "&".join( ( - f"custom_field_lookup={query_string}", + f"custom_field_query={query_string}", "ordering=archive_serial_number", "page=1", f"page_size={len(self.documents)}", @@ -313,32 +307,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): # ==========================================================# # Expressions for string, URL, and monetary fields # # ==========================================================# - @pytest.mark.skipif( - not string_expr_opted_in("iexact"), - reason="iexact expr is disabled.", - ) - def test_iexact(self): - self._assert_query_match_predicate( - ["string_field", "iexact", "paperless"], - lambda document: "string_field" in document - and document["string_field"] is not None - and document["string_field"].lower() == "paperless", - ) - - @pytest.mark.skipif( - not string_expr_opted_in("contains"), - reason="contains expr is disabled.", - ) - def test_contains(self): - # WARNING: SQLite treats "contains" as "icontains"! - # You should avoid "contains" unless you know what you are doing! - self._assert_query_match_predicate( - ["string_field", "contains", "aper"], - lambda document: "string_field" in document - and document["string_field"] is not None - and "aper" in document["string_field"], - ) - def test_icontains(self): self._assert_query_match_predicate( ["string_field", "icontains", "aper"], @@ -347,20 +315,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): and "aper" in document["string_field"].lower(), ) - @pytest.mark.skipif( - not string_expr_opted_in("startswith"), - reason="startswith expr is disabled.", - ) - def test_startswith(self): - # WARNING: SQLite treats "startswith" as "istartswith"! - # You should avoid "startswith" unless you know what you are doing! - self._assert_query_match_predicate( - ["string_field", "startswith", "paper"], - lambda document: "string_field" in document - and document["string_field"] is not None - and document["string_field"].startswith("paper"), - ) - def test_istartswith(self): self._assert_query_match_predicate( ["string_field", "istartswith", "paper"], @@ -369,20 +323,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): and document["string_field"].lower().startswith("paper"), ) - @pytest.mark.skipif( - not string_expr_opted_in("endswith"), - reason="endswith expr is disabled.", - ) - def test_endswith(self): - # WARNING: SQLite treats "endswith" as "iendswith"! - # You should avoid "endswith" unless you know what you are doing! - self._assert_query_match_predicate( - ["string_field", "iendswith", "less"], - lambda document: "string_field" in document - and document["string_field"] is not None - and document["string_field"].lower().endswith("less"), - ) - def test_iendswith(self): self._assert_query_match_predicate( ["string_field", "iendswith", "less"], @@ -391,32 +331,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): and document["string_field"].lower().endswith("less"), ) - @pytest.mark.skipif( - not string_expr_opted_in("regex"), - reason="regex expr is disabled.", - ) - def test_regex(self): - # WARNING: the regex syntax is database dependent! - self._assert_query_match_predicate( - ["string_field", "regex", r"^p.+s$"], - lambda document: "string_field" in document - and document["string_field"] is not None - and re.match(r"^p.+s$", document["string_field"]), - ) - - @pytest.mark.skipif( - not string_expr_opted_in("iregex"), - reason="iregex expr is disabled.", - ) - def test_iregex(self): - # WARNING: the regex syntax is database dependent! - self._assert_query_match_predicate( - ["string_field", "iregex", r"^p.+s$"], - lambda document: "string_field" in document - and document["string_field"] is not None - and re.match(r"^p.+s$", document["string_field"], re.IGNORECASE), - ) - def test_url_field_istartswith(self): # URL fields supports all of the expressions above. # Just showing one of them here. @@ -427,28 +341,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): and document["url_field"].startswith("http://"), ) - @pytest.mark.skipif( - not string_expr_opted_in("iregex"), - reason="regex expr is disabled.", - ) - def test_monetary_field_iregex(self): - # Monetary fields supports all of the expressions above. - # Just showing one of them here. - # - # Unfortunately we can't do arithmetic comparisons on monetary field, - # but you are welcome to use regex to do some of that. - # E.g., USD between 100.00 and 999.99: - self._assert_query_match_predicate( - ["monetary_field", "regex", r"USD[1-9][0-9]{2}\.[0-9]{2}"], - lambda document: "monetary_field" in document - and document["monetary_field"] is not None - and re.match( - r"USD[1-9][0-9]{2}\.[0-9]{2}", - document["monetary_field"], - re.IGNORECASE, - ), - ) - # ==========================================================# # Arithmetic comparisons # # ==========================================================# @@ -502,6 +394,17 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): and document["date_field"].year >= 2024, ) + def test_gt_monetary(self): + self._assert_query_match_predicate( + ["monetary_field", "gt", "99"], + lambda document: "monetary_field" in document + and document["monetary_field"] is not None + and ( + document["monetary_field"] == "USD100.00" # With currency symbol + or document["monetary_field"] == "101.00" # No currency symbol + ), + ) + # ==========================================================# # Subset check (document link field only) # # ==========================================================# @@ -586,68 +489,57 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): def test_invalid_json(self): self._assert_validation_error( "not valid json", - ["custom_field_lookup"], + ["custom_field_query"], "must be valid JSON", ) def test_invalid_expression(self): self._assert_validation_error( json.dumps("valid json but not valid expr"), - ["custom_field_lookup"], - "Invalid custom field lookup expression", + ["custom_field_query"], + "Invalid custom field query expression", ) def test_invalid_custom_field_name(self): self._assert_validation_error( json.dumps(["invalid name", "iexact", "foo"]), - ["custom_field_lookup", "0"], + ["custom_field_query", "0"], "is not a valid custom field", ) def test_invalid_operator(self): self._assert_validation_error( json.dumps(["integer_field", "iexact", "foo"]), - ["custom_field_lookup", "1"], - "does not support lookup expr", + ["custom_field_query", "1"], + "does not support query expr", ) def test_invalid_value(self): self._assert_validation_error( json.dumps(["select_field", "exact", "not an option"]), - ["custom_field_lookup", "2"], + ["custom_field_query", "2"], "integer", ) def test_invalid_logical_operator(self): self._assert_validation_error( json.dumps(["invalid op", ["integer_field", "gt", 0]]), - ["custom_field_lookup", "0"], + ["custom_field_query", "0"], "Invalid logical operator", ) def test_invalid_expr_list(self): self._assert_validation_error( json.dumps(["AND", "not a list"]), - ["custom_field_lookup", "1"], + ["custom_field_query", "1"], "Invalid expression list", ) def test_invalid_operator_prefix(self): self._assert_validation_error( json.dumps(["integer_field", "foo__gt", 0]), - ["custom_field_lookup", "1"], - "does not support lookup expr", - ) - - @pytest.mark.skipif( - string_expr_opted_in("regex"), - reason="user opted into allowing regex expr", - ) - def test_disabled_operator(self): - self._assert_validation_error( - json.dumps(["string_field", "regex", r"^p.+s$"]), - ["custom_field_lookup", "1"], - "disabled by default", + ["custom_field_query", "1"], + "does not support query expr", ) def test_query_too_deep(self): @@ -656,7 +548,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): query = ["NOT", query] self._assert_validation_error( json.dumps(query), - ["custom_field_lookup", *(["1"] * 10)], + ["custom_field_query", *(["1"] * 10)], "Maximum nesting depth exceeded", ) @@ -665,6 +557,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): query = ["AND", [atom for _ in range(21)]] self._assert_validation_error( json.dumps(query), - ["custom_field_lookup", "1", "20"], + ["custom_field_query", "1", "20"], "Maximum number of query conditions exceeded", ) diff --git a/src/paperless/settings.py b/src/paperless/settings.py index 023e826c9..ab943f30f 100644 --- a/src/paperless/settings.py +++ b/src/paperless/settings.py @@ -1195,20 +1195,3 @@ EMAIL_ENABLE_GPG_DECRYPTOR: Final[bool] = __get_boolean( # Soft Delete # ############################################################################### EMPTY_TRASH_DELAY = max(__get_int("PAPERLESS_EMPTY_TRASH_DELAY", 30), 1) - -############################################################################### -# custom_field_lookup Filter Settings # -############################################################################### - -CUSTOM_FIELD_LOOKUP_OPT_IN = __get_list( - "PAPERLESS_CUSTOM_FIELD_LOOKUP_OPT_IN", - default=[], -) -CUSTOM_FIELD_LOOKUP_MAX_DEPTH = __get_int( - "PAPERLESS_CUSTOM_FIELD_LOOKUP_MAX_DEPTH", - default=10, -) -CUSTOM_FIELD_LOOKUP_MAX_ATOMS = __get_int( - "PAPERLESS_CUSTOM_FIELD_LOOKUP_MAX_ATOMS", - default=20, -)