Feature: custom fields queries (#7761)

src-ui/src/app/utils/custom-field-query-element.spec.ts

@@ -0,0 +1,245 @@
+import {
+  CustomFieldQueryElement,
+  CustomFieldQueryAtom,
+  CustomFieldQueryExpression,
+} from './custom-field-query-element'
+import {
+  CustomFieldQueryElementType,
+  CustomFieldQueryLogicalOperator,
+  CustomFieldQueryOperator,
+} from '../data/custom-field-query'
+import { fakeAsync, tick } from '@angular/core/testing'
+
+describe('CustomFieldQueryElement', () => {
+  it('should initialize with correct type and id', () => {
+    const element = new CustomFieldQueryElement(
+      CustomFieldQueryElementType.Atom
+    )
+    expect(element.type).toBe(CustomFieldQueryElementType.Atom)
+    expect(element.id).toBeDefined()
+  })
+
+  it('should trigger changed on operator change', () => {
+    const element = new CustomFieldQueryElement(
+      CustomFieldQueryElementType.Atom
+    )
+    element.changed.subscribe((changedElement) => {
+      expect(changedElement).toBe(element)
+    })
+    element.operator = CustomFieldQueryOperator.Exists
+  })
+
+  it('should trigger changed subject on value change', () => {
+    const element = new CustomFieldQueryElement(
+      CustomFieldQueryElementType.Atom
+    )
+    element.changed.subscribe((changedElement) => {
+      expect(changedElement).toBe(element)
+    })
+    element.value = 'new value'
+  })
+
+  it('should throw error on serialize call', () => {
+    const element = new CustomFieldQueryElement(
+      CustomFieldQueryElementType.Atom
+    )
+    expect(() => element.serialize()).toThrow('Implemented in subclass')
+  })
+})
+
+describe('CustomFieldQueryAtom', () => {
+  it('should initialize with correct field, operator, and value', () => {
+    const atom = new CustomFieldQueryAtom([1, 'operator', 'value'])
+    expect(atom.field).toBe(1)
+    expect(atom.operator).toBe('operator')
+    expect(atom.value).toBe('value')
+  })
+
+  it('should trigger changed subject on field change', () => {
+    const atom = new CustomFieldQueryAtom()
+    atom.changed.subscribe((changedAtom) => {
+      expect(changedAtom).toBe(atom)
+    })
+    atom.field = 2
+  })
+
+  it('should set value to null if operator is not found in CUSTOM_FIELD_QUERY_VALUE_TYPES_BY_OPERATOR', () => {
+    const atom = new CustomFieldQueryAtom()
+    atom.operator = 'nonexistent_operator'
+    expect(atom.value).toBeNull()
+  })
+
+  it('should set value to empty string if new type is string', () => {
+    const atom = new CustomFieldQueryAtom()
+    atom.operator = CustomFieldQueryOperator.IContains
+    expect(atom.value).toBe('')
+  })
+
+  it('should set value to "true" if new type is boolean', () => {
+    const atom = new CustomFieldQueryAtom()
+    atom.operator = CustomFieldQueryOperator.Exists
+    expect(atom.value).toBe('true')
+  })
+
+  it('should set value to empty array if new type is array', () => {
+    const atom = new CustomFieldQueryAtom()
+    atom.operator = CustomFieldQueryOperator.In
+    expect(atom.value).toEqual([])
+  })
+
+  it('should try to set existing value to number if new type is number', () => {
+    const atom = new CustomFieldQueryAtom()
+    atom.value = '42'
+    atom.operator = CustomFieldQueryOperator.GreaterThan
+    expect(atom.value).toBe('42')
+
+    // fallback to null if value is not parseable
+    atom.value = 'not_a_number'
+    atom.operator = CustomFieldQueryOperator.GreaterThan
+    expect(atom.value).toBeNull()
+  })
+
+  it('should change boolean values to empty string if operator is not boolean', () => {
+    const atom = new CustomFieldQueryAtom()
+    atom.value = 'true'
+    atom.operator = CustomFieldQueryOperator.Exact
+    expect(atom.value).toBe('')
+  })
+
+  it('should serialize correctly', () => {
+    const atom = new CustomFieldQueryAtom([1, 'operator', 'value'])
+    expect(atom.serialize()).toEqual([1, 'operator', 'value'])
+  })
+
+  it('should emit changed on value change after debounce', fakeAsync(() => {
+    const atom = new CustomFieldQueryAtom()
+    const changeSpy = jest.spyOn(atom.changed, 'next')
+    atom.value = 'new value'
+    tick(1000)
+    expect(changeSpy).toHaveBeenCalled()
+  }))
+})
+
+describe('CustomFieldQueryExpression', () => {
+  it('should initialize with default operator and empty value', () => {
+    const expression = new CustomFieldQueryExpression()
+    expect(expression.operator).toBe(CustomFieldQueryLogicalOperator.Or)
+    expect(expression.value).toEqual([])
+  })
+
+  it('should initialize with correct operator and value, propagate changes', () => {
+    const expression = new CustomFieldQueryExpression([
+      CustomFieldQueryLogicalOperator.And,
+      [
+        [1, 'exists', 'true'],
+        [2, 'exists', 'true'],
+      ],
+    ])
+    expect(expression.operator).toBe(CustomFieldQueryLogicalOperator.And)
+    expect(expression.value.length).toBe(2)
+
+    // propagate changes
+    const expressionChangeSpy = jest.spyOn(expression.changed, 'next')
+    ;(expression.value[0] as CustomFieldQueryAtom).changed.next(
+      expression.value[0] as any
+    )
+    expect(expressionChangeSpy).toHaveBeenCalled()
+
+    const expression2 = new CustomFieldQueryExpression([
+      CustomFieldQueryLogicalOperator.Not,
+      [[CustomFieldQueryLogicalOperator.Or, []]],
+    ])
+    const expressionChangeSpy2 = jest.spyOn(expression2.changed, 'next')
+    ;(expression2.value[0] as CustomFieldQueryExpression).changed.next(
+      expression2.value[0] as any
+    )
+    expect(expressionChangeSpy2).toHaveBeenCalled()
+  })
+
+  it('should initialize with a sub-expression i.e. NOT', () => {
+    const expression = new CustomFieldQueryExpression([
+      CustomFieldQueryLogicalOperator.Not,
+      [
+        'AND',
+        [
+          [1, 'exists', 'true'],
+          [2, 'exists', 'true'],
+        ],
+      ],
+    ])
+    expect(expression.value).toHaveLength(1)
+    const changedSpy = jest.spyOn(expression.changed, 'next')
+    ;(expression.value[0] as CustomFieldQueryExpression).changed.next(
+      expression.value[0] as any
+    )
+    expect(changedSpy).toHaveBeenCalled()
+  })
+
+  it('should add atom correctly, propagate changes', () => {
+    const expression = new CustomFieldQueryExpression()
+    const atom = new CustomFieldQueryAtom([
+      1,
+      CustomFieldQueryOperator.Exists,
+      'true',
+    ])
+    expression.addAtom(atom)
+    expect(expression.value).toContain(atom)
+    const changeSpy = jest.spyOn(expression.changed, 'next')
+    atom.changed.next(atom)
+    expect(changeSpy).toHaveBeenCalled()
+    // coverage
+    expression.addAtom()
+  })
+
+  it('should add expression correctly, propagate changes', () => {
+    const expression = new CustomFieldQueryExpression()
+    const subExpression = new CustomFieldQueryExpression([
+      CustomFieldQueryLogicalOperator.Or,
+      [],
+    ])
+    expression.addExpression(subExpression)
+    expect(expression.value).toContain(subExpression)
+    const changeSpy = jest.spyOn(expression.changed, 'next')
+    subExpression.changed.next(subExpression)
+    expect(changeSpy).toHaveBeenCalled()
+    // coverage
+    expression.addExpression()
+  })
+
+  it('should serialize correctly', () => {
+    const expression = new CustomFieldQueryExpression([
+      CustomFieldQueryLogicalOperator.And,
+      [[1, 'exists', 'true']],
+    ])
+    expect(expression.serialize()).toEqual([
+      CustomFieldQueryLogicalOperator.And,
+      [[1, 'exists', 'true']],
+    ])
+  })
+
+  it('should serialize NOT expressions correctly', () => {
+    const expression = new CustomFieldQueryExpression()
+    expression.addExpression(
+      new CustomFieldQueryExpression([
+        CustomFieldQueryLogicalOperator.And,
+        [
+          [1, 'exists', 'true'],
+          [2, 'exists', 'true'],
+        ],
+      ])
+    )
+    expression.operator = CustomFieldQueryLogicalOperator.Not
+    const serialized = expression.serialize()
+    expect(serialized[0]).toBe(CustomFieldQueryLogicalOperator.Not)
+    expect(serialized[1][0]).toBe(CustomFieldQueryLogicalOperator.And)
+    expect(serialized[1][1].length).toBe(2)
+  })
+
+  it('should be negatable if it has one child which is an expression', () => {
+    const expression = new CustomFieldQueryExpression([
+      CustomFieldQueryLogicalOperator.Not,
+      [[CustomFieldQueryLogicalOperator.Or, []]],
+    ])
+    expect(expression.negatable).toBe(true)
+  })
+})

src-ui/src/app/utils/custom-field-query-element.ts

@@ -0,0 +1,210 @@
+import { Subject, debounceTime, distinctUntilChanged } from 'rxjs'
+import { v4 as uuidv4 } from 'uuid'
+import {
+  CustomFieldQueryElementType,
+  CUSTOM_FIELD_QUERY_VALUE_TYPES_BY_OPERATOR,
+  CustomFieldQueryLogicalOperator,
+  CustomFieldQueryOperator,
+} from '../data/custom-field-query'
+
+export class CustomFieldQueryElement {
+  public readonly type: CustomFieldQueryElementType
+  public changed: Subject<CustomFieldQueryElement>
+  protected valueModelChanged: Subject<
+    string | string[] | number[] | CustomFieldQueryElement[]
+  >
+  public depth: number = 0
+  public id: string = uuidv4()
+
+  constructor(type: CustomFieldQueryElementType) {
+    this.type = type
+    this.changed = new Subject<CustomFieldQueryElement>()
+    this.valueModelChanged = new Subject<string | CustomFieldQueryElement[]>()
+    this.connectValueModelChanged()
+  }
+
+  protected connectValueModelChanged() {
+    // Allows overriding in subclasses
+    this.valueModelChanged.subscribe(() => {
+      this.changed.next(this)
+    })
+  }
+
+  public serialize() {
+    throw new Error('Implemented in subclass')
+  }
+
+  protected _operator: string = null
+  public set operator(value: string) {
+    this._operator = value
+    this.changed.next(this)
+  }
+  public get operator(): string {
+    return this._operator
+  }
+
+  protected _value: string | string[] | number[] | CustomFieldQueryElement[] =
+    null
+  public set value(
+    value: string | string[] | number[] | CustomFieldQueryElement[]
+  ) {
+    this._value = value
+    this.valueModelChanged.next(value)
+  }
+  public get value(): string | string[] | number[] | CustomFieldQueryElement[] {
+    return this._value
+  }
+}
+
+export class CustomFieldQueryAtom extends CustomFieldQueryElement {
+  protected _field: number
+  set field(field: any) {
+    this._field = parseInt(field, 10)
+    this.changed.next(this)
+  }
+  get field(): number {
+    return this._field
+  }
+
+  override set operator(operator: string) {
+    const newTypes: string[] =
+      CUSTOM_FIELD_QUERY_VALUE_TYPES_BY_OPERATOR[operator]?.split('|')
+    if (!newTypes) {
+      this.value = null
+    } else {
+      if (!newTypes.includes(typeof this.value)) {
+        switch (newTypes[0]) {
+          case 'string':
+            this.value = ''
+            break
+          case 'boolean':
+            this.value = 'true'
+            break
+          case 'array':
+            this.value = []
+            break
+          case 'number':
+            const num = parseFloat(this.value as string)
+            this.value = isNaN(num) ? null : num.toString()
+            break
+        }
+      } else if (
+        ['true', 'false'].includes(this.value as string) &&
+        newTypes.includes('string')
+      ) {
+        this.value = ''
+      }
+    }
+    super.operator = operator
+  }
+
+  override get operator(): string {
+    // why?
+    return super.operator
+  }
+
+  constructor(queryArray: [number, string, string] = [null, null, null]) {
+    super(CustomFieldQueryElementType.Atom)
+    ;[this._field, this._operator, this._value] = queryArray
+  }
+
+  protected override connectValueModelChanged(): void {
+    this.valueModelChanged
+      .pipe(debounceTime(1000), distinctUntilChanged())
+      .subscribe(() => {
+        this.changed.next(this)
+      })
+  }
+
+  public override serialize() {
+    return [this._field, this._operator, this._value]
+  }
+}
+
+export class CustomFieldQueryExpression extends CustomFieldQueryElement {
+  protected _value: string[] | number[] | CustomFieldQueryElement[]
+
+  constructor(
+    expressionArray: [CustomFieldQueryLogicalOperator, any[]] = [
+      CustomFieldQueryLogicalOperator.Or,
+      null,
+    ]
+  ) {
+    super(CustomFieldQueryElementType.Expression)
+    let values
+    ;[this._operator, values] = expressionArray
+    if (!values || values.length === 0) {
+      this._value = []
+    } else if (values?.length > 0 && values[0] instanceof Array) {
+      this._value = values.map((value) => {
+        if (value.length === 3) {
+          const atom = new CustomFieldQueryAtom(value)
+          atom.depth = this.depth + 1
+          atom.changed.subscribe(() => {
+            this.changed.next(this)
+          })
+          return atom
+        } else {
+          const expression = new CustomFieldQueryExpression(value)
+          expression.depth = this.depth + 1
+          expression.changed.subscribe(() => {
+            this.changed.next(this)
+          })
+          return expression
+        }
+      })
+    } else {
+      const expression = new CustomFieldQueryExpression(values as any)
+      expression.depth = this.depth + 1
+      expression.changed.subscribe(() => {
+        this.changed.next(this)
+      })
+      this._value = [expression]
+    }
+  }
+
+  public override serialize() {
+    let value
+    value = this._value.map((element) => element.serialize())
+    // If the expression is negated it should have only one child which is an expression
+    if (
+      this._operator === CustomFieldQueryLogicalOperator.Not &&
+      value.length === 1
+    ) {
+      value = value[0]
+    }
+    return [this._operator, value]
+  }
+
+  public addAtom(
+    atom: CustomFieldQueryAtom = new CustomFieldQueryAtom([
+      null,
+      CustomFieldQueryOperator.Exists,
+      'true',
+    ])
+  ) {
+    atom.depth = this.depth + 1
+    ;(this._value as CustomFieldQueryElement[]).push(atom)
+    atom.changed.subscribe(() => {
+      this.changed.next(this)
+    })
+  }
+
+  public addExpression(
+    expression: CustomFieldQueryExpression = new CustomFieldQueryExpression()
+  ) {
+    expression.depth = this.depth + 1
+    ;(this._value as CustomFieldQueryElement[]).push(expression)
+    expression.changed.subscribe(() => {
+      this.changed.next(this)
+    })
+  }
+
+  public get negatable(): boolean {
+    return (
+      this.value.length === 1 &&
+      (this.value[0] as CustomFieldQueryElement).type ===
+        CustomFieldQueryElementType.Expression
+    )
+  }
+}

src-ui/src/app/utils/query-params.spec.ts

@@ -2,13 +2,17 @@ import { convertToParamMap } from '@angular/router'
 import { FilterRule } from '../data/filter-rule'
 import {
   FILTER_CORRESPONDENT,
+  FILTER_CUSTOM_FIELDS_QUERY,
   FILTER_HAS_ANY_TAG,
+  FILTER_HAS_CUSTOM_FIELDS_ALL,
+  FILTER_HAS_CUSTOM_FIELDS_ANY,
   FILTER_HAS_TAGS_ALL,
 } from '../data/filter-rule-type'
-import { paramsToViewState } from './query-params'
+import { paramsToViewState, transformLegacyFilterRules } from './query-params'
 import { paramsFromViewState } from './query-params'
 import { queryParamsFromFilterRules } from './query-params'
 import { filterRulesFromQueryParams } from './query-params'
+import { CustomFieldQueryLogicalOperator } from '../data/custom-field-query'
 
 const tags__id__all = '9'
 const filterRules: FilterRule[] = [
@@ -193,4 +197,58 @@ describe('QueryParams Utils', () => {
       },
     ])
   })
+
+  it('should transform legacy filter rules', () => {
+    let filterRules: FilterRule[] = [
+      {
+        rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY,
+        value: '1',
+      },
+      {
+        rule_type: FILTER_HAS_CUSTOM_FIELDS_ANY,
+        value: '2',
+      },
+    ]
+
+    let transformedFilterRules = transformLegacyFilterRules(filterRules)
+
+    expect(transformedFilterRules).toEqual([
+      {
+        rule_type: FILTER_CUSTOM_FIELDS_QUERY,
+        value: JSON.stringify([
+          CustomFieldQueryLogicalOperator.Or,
+          [
+            [1, 'exists', true],
+            [2, 'exists', true],
+          ],
+        ]),
+      },
+    ])
+
+    filterRules = [
+      {
+        rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL,
+        value: '3',
+      },
+      {
+        rule_type: FILTER_HAS_CUSTOM_FIELDS_ALL,
+        value: '4',
+      },
+    ]
+
+    transformedFilterRules = transformLegacyFilterRules(filterRules)
+
+    expect(transformedFilterRules).toEqual([
+      {
+        rule_type: FILTER_CUSTOM_FIELDS_QUERY,
+        value: JSON.stringify([
+          CustomFieldQueryLogicalOperator.And,
+          [
+            [3, 'exists', true],
+            [4, 'exists', true],
+          ],
+        ]),
+      },
+    ])
+  })
 })

src-ui/src/app/utils/query-params.ts

@@ -1,7 +1,17 @@
 import { ParamMap, Params } from '@angular/router'
 import { FilterRule } from '../data/filter-rule'
-import { FilterRuleType, FILTER_RULE_TYPES } from '../data/filter-rule-type'
+import {
+  FilterRuleType,
+  FILTER_RULE_TYPES,
+  FILTER_HAS_CUSTOM_FIELDS_ANY,
+  FILTER_CUSTOM_FIELDS_QUERY,
+  FILTER_HAS_CUSTOM_FIELDS_ALL,
+} from '../data/filter-rule-type'
 import { ListViewState } from '../services/document-list-view.service'
+import {
+  CustomFieldQueryLogicalOperator,
+  CustomFieldQueryOperator,
+} from '../data/custom-field-query'
 
 const SORT_FIELD_PARAMETER = 'sort'
 const SORT_REVERSE_PARAMETER = 'reverse'
@@ -40,6 +50,49 @@ export function paramsToViewState(queryParams: ParamMap): ListViewState {
   }
 }
 
+export function transformLegacyFilterRules(
+  filterRules: FilterRule[]
+): FilterRule[] {
+  const LEGACY_CUSTOM_FIELD_FILTER_RULE_TYPES = [
+    FILTER_HAS_CUSTOM_FIELDS_ANY,
+    FILTER_HAS_CUSTOM_FIELDS_ALL,
+  ]
+  if (
+    filterRules.filter((rule) =>
+      LEGACY_CUSTOM_FIELD_FILTER_RULE_TYPES.includes(rule.rule_type)
+    ).length
+  ) {
+    const anyRules = filterRules.filter(
+      (rule) => rule.rule_type === FILTER_HAS_CUSTOM_FIELDS_ANY
+    )
+    const allRules = filterRules.filter(
+      (rule) => rule.rule_type === FILTER_HAS_CUSTOM_FIELDS_ALL
+    )
+    const customFieldQueryLogicalOperator = allRules.length
+      ? CustomFieldQueryLogicalOperator.And
+      : CustomFieldQueryLogicalOperator.Or
+    const valueRules = allRules.length ? allRules : anyRules
+    const customFieldQueryExpression = [
+      customFieldQueryLogicalOperator,
+      [
+        ...valueRules.map((rule) => [
+          parseInt(rule.value),
+          CustomFieldQueryOperator.Exists,
+          true,
+        ]),
+      ],
+    ]
+    filterRules.push({
+      rule_type: FILTER_CUSTOM_FIELDS_QUERY,
+      value: JSON.stringify(customFieldQueryExpression),
+    })
+  }
+  // TODO: can we support FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS or FILTER_HAS_ANY_CUSTOM_FIELDS?
+  return filterRules.filter(
+    (rule) => !LEGACY_CUSTOM_FIELD_FILTER_RULE_TYPES.includes(rule.rule_type)
+  )
+}
+
 export function filterRulesFromQueryParams(
   queryParams: ParamMap
 ): FilterRule[] {
@@ -77,7 +130,9 @@ export function filterRulesFromQueryParams(
         })
       )
     })
-
+  filterRulesFromQueryParams = transformLegacyFilterRules(
+    filterRulesFromQueryParams
+  )
   return filterRulesFromQueryParams
 }