mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2025-10-24 03:26:11 -05:00
Compare commits
6 Commits
c212c823f6
...
8e19d009d9
Author | SHA1 | Date | |
---|---|---|---|
![]() |
8e19d009d9 | ||
![]() |
764ad059d1 | ||
![]() |
5e47069934 | ||
![]() |
4ff09c4cf4 | ||
![]() |
53b393dab5 | ||
![]() |
6119c215e7 |
140
.github/workflows/ci.yml
vendored
140
.github/workflows/ci.yml
vendored
@@ -17,11 +17,52 @@ env:
|
||||
DEFAULT_PYTHON_VERSION: "3.11"
|
||||
NLTK_DATA: "/usr/share/nltk_data"
|
||||
jobs:
|
||||
detect-duplicate:
|
||||
name: Detect Duplicate Run
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
should_run: ${{ steps.check.outputs.should_run }}
|
||||
steps:
|
||||
- name: Check if workflow should run
|
||||
id: check
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
if (context.eventName !== 'push') {
|
||||
core.info('Not a push event; running workflow.');
|
||||
core.setOutput('should_run', 'true');
|
||||
return;
|
||||
}
|
||||
|
||||
const ref = context.ref || '';
|
||||
if (!ref.startsWith('refs/heads/')) {
|
||||
core.info('Push is not to a branch; running workflow.');
|
||||
core.setOutput('should_run', 'true');
|
||||
return;
|
||||
}
|
||||
|
||||
const branch = ref.substring('refs/heads/'.length);
|
||||
const { owner, repo } = context.repo;
|
||||
const prs = await github.paginate(github.rest.pulls.list, {
|
||||
owner,
|
||||
repo,
|
||||
state: 'open',
|
||||
head: `${owner}:${branch}`,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
if (prs.length === 0) {
|
||||
core.info(`No open PR found for ${branch}; running workflow.`);
|
||||
core.setOutput('should_run', 'true');
|
||||
} else {
|
||||
core.info(`Found ${prs.length} open PR(s) for ${branch}; skipping duplicate push run.`);
|
||||
core.setOutput('should_run', 'false');
|
||||
}
|
||||
pre-commit:
|
||||
# We want to run on external PRs, but not on our own internal PRs as they'll be run
|
||||
# by the push to the branch. Without this if check, checks are duplicated since
|
||||
# internal PRs match both the push and pull_request events.
|
||||
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
|
||||
needs:
|
||||
- detect-duplicate
|
||||
if: needs.detect-duplicate.outputs.should_run == 'true'
|
||||
name: Linting Checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
@@ -151,18 +192,6 @@ jobs:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
flags: backend-python-${{ matrix.python-version }}
|
||||
files: coverage.xml
|
||||
- name: Upload coverage artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: backend-coverage-${{ matrix.python-version }}
|
||||
path: |
|
||||
.coverage
|
||||
coverage.xml
|
||||
junit.xml
|
||||
retention-days: 1
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
- name: Stop containers
|
||||
if: always()
|
||||
run: |
|
||||
@@ -245,17 +274,6 @@ jobs:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
flags: frontend-node-${{ matrix.node-version }}
|
||||
directory: src-ui/coverage/
|
||||
- name: Upload coverage artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: frontend-coverage-${{ matrix.shard-index }}
|
||||
path: |
|
||||
src-ui/coverage/lcov.info
|
||||
src-ui/coverage/coverage-final.json
|
||||
src-ui/junit.xml
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
tests-frontend-e2e:
|
||||
name: "Frontend E2E Tests (Node ${{ matrix.node-version }} - ${{ matrix.shard-index }}/${{ matrix.shard-count }})"
|
||||
runs-on: ubuntu-24.04
|
||||
@@ -336,74 +354,6 @@ jobs:
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
run: cd src-ui && pnpm run build --configuration=production
|
||||
sonarqube-analysis:
|
||||
name: "SonarQube Analysis"
|
||||
runs-on: ubuntu-24.04
|
||||
needs:
|
||||
- tests-backend
|
||||
- tests-frontend
|
||||
if: github.repository_owner == 'paperless-ngx'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Download all backend coverage
|
||||
uses: actions/download-artifact@v5.0.0
|
||||
with:
|
||||
pattern: backend-coverage-*
|
||||
path: ./coverage/
|
||||
- name: Download all frontend coverage
|
||||
uses: actions/download-artifact@v5.0.0
|
||||
with:
|
||||
pattern: frontend-coverage-*
|
||||
path: ./coverage/
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
|
||||
- name: Install coverage tools
|
||||
run: |
|
||||
pip install coverage
|
||||
npm install -g nyc
|
||||
# Merge backend coverage from all Python versions
|
||||
- name: Merge backend coverage
|
||||
run: |
|
||||
coverage combine coverage/backend-coverage-*/.coverage
|
||||
coverage xml -o merged-backend-coverage.xml
|
||||
# Merge frontend coverage from all shards
|
||||
- name: Merge frontend coverage
|
||||
run: |
|
||||
# Find all coverage-final.json files from the shards, exit with error if none found
|
||||
shopt -s nullglob
|
||||
files=(coverage/frontend-coverage-*/coverage/coverage-final.json)
|
||||
if [ ${#files[@]} -eq 0 ]; then
|
||||
echo "No frontend coverage JSON found under coverage/" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Create .nyc_output directory and copy each shard's coverage JSON into it with a unique name
|
||||
mkdir -p .nyc_output
|
||||
for coverage_json in "${files[@]}"; do
|
||||
shard=$(basename "$(dirname "$(dirname "$coverage_json")")")
|
||||
cp "$coverage_json" ".nyc_output/${shard}.json"
|
||||
done
|
||||
npx nyc merge .nyc_output .nyc_output/out.json
|
||||
npx nyc report --reporter=lcovonly --report-dir coverage
|
||||
- name: Upload coverage artifacts
|
||||
uses: actions/upload-artifact@v4.6.2
|
||||
with:
|
||||
name: merged-coverage
|
||||
path: |
|
||||
merged-backend-coverage.xml
|
||||
.nyc_output/*
|
||||
coverage/lcov.info
|
||||
retention-days: 7
|
||||
if-no-files-found: error
|
||||
include-hidden-files: true
|
||||
- name: SonarQube Analysis
|
||||
uses: SonarSource/sonarqube-scan-action@v5
|
||||
env:
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
build-docker-image:
|
||||
name: Build Docker image for ${{ github.ref_name }}
|
||||
runs-on: ubuntu-24.04
|
||||
|
@@ -33,7 +33,7 @@ dependencies = [
|
||||
"django-cors-headers~=4.8.0",
|
||||
"django-extensions~=4.1",
|
||||
"django-filter~=25.1",
|
||||
"django-guardian~=3.1.2",
|
||||
"django-guardian~=3.2.0",
|
||||
"django-multiselectfield~=1.0.1",
|
||||
"django-soft-delete~=1.0.18",
|
||||
"django-treenode>=0.23.2",
|
||||
@@ -255,7 +255,6 @@ PAPERLESS_DISABLE_DBHANDLER = "true"
|
||||
PAPERLESS_CACHE_BACKEND = "django.core.cache.backends.locmem.LocMemCache"
|
||||
|
||||
[tool.coverage.run]
|
||||
relative_files = true
|
||||
source = [
|
||||
"src/",
|
||||
]
|
||||
|
@@ -1,24 +0,0 @@
|
||||
sonar.projectKey=paperless-ngx_paperless-ngx
|
||||
sonar.organization=paperless-ngx
|
||||
sonar.projectName=Paperless-ngx
|
||||
sonar.projectVersion=1.0
|
||||
|
||||
# Source and test directories
|
||||
sonar.sources=src/,src-ui/
|
||||
sonar.test.inclusions=**/test_*.py,**/tests.py,**/*.spec.ts,**/*.test.ts
|
||||
|
||||
# Language specific settings
|
||||
sonar.python.version=3.10,3.11,3.12,3.13
|
||||
|
||||
# Coverage reports
|
||||
sonar.python.coverage.reportPaths=merged-backend-coverage.xml
|
||||
sonar.javascript.lcov.reportPaths=coverage/lcov.info
|
||||
|
||||
# Test execution reports
|
||||
sonar.junit.reportPaths=**/junit.xml,**/test-results.xml
|
||||
|
||||
# Encoding
|
||||
sonar.sourceEncoding=UTF-8
|
||||
|
||||
# Exclusions
|
||||
sonar.exclusions=**/migrations/**,**/node_modules/**,**/static/**,**/venv/**,**/.venv/**,**/dist/**
|
@@ -177,10 +177,16 @@ export class CustomFieldEditDialogComponent
|
||||
}
|
||||
|
||||
public removeSelectOption(index: number) {
|
||||
this.selectOptions.removeAt(index)
|
||||
this._allSelectOptions.splice(
|
||||
index + (this.selectOptionsPage - 1) * SELECT_OPTION_PAGE_SIZE,
|
||||
1
|
||||
const globalIndex =
|
||||
index + (this.selectOptionsPage - 1) * SELECT_OPTION_PAGE_SIZE
|
||||
this._allSelectOptions.splice(globalIndex, 1)
|
||||
|
||||
const totalPages = Math.max(
|
||||
1,
|
||||
Math.ceil(this._allSelectOptions.length / SELECT_OPTION_PAGE_SIZE)
|
||||
)
|
||||
const targetPage = Math.min(this.selectOptionsPage, totalPages)
|
||||
|
||||
this.selectOptionsPage = targetPage
|
||||
}
|
||||
}
|
||||
|
@@ -164,6 +164,9 @@ class BarcodePlugin(ConsumeTaskPlugin):
|
||||
mailrule_id=self.input_doc.mailrule_id,
|
||||
# Can't use same folder or the consume might grab it again
|
||||
original_file=(tmp_dir / new_document.name).resolve(),
|
||||
# Adding optional original_path for later uses in
|
||||
# workflow matching
|
||||
original_path=self.input_doc.original_file,
|
||||
),
|
||||
# All the same metadata
|
||||
self.metadata,
|
||||
|
@@ -156,6 +156,7 @@ class ConsumableDocument:
|
||||
|
||||
source: DocumentSource
|
||||
original_file: Path
|
||||
original_path: Path | None = None
|
||||
mailrule_id: int | None = None
|
||||
mime_type: str = dataclasses.field(init=False, default=None)
|
||||
|
||||
|
@@ -92,6 +92,9 @@ class Command(MultiProcessMixin, ProgressBarMixin, BaseCommand):
|
||||
# doc to doc is obviously not useful
|
||||
if first_doc.pk == second_doc.pk:
|
||||
continue
|
||||
# Skip empty documents (e.g. password-protected)
|
||||
if first_doc.content.strip() == "" or second_doc.content.strip() == "":
|
||||
continue
|
||||
# Skip matching which have already been matched together
|
||||
# doc 1 to doc 2 is the same as doc 2 to doc 1
|
||||
doc_1_to_doc_2 = (first_doc.pk, second_doc.pk)
|
||||
|
@@ -314,11 +314,19 @@ def consumable_document_matches_workflow(
|
||||
trigger_matched = False
|
||||
|
||||
# Document path vs trigger path
|
||||
|
||||
# Use the original_path if set, else us the original_file
|
||||
match_against = (
|
||||
document.original_path
|
||||
if document.original_path is not None
|
||||
else document.original_file
|
||||
)
|
||||
|
||||
if (
|
||||
trigger.filter_path is not None
|
||||
and len(trigger.filter_path) > 0
|
||||
and not fnmatch(
|
||||
document.original_file,
|
||||
match_against,
|
||||
trigger.filter_path,
|
||||
)
|
||||
):
|
||||
|
@@ -614,14 +614,16 @@ class TestBarcodeNewConsume(
|
||||
self.assertIsNotFile(temp_copy)
|
||||
|
||||
# Check the split files exist
|
||||
# Check the original_path is set
|
||||
# Check the source is unchanged
|
||||
# Check the overrides are unchanged
|
||||
for (
|
||||
new_input_doc,
|
||||
new_doc_overrides,
|
||||
) in self.get_all_consume_delay_call_args():
|
||||
self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder)
|
||||
self.assertIsFile(new_input_doc.original_file)
|
||||
self.assertEqual(new_input_doc.original_path, temp_copy)
|
||||
self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder)
|
||||
self.assertEqual(overrides, new_doc_overrides)
|
||||
|
||||
|
||||
|
@@ -206,3 +206,29 @@ class TestFuzzyMatchCommand(TestCase):
|
||||
self.assertEqual(Document.objects.count(), 2)
|
||||
self.assertIsNotNone(Document.objects.get(pk=1))
|
||||
self.assertIsNotNone(Document.objects.get(pk=2))
|
||||
|
||||
def test_empty_content(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- 2 documents exist, content is empty (pw-protected)
|
||||
WHEN:
|
||||
- Command is called
|
||||
THEN:
|
||||
- No matches are found
|
||||
"""
|
||||
Document.objects.create(
|
||||
checksum="BEEFCAFE",
|
||||
title="A",
|
||||
content="",
|
||||
mime_type="application/pdf",
|
||||
filename="test.pdf",
|
||||
)
|
||||
Document.objects.create(
|
||||
checksum="DEADBEAF",
|
||||
title="A",
|
||||
content="",
|
||||
mime_type="application/pdf",
|
||||
filename="other_test.pdf",
|
||||
)
|
||||
stdout, _ = self.call_command()
|
||||
self.assertIn("No matches found", stdout)
|
||||
|
8
uv.lock
generated
8
uv.lock
generated
@@ -782,15 +782,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "django-guardian"
|
||||
version = "3.1.3"
|
||||
version = "3.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "django", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "typing-extensions", marker = "(python_full_version < '3.13' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'linux')" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/81/d3/436a44c7688fce1a978224c349ba66c95bf9103d548596b7a2694fd58c03/django_guardian-3.1.3.tar.gz", hash = "sha256:12b5e66c18c97088b0adfa033ab14be68c321c170fd3ec438898271f00a71699", size = 93571, upload-time = "2025-09-10T08:36:23.928Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e2/f9/bcff6a931298b9eb55e1550b55ab964fab747f594ba6d2d81cbe19736c5f/django_guardian-3.2.0.tar.gz", hash = "sha256:9e18ecd2e211b665972690c2d03d27bce0ea4932b5efac24a4bb9d526950a69e", size = 99940, upload-time = "2025-09-16T10:35:53.609Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/83/fc/6fd7b8bc7c52cbbfd1714673cfd28ff0b3fae32265c52d492ec0dee22cb8/django_guardian-3.1.3-py3-none-any.whl", hash = "sha256:90e28b40eea65c326a3a961908cc300f9e1cd69b74e88d38317a9befa167b71c", size = 127687, upload-time = "2025-09-10T08:36:22.533Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2f/23/63a7d868373a73d25c4a5c2dd3cce3aaeb22fbee82560d42b6e93ba01403/django_guardian-3.2.0-py3-none-any.whl", hash = "sha256:0768565a057988a93fc4a1d93649c4a794abfd7473a8408a079cfbf83c559d77", size = 134674, upload-time = "2025-09-16T10:35:51.69Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2185,7 +2185,7 @@ requires-dist = [
|
||||
{ name = "django-cors-headers", specifier = "~=4.8.0" },
|
||||
{ name = "django-extensions", specifier = "~=4.1" },
|
||||
{ name = "django-filter", specifier = "~=25.1" },
|
||||
{ name = "django-guardian", specifier = "~=3.1.2" },
|
||||
{ name = "django-guardian", specifier = "~=3.2.0" },
|
||||
{ name = "django-multiselectfield", specifier = "~=1.0.1" },
|
||||
{ name = "django-soft-delete", specifier = "~=1.0.18" },
|
||||
{ name = "django-treenode", specifier = ">=0.23.2" },
|
||||
|
Reference in New Issue
Block a user