From a3c5981865ee0293d42563b7d61741cedc1d1737 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Sat, 10 Jan 2026 14:57:46 -0800 Subject: [PATCH] Re-works the testing to resolve warnings. See about CI? --- .../tests/test_management_consumer.py | 766 +++++++----------- 1 file changed, 286 insertions(+), 480 deletions(-) diff --git a/src/documents/tests/test_management_consumer.py b/src/documents/tests/test_management_consumer.py index 91792ed1b..13ba4635e 100644 --- a/src/documents/tests/test_management_consumer.py +++ b/src/documents/tests/test_management_consumer.py @@ -19,11 +19,11 @@ from pathlib import Path from threading import Thread from time import monotonic from time import sleep -from unittest import mock +from typing import TYPE_CHECKING import pytest +from django import db from django.core.management import CommandError -from django.test import override_settings from watchfiles import Change from documents.data_models import ConsumableDocument @@ -36,6 +36,15 @@ from documents.management.commands.document_consumer import _consume_file from documents.management.commands.document_consumer import _tags_from_path from documents.models import Tag +if TYPE_CHECKING: + from collections.abc import Callable + from collections.abc import Generator + from unittest.mock import MagicMock + + from pytest_django.fixtures import SettingsWrapper + from pytest_mock import MockerFixture + + # -- Fixtures -- @@ -72,7 +81,6 @@ def scratch_dir(tmp_path: Path) -> Path: @pytest.fixture def sample_pdf(tmp_path: Path) -> Path: """Create a sample PDF file.""" - # Use a minimal valid-ish PDF header pdf_content = b"%PDF-1.4\n%test\n1 0 obj\n<<>>\nendobj\ntrailer\n<<>>\n%%EOF" pdf_path = tmp_path / "sample.pdf" pdf_path.write_bytes(pdf_content) @@ -84,18 +92,27 @@ def consumer_filter() -> ConsumerFilter: """Create a ConsumerFilter for testing.""" return ConsumerFilter( supported_extensions=frozenset({".pdf", ".png", ".jpg"}), - ignore_patterns=[r"^custom_ignore.*"], + ignore_patterns=[r"^custom_ignore"], ) @pytest.fixture -def mock_consume_file_delay(): +def mock_consume_file_delay(mocker: MockerFixture) -> MagicMock: """Mock the consume_file.delay celery task.""" - with mock.patch( + mock_task = mocker.patch( "documents.management.commands.document_consumer.consume_file", - ) as mock_task: - mock_task.delay = mock.MagicMock() - yield mock_task + ) + mock_task.delay = mocker.MagicMock() + return mock_task + + +@pytest.fixture +def mock_supported_extensions(mocker: MockerFixture) -> MagicMock: + """Mock get_supported_file_extensions to return only .pdf.""" + return mocker.patch( + "documents.management.commands.document_consumer.get_supported_file_extensions", + return_value={".pdf"}, + ) # -- TrackedFile Tests -- @@ -132,10 +149,7 @@ class TestTrackedFile: """Test is_unchanged returns False when file is modified.""" tracked = TrackedFile(path=temp_file, last_event_time=monotonic()) tracked.update_stats() - - # Modify the file temp_file.write_bytes(b"modified content that is longer") - assert tracked.is_unchanged() is False def test_is_unchanged_deleted_file(self, temp_file: Path) -> None: @@ -171,8 +185,6 @@ class TestFileStabilityTracker: stability_tracker.track(temp_file, Change.added) sleep(0.05) stability_tracker.track(temp_file, Change.modified) - - # File should still be pending, not yet stable assert stability_tracker.pending_count == 1 def test_track_deleted_file( @@ -183,7 +195,6 @@ class TestFileStabilityTracker: """Test tracking a deleted file removes it from pending.""" stability_tracker.track(temp_file, Change.added) assert stability_tracker.pending_count == 1 - stability_tracker.track(temp_file, Change.deleted) assert stability_tracker.pending_count == 0 assert stability_tracker.has_pending_files() is False @@ -216,8 +227,7 @@ class TestFileStabilityTracker: ) -> None: """Test get_stable_files returns file after delay expires.""" stability_tracker.track(temp_file, Change.added) - sleep(0.15) # Wait longer than stability_delay (0.1s) - + sleep(0.15) stable = list(stability_tracker.get_stable_files()) assert len(stable) == 1 assert stable[0] == temp_file @@ -231,27 +241,17 @@ class TestFileStabilityTracker: """Test file is not returned if modified during stability check.""" stability_tracker.track(temp_file, Change.added) sleep(0.12) - - # Modify file just before checking temp_file.write_bytes(b"modified content") - stable = list(stability_tracker.get_stable_files()) assert len(stable) == 0 - # File should be re-tracked with new event time assert stability_tracker.pending_count == 1 - def test_get_stable_files_deleted_during_check( - self, - temp_file: Path, - ) -> None: + def test_get_stable_files_deleted_during_check(self, temp_file: Path) -> None: """Test deleted file is not returned during stability check.""" tracker = FileStabilityTracker(stability_delay=0.1) tracker.track(temp_file, Change.added) sleep(0.12) - - # Delete file just before checking temp_file.unlink() - stable = list(tracker.get_stable_files()) assert len(stable) == 0 assert tracker.pending_count == 0 @@ -273,13 +273,11 @@ class TestFileStabilityTracker: assert stability_tracker.pending_count == 2 - # Wait for file1 to be stable (but not file2) sleep(0.06) stable = list(stability_tracker.get_stable_files()) assert len(stable) == 1 assert stable[0] == file1 - # Now wait for file2 sleep(0.06) stable = list(stability_tracker.get_stable_files()) assert len(stable) == 1 @@ -293,7 +291,6 @@ class TestFileStabilityTracker: """Test clear removes all tracked files.""" stability_tracker.track(temp_file, Change.added) assert stability_tracker.pending_count == 1 - stability_tracker.clear() assert stability_tracker.pending_count == 0 assert stability_tracker.has_pending_files() is False @@ -304,12 +301,8 @@ class TestFileStabilityTracker: temp_file: Path, ) -> None: """Test that tracking resolves paths consistently.""" - # Track with relative-looking path stability_tracker.track(temp_file, Change.added) - - # Track again with resolved path - should update, not add stability_tracker.track(temp_file.resolve(), Change.modified) - assert stability_tracker.pending_count == 1 @@ -319,179 +312,101 @@ class TestFileStabilityTracker: class TestConsumerFilter: """Tests for the ConsumerFilter class.""" - def test_accepts_supported_extension( + @pytest.mark.parametrize( + ("filename", "should_accept"), + [ + pytest.param("document.pdf", True, id="supported_pdf"), + pytest.param("image.png", True, id="supported_png"), + pytest.param("photo.jpg", True, id="supported_jpg"), + pytest.param("document.PDF", True, id="case_insensitive"), + pytest.param("document.xyz", False, id="unsupported_ext"), + pytest.param("document", False, id="no_extension"), + pytest.param(".DS_Store", False, id="ds_store"), + pytest.param(".DS_STORE", False, id="ds_store_upper"), + pytest.param("._document.pdf", False, id="macos_resource_fork"), + pytest.param("._hidden", False, id="macos_resource_no_ext"), + pytest.param("Thumbs.db", False, id="thumbs_db"), + pytest.param("desktop.ini", False, id="desktop_ini"), + pytest.param("custom_ignore_this.pdf", False, id="custom_pattern"), + pytest.param("stfolder.pdf", True, id="similar_to_ignored"), + pytest.param("my_document.pdf", True, id="normal_with_underscore"), + ], + ) + def test_file_filtering( self, consumer_filter: ConsumerFilter, tmp_path: Path, + filename: str, + should_accept: bool, # noqa: FBT001 ) -> None: - """Test filter accepts files with supported extensions.""" - test_file = tmp_path / "document.pdf" + """Test filter correctly accepts or rejects files.""" + test_file = tmp_path / filename test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is True + assert consumer_filter(Change.added, str(test_file)) is should_accept - def test_rejects_unsupported_extension( + @pytest.mark.parametrize( + ("dirname", "should_accept"), + [ + pytest.param(".stfolder", False, id="syncthing_stfolder"), + pytest.param(".stversions", False, id="syncthing_stversions"), + pytest.param("@eaDir", False, id="synology_eadir"), + pytest.param(".Spotlight-V100", False, id="macos_spotlight"), + pytest.param(".Trashes", False, id="macos_trashes"), + pytest.param("__MACOSX", False, id="macos_archive"), + pytest.param(".localized", False, id="macos_localized"), + pytest.param("documents", True, id="normal_dir"), + pytest.param("invoices", True, id="normal_dir_2"), + ], + ) + def test_directory_filtering( self, consumer_filter: ConsumerFilter, tmp_path: Path, + dirname: str, + should_accept: bool, # noqa: FBT001 ) -> None: - """Test filter rejects files with unsupported extensions.""" - test_file = tmp_path / "document.xyz" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is False + """Test filter correctly accepts or rejects directories.""" + test_dir = tmp_path / dirname + test_dir.mkdir() + assert consumer_filter(Change.added, str(test_dir)) is should_accept - def test_rejects_no_extension( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects files without extensions.""" - test_file = tmp_path / "document" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is False - - def test_case_insensitive_extension( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter handles extensions case-insensitively.""" - test_file = tmp_path / "document.PDF" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is True - - def test_rejects_ds_store( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects .DS_Store files.""" - test_file = tmp_path / ".DS_Store" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is False - - def test_rejects_macos_resource_fork( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects macOS resource fork files (._*).""" - test_file = tmp_path / "._document.pdf" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is False - - def test_rejects_syncthing_folder( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects .stfolder directory via ignore_dirs.""" - stfolder = tmp_path / ".stfolder" - stfolder.mkdir() - # DefaultFilter ignores directories by name - assert consumer_filter(Change.added, str(stfolder)) is False - - def test_rejects_syncthing_versions( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects .stversions directory via ignore_dirs.""" - stversions = tmp_path / ".stversions" - stversions.mkdir() - assert consumer_filter(Change.added, str(stversions)) is False - - def test_rejects_synology_eadir( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects Synology @eaDir directory via ignore_dirs.""" - eadir = tmp_path / "@eaDir" - eadir.mkdir() - assert consumer_filter(Change.added, str(eadir)) is False - - def test_rejects_thumbs_db( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects Thumbs.db.""" - test_file = tmp_path / "Thumbs.db" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is False - - def test_rejects_desktop_ini( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter rejects desktop.ini.""" - test_file = tmp_path / "desktop.ini" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is False - - def test_custom_ignore_pattern( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter respects custom ignore patterns.""" - test_file = tmp_path / "custom_ignore_this.pdf" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is False - - def test_accepts_similar_to_ignored( - self, - consumer_filter: ConsumerFilter, - tmp_path: Path, - ) -> None: - """Test filter accepts files similar to but not matching ignore patterns.""" - test_file = tmp_path / "stfolder.pdf" - test_file.touch() - assert consumer_filter(Change.added, str(test_file)) is True - - def test_default_patterns_are_regex(self) -> None: + def test_default_patterns_are_valid_regex(self) -> None: """Test that default patterns are valid regex.""" for pattern in ConsumerFilter.DEFAULT_IGNORE_PATTERNS: - # Should not raise re.compile(pattern) -class TestConsumerFilterWithoutExtensions: - """Tests for ConsumerFilter edge cases.""" +class TestConsumerFilterDefaults: + """Tests for ConsumerFilter with default settings.""" - def test_filter_works_with_default_extensions(self, tmp_path: Path) -> None: - """Test filter works when using default extensions.""" - # This would use get_supported_file_extensions() in real usage - filter_obj = ConsumerFilter( - supported_extensions=frozenset({".pdf"}), + def test_filter_with_mocked_extensions( + self, + tmp_path: Path, + mocker: MockerFixture, + ) -> None: + """Test filter works when using mocked extensions from parser.""" + mocker.patch( + "documents.management.commands.document_consumer.get_supported_file_extensions", + return_value={".pdf", ".png"}, ) + filter_obj = ConsumerFilter() test_file = tmp_path / "document.pdf" test_file.touch() assert filter_obj(Change.added, str(test_file)) is True - def test_ignores_patterns_by_filename(self, tmp_path: Path) -> None: - """Test filter ignores patterns matched against filename only.""" - filter_obj = ConsumerFilter( - supported_extensions=frozenset({".pdf"}), - ) - test_file = tmp_path / ".DS_Store" - test_file.touch() - assert filter_obj(Change.added, str(test_file)) is False - # -- _consume_file Tests -- +@pytest.mark.django_db class TestConsumeFile: """Tests for the _consume_file function.""" - @pytest.mark.django_db def test_consume_queues_file( self, consumption_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, ) -> None: """Test _consume_file queues a valid file.""" target = consumption_dir / "document.pdf" @@ -510,11 +425,10 @@ class TestConsumeFile: assert consumable_doc.original_file == target assert consumable_doc.source == DocumentSource.ConsumeFolder - @pytest.mark.django_db def test_consume_nonexistent_file( self, consumption_dir: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, ) -> None: """Test _consume_file handles nonexistent files gracefully.""" _consume_file( @@ -522,14 +436,12 @@ class TestConsumeFile: consumption_dir=consumption_dir, subdirs_as_tags=False, ) - mock_consume_file_delay.delay.assert_not_called() - @pytest.mark.django_db def test_consume_directory( self, consumption_dir: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, ) -> None: """Test _consume_file ignores directories.""" subdir = consumption_dir / "subdir" @@ -540,37 +452,35 @@ class TestConsumeFile: consumption_dir=consumption_dir, subdirs_as_tags=False, ) - mock_consume_file_delay.delay.assert_not_called() - @pytest.mark.django_db def test_consume_with_permission_error( self, consumption_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + mocker: MockerFixture, ) -> None: """Test _consume_file handles permission errors.""" target = consumption_dir / "document.pdf" shutil.copy(sample_pdf, target) - with mock.patch.object(Path, "is_file", side_effect=PermissionError("denied")): - _consume_file( - filepath=target, - consumption_dir=consumption_dir, - subdirs_as_tags=False, - ) - + mocker.patch.object(Path, "is_file", side_effect=PermissionError("denied")) + _consume_file( + filepath=target, + consumption_dir=consumption_dir, + subdirs_as_tags=False, + ) mock_consume_file_delay.delay.assert_not_called() # -- _tags_from_path Tests -- +@pytest.mark.django_db class TestTagsFromPath: """Tests for the _tags_from_path function.""" - @pytest.mark.django_db def test_creates_tags_from_subdirectories(self, consumption_dir: Path) -> None: """Test tags are created for each subdirectory.""" subdir = consumption_dir / "Invoice" / "2024" @@ -584,7 +494,6 @@ class TestTagsFromPath: assert Tag.objects.filter(name="Invoice").exists() assert Tag.objects.filter(name="2024").exists() - @pytest.mark.django_db def test_reuses_existing_tags(self, consumption_dir: Path) -> None: """Test existing tags are reused (case-insensitive).""" existing_tag = Tag.objects.create(name="existing") @@ -598,10 +507,8 @@ class TestTagsFromPath: assert len(tag_ids) == 1 assert existing_tag.pk in tag_ids - # Should not create a duplicate assert Tag.objects.filter(name__iexact="existing").count() == 1 - @pytest.mark.django_db def test_no_tags_for_root_file(self, consumption_dir: Path) -> None: """Test no tags created for files directly in consumption dir.""" target = consumption_dir / "document.pdf" @@ -618,12 +525,15 @@ class TestTagsFromPath: class TestCommandValidation: """Tests for command argument validation.""" - def test_raises_for_missing_consumption_dir(self) -> None: - """Test command raises error when directory is not provided and setting is unset.""" - with override_settings(CONSUMPTION_DIR=None): - with pytest.raises(CommandError, match="not configured"): - cmd = Command() - cmd.handle(directory=None, oneshot=True, testing=False) + def test_raises_for_missing_consumption_dir( + self, + settings: SettingsWrapper, + ) -> None: + """Test command raises error when directory is not provided.""" + settings.CONSUMPTION_DIR = None + with pytest.raises(CommandError, match="not configured"): + cmd = Command() + cmd.handle(directory=None, oneshot=True, testing=False) def test_raises_for_nonexistent_directory(self, tmp_path: Path) -> None: """Test command raises error for nonexistent directory.""" @@ -633,10 +543,7 @@ class TestCommandValidation: cmd = Command() cmd.handle(directory=str(nonexistent), oneshot=True, testing=False) - def test_raises_for_file_instead_of_directory( - self, - sample_pdf: Path, - ) -> None: + def test_raises_for_file_instead_of_directory(self, sample_pdf: Path) -> None: """Test command raises error when path is a file, not directory.""" with pytest.raises(CommandError, match="not a directory"): cmd = Command() @@ -646,40 +553,38 @@ class TestCommandValidation: # -- Command Oneshot Tests -- +@pytest.mark.django_db +@pytest.mark.usefixtures("mock_supported_extensions") class TestCommandOneshot: """Tests for oneshot mode.""" - @pytest.mark.django_db def test_processes_existing_files( self, consumption_dir: Path, scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + settings: SettingsWrapper, ) -> None: """Test oneshot mode processes existing files.""" target = consumption_dir / "document.pdf" shutil.copy(sample_pdf, target) - with ( - override_settings(SCRATCH_DIR=scratch_dir), - mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ), - ): - cmd = Command() - cmd.handle(directory=str(consumption_dir), oneshot=True, testing=False) + settings.SCRATCH_DIR = scratch_dir + settings.CONSUMER_IGNORE_PATTERNS = [] + + cmd = Command() + cmd.handle(directory=str(consumption_dir), oneshot=True, testing=False) mock_consume_file_delay.delay.assert_called_once() - @pytest.mark.django_db def test_processes_recursive( self, consumption_dir: Path, scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + settings: SettingsWrapper, ) -> None: """Test oneshot mode processes files recursively.""" subdir = consumption_dir / "subdir" @@ -687,38 +592,31 @@ class TestCommandOneshot: target = subdir / "document.pdf" shutil.copy(sample_pdf, target) - with ( - override_settings(SCRATCH_DIR=scratch_dir, CONSUMER_RECURSIVE=True), - mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ), - ): - cmd = Command() - cmd.handle(directory=str(consumption_dir), oneshot=True, testing=False) + settings.SCRATCH_DIR = scratch_dir + settings.CONSUMER_RECURSIVE = True + settings.CONSUMER_IGNORE_PATTERNS = [] + + cmd = Command() + cmd.handle(directory=str(consumption_dir), oneshot=True, testing=False) mock_consume_file_delay.delay.assert_called_once() - @pytest.mark.django_db def test_ignores_unsupported_extensions( self, consumption_dir: Path, scratch_dir: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + settings: SettingsWrapper, ) -> None: """Test oneshot mode ignores unsupported file extensions.""" target = consumption_dir / "document.xyz" target.write_bytes(b"content") - with ( - override_settings(SCRATCH_DIR=scratch_dir), - mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ), - ): - cmd = Command() - cmd.handle(directory=str(consumption_dir), oneshot=True, testing=False) + settings.SCRATCH_DIR = scratch_dir + settings.CONSUMER_IGNORE_PATTERNS = [] + + cmd = Command() + cmd.handle(directory=str(consumption_dir), oneshot=True, testing=False) mock_consume_file_delay.delay.assert_not_called() @@ -727,7 +625,7 @@ class TestCommandOneshot: class ConsumerThread(Thread): - """Thread wrapper for running the consumer command.""" + """Thread wrapper for running the consumer command with proper cleanup.""" def __init__( self, @@ -748,409 +646,317 @@ class ConsumerThread(Thread): self.stability_delay = stability_delay self.cmd = Command() self.cmd.stop_flag.clear() - self.daemon = True + # Non-daemon ensures finally block runs and connections are closed + self.daemon = False self.exception: Exception | None = None def run(self) -> None: try: - # Apply settings overrides within the thread - with override_settings( - SCRATCH_DIR=self.scratch_dir, - CONSUMER_RECURSIVE=self.recursive, - CONSUMER_SUBDIRS_AS_TAGS=self.subdirs_as_tags, - CONSUMER_POLLING_INTERVAL=self.polling_interval, - CONSUMER_STABILITY_DELAY=self.stability_delay, - CONSUMER_IGNORE_PATTERNS=[], - ): - self.cmd.handle( - directory=str(self.consumption_dir), - oneshot=False, - testing=True, - ) + from django.conf import settings + + # Apply settings directly (thread-safe for reading) + settings.SCRATCH_DIR = self.scratch_dir + settings.CONSUMER_RECURSIVE = self.recursive + settings.CONSUMER_SUBDIRS_AS_TAGS = self.subdirs_as_tags + settings.CONSUMER_POLLING_INTERVAL = self.polling_interval + settings.CONSUMER_STABILITY_DELAY = self.stability_delay + settings.CONSUMER_IGNORE_PATTERNS = [] + + self.cmd.handle( + directory=str(self.consumption_dir), + oneshot=False, + testing=True, + ) except Exception as e: self.exception = e + finally: + # Close database connections created in this thread + db.connections.close_all() def stop(self) -> None: self.cmd.stop_flag.set() + def stop_and_wait(self, timeout: float = 5.0) -> None: + """Stop the thread and wait for it to finish, with cleanup.""" + self.stop() + self.join(timeout=timeout) + if self.is_alive(): + # Thread didn't stop in time - this is a test failure + raise RuntimeError( + f"Consumer thread did not stop within {timeout}s timeout", + ) + +@pytest.fixture +def start_consumer( + consumption_dir: Path, + scratch_dir: Path, + mock_supported_extensions: MagicMock, +) -> Generator[Callable[..., ConsumerThread], None, None]: + """Start a consumer thread and ensure cleanup.""" + threads: list[ConsumerThread] = [] + + def _start(**kwargs) -> ConsumerThread: + thread = ConsumerThread(consumption_dir, scratch_dir, **kwargs) + threads.append(thread) + thread.start() + sleep(0.5) # Give thread time to start + return thread + + yield _start + + # Cleanup all threads that were started + for thread in threads: + thread.stop() + + for thread in threads: + thread.join(timeout=5.0) + if thread.is_alive(): + pytest.fail("Consumer thread did not stop within timeout") + + +@pytest.mark.django_db class TestCommandWatch: """Integration tests for the watch loop.""" - @pytest.mark.django_db def test_detects_new_file( self, consumption_dir: Path, - scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test watch mode detects and consumes new files.""" - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread(consumption_dir, scratch_dir) - thread.start() + thread = start_consumer() - # Give thread time to start watching - sleep(0.5) + target = consumption_dir / "document.pdf" + shutil.copy(sample_pdf, target) + sleep(0.5) - # Copy file - target = consumption_dir / "document.pdf" - shutil.copy(sample_pdf, target) - - # Wait for stability delay + processing - sleep(0.5) - - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception + if thread.exception: + raise thread.exception mock_consume_file_delay.delay.assert_called() - @pytest.mark.django_db def test_detects_moved_file( self, consumption_dir: Path, scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test watch mode detects moved/renamed files.""" - # Create temp file outside consumption dir temp_location = scratch_dir / "temp.pdf" shutil.copy(sample_pdf, temp_location) - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread(consumption_dir, scratch_dir) - thread.start() + thread = start_consumer() - sleep(0.5) + target = consumption_dir / "document.pdf" + shutil.move(temp_location, target) + sleep(0.5) - # Move file into consumption dir - target = consumption_dir / "document.pdf" - shutil.move(temp_location, target) - - sleep(0.5) - - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception + if thread.exception: + raise thread.exception mock_consume_file_delay.delay.assert_called() - @pytest.mark.django_db def test_handles_slow_write( self, consumption_dir: Path, - scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test watch mode waits for slow writes to complete.""" pdf_bytes = sample_pdf.read_bytes() - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread( - consumption_dir, - scratch_dir, - stability_delay=0.2, - ) - thread.start() + thread = start_consumer(stability_delay=0.2) - sleep(0.5) + target = consumption_dir / "document.pdf" + with target.open("wb") as f: + for i in range(0, len(pdf_bytes), 100): + f.write(pdf_bytes[i : i + 100]) + f.flush() + sleep(0.05) - # Simulate slow write - target = consumption_dir / "document.pdf" - with target.open("wb") as f: - for i in range(0, len(pdf_bytes), 100): - f.write(pdf_bytes[i : i + 100]) - f.flush() - sleep(0.05) + sleep(0.5) - # Wait for stability - sleep(0.5) - - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception + if thread.exception: + raise thread.exception mock_consume_file_delay.delay.assert_called() - @pytest.mark.django_db def test_ignores_macos_files( self, consumption_dir: Path, - scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test watch mode ignores macOS system files.""" - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread(consumption_dir, scratch_dir) - thread.start() + thread = start_consumer() - sleep(0.5) + (consumption_dir / ".DS_Store").write_bytes(b"test") + (consumption_dir / "._document.pdf").write_bytes(b"test") + shutil.copy(sample_pdf, consumption_dir / "valid.pdf") - # Create macOS files - (consumption_dir / ".DS_Store").write_bytes(b"test") - (consumption_dir / "._document.pdf").write_bytes(b"test") + sleep(0.5) - # Also create a valid file to confirm filtering works - shutil.copy(sample_pdf, consumption_dir / "valid.pdf") + if thread.exception: + raise thread.exception - sleep(0.5) - - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception - - # Should only consume the valid file assert mock_consume_file_delay.delay.call_count == 1 call_args = mock_consume_file_delay.delay.call_args[0][0] assert call_args.original_file.name == "valid.pdf" - @pytest.mark.django_db + @pytest.mark.usefixtures("mock_supported_extensions") def test_stop_flag_stops_consumer( self, consumption_dir: Path, scratch_dir: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, ) -> None: """Test stop flag properly stops the consumer.""" - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread(consumption_dir, scratch_dir) + thread = ConsumerThread(consumption_dir, scratch_dir) + try: thread.start() - sleep(0.3) assert thread.is_alive() + finally: + thread.stop_and_wait(timeout=5.0) - thread.stop() - thread.join(timeout=2.0) - - assert not thread.is_alive() + assert not thread.is_alive() +@pytest.mark.django_db class TestCommandWatchPolling: """Tests for polling mode.""" - @pytest.mark.django_db def test_polling_mode_works( self, consumption_dir: Path, - scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test polling mode detects files.""" - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread( - consumption_dir, - scratch_dir, - polling_interval=0.5, # Enable polling - ) - thread.start() + thread = start_consumer(polling_interval=0.5) - sleep(0.5) + target = consumption_dir / "document.pdf" + shutil.copy(sample_pdf, target) - target = consumption_dir / "document.pdf" - shutil.copy(sample_pdf, target) + sleep(1.5) - # Polling needs more time - sleep(1.5) - - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception + if thread.exception: + raise thread.exception mock_consume_file_delay.delay.assert_called() +@pytest.mark.django_db class TestCommandWatchRecursive: """Tests for recursive watching.""" - @pytest.mark.django_db def test_recursive_detects_nested_files( self, consumption_dir: Path, - scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test recursive mode detects files in subdirectories.""" subdir = consumption_dir / "level1" / "level2" subdir.mkdir(parents=True) - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread( - consumption_dir, - scratch_dir, - recursive=True, - ) - thread.start() + thread = start_consumer(recursive=True) - sleep(0.5) + target = subdir / "document.pdf" + shutil.copy(sample_pdf, target) + sleep(0.5) - target = subdir / "document.pdf" - shutil.copy(sample_pdf, target) - - sleep(0.5) - - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception + if thread.exception: + raise thread.exception mock_consume_file_delay.delay.assert_called() - @pytest.mark.django_db def test_subdirs_as_tags( self, consumption_dir: Path, - scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test subdirs_as_tags creates tags from directory names.""" subdir = consumption_dir / "Invoices" / "2024" subdir.mkdir(parents=True) - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread( - consumption_dir, - scratch_dir, - recursive=True, - subdirs_as_tags=True, - ) - thread.start() + thread = start_consumer(recursive=True, subdirs_as_tags=True) - sleep(0.5) + target = subdir / "document.pdf" + shutil.copy(sample_pdf, target) + sleep(0.5) - target = subdir / "document.pdf" - shutil.copy(sample_pdf, target) - - sleep(0.5) - - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception + if thread.exception: + raise thread.exception mock_consume_file_delay.delay.assert_called() - # Check tags were passed call_args = mock_consume_file_delay.delay.call_args overrides = call_args[0][1] assert overrides.tag_ids is not None assert len(overrides.tag_ids) == 2 +@pytest.mark.django_db class TestCommandWatchEdgeCases: """Tests for edge cases and error handling.""" - @pytest.mark.django_db def test_handles_deleted_before_stable( self, consumption_dir: Path, - scratch_dir: Path, sample_pdf: Path, - mock_consume_file_delay, + mock_consume_file_delay: MagicMock, + start_consumer: Callable[..., ConsumerThread], ) -> None: """Test handles files deleted before becoming stable.""" - with mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ): - thread = ConsumerThread( - consumption_dir, - scratch_dir, - stability_delay=0.3, # Longer delay - ) - thread.start() + thread = start_consumer(stability_delay=0.3) - sleep(0.3) + target = consumption_dir / "document.pdf" + shutil.copy(sample_pdf, target) + sleep(0.1) + target.unlink() - # Create and quickly delete - target = consumption_dir / "document.pdf" - shutil.copy(sample_pdf, target) - sleep(0.1) # Before stability delay - target.unlink() + sleep(0.5) - sleep(0.5) + if thread.exception: + raise thread.exception - thread.stop() - thread.join(timeout=2.0) - - if thread.exception: - raise thread.exception - - # Should not have consumed the deleted file mock_consume_file_delay.delay.assert_not_called() - @pytest.mark.django_db + @pytest.mark.usefixtures("mock_supported_extensions") def test_handles_task_exception( self, consumption_dir: Path, scratch_dir: Path, sample_pdf: Path, + mocker: MockerFixture, ) -> None: """Test handles exceptions from consume task gracefully.""" - with ( - mock.patch( - "documents.management.commands.document_consumer.consume_file", - ) as mock_task, - mock.patch( - "documents.management.commands.document_consumer.get_supported_file_extensions", - return_value={".pdf"}, - ), - ): - mock_task.delay.side_effect = Exception("Task error") + mock_task = mocker.patch( + "documents.management.commands.document_consumer.consume_file", + ) + mock_task.delay.side_effect = Exception("Task error") - thread = ConsumerThread(consumption_dir, scratch_dir) + thread = ConsumerThread(consumption_dir, scratch_dir) + try: thread.start() - sleep(0.3) target = consumption_dir / "document.pdf" shutil.copy(sample_pdf, target) - sleep(0.5) # Consumer should still be running despite the exception assert thread.is_alive() - - thread.stop() - thread.join(timeout=2.0) + finally: + thread.stop_and_wait(timeout=5.0)