diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py
index 27aa37908..586897585 100755
--- a/src/documents/signals/handlers.py
+++ b/src/documents/signals/handlers.py
@@ -124,9 +124,9 @@ def set_tags(sender,
              **kwargs):
 
     if replace:
-        document.tags.exclude(
-            Q(is_inbox_tag=True) |
-            (Q(match="") & ~Q(matching_algorithm=Tag.MATCH_AUTO))
+        Document.tags.through.objects.filter(document=document).exclude(
+            Q(tag__is_inbox_tag=True)).exclude(
+            Q(tag__match="") & ~Q(tag__matching_algorithm=Tag.MATCH_AUTO)
         ).delete()
 
     current_tags = set(document.tags.all())
diff --git a/src/documents/tests/test_management_retagger.py b/src/documents/tests/test_management_retagger.py
index 2d2533341..907a23d09 100644
--- a/src/documents/tests/test_management_retagger.py
+++ b/src/documents/tests/test_management_retagger.py
@@ -20,6 +20,7 @@ class TestRetagger(DirectoriesMixin, TestCase):
         self.d3.tags.add(self.tag_inbox)
         self.d3.tags.add(self.tag_no_match)
 
+
         self.correspondent_first = Correspondent.objects.create(
             name="c1", match="first", matching_algorithm=Correspondent.MATCH_ANY)
         self.correspondent_second = Correspondent.objects.create(
@@ -62,11 +63,16 @@ class TestRetagger(DirectoriesMixin, TestCase):
         self.assertEqual(d_first.correspondent, self.correspondent_first)
         self.assertEqual(d_second.correspondent, self.correspondent_second)
 
-    def test_force_preserve_inbox(self):
+    def test_overwrite_preserve_inbox(self):
+        self.d1.tags.add(self.tag_second)
+
         call_command('document_retagger', '--tags', '--overwrite')
 
         d_first, d_second, d_unrelated = self.get_updated_docs()
 
+        self.assertIsNotNone(Tag.objects.get(id=self.tag_second.id))
+
         self.assertCountEqual([tag.id for tag in d_first.tags.all()], [self.tag_first.id])
         self.assertCountEqual([tag.id for tag in d_second.tags.all()], [self.tag_second.id])
         self.assertCountEqual([tag.id for tag in d_unrelated.tags.all()], [self.tag_inbox.id, self.tag_no_match.id])
+