paradizelost/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2025-07-30 18:27:45 -05:00
Code Issues Packages Projects Releases Wiki Activity

Allows configuration of the NLTK processing language

Browse Source
This commit is contained in:
Trenton H
2022-09-16 13:57:37 -07:00
parent 70b1988a55
commit 0bc13c2a72
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/documents/classifier.py
View File

@@ -320,10 +320,9 @@ class DocumentClassifier:
# Get only the letters (remove punctuation too)
content = re.sub(r"[^\w\s]", " ", content)
# Tokenize
# TODO configurable language
words: List[str] = word_tokenize(content, language="english")
words: List[str] = word_tokenize(content, language=settings.NLTK_LANGUAGE)
# Remove stop words
stops = set(stopwords.words("english"))
stops = set(stopwords.words(settings.NLTK_LANGUAGE))
meaningful_words = [w for w in words if w not in stops]
# Stem words
meaningful_words = [self.stemmer.stem(w) for w in meaningful_words]