fixes and updates

Author: clovis

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name="text_preprocessing",
-    version="0.8.4",
+    version="1.0",
     author="The ARTFL Project",
     author_email="clovisgladstone@gmail.com",
     packages=["text_preprocessing", "text_preprocessing.lang"],

text_preprocessing/preprocessor.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 """Text Preprocessor"""
 
-import json
 import os
 import re
 import sqlite3
@@ -25,7 +24,7 @@
 )
 
 import lz4.frame
-import rapidjson
+import orjson
 from multiprocess.pool import Pool
 from spacy.tokens import Doc
 
@@ -287,13 +286,14 @@ def save(self, path):
         tokens_to_serialize = {"tokens": [], "metadata": self.metadata}
         for token in self:
             tokens_to_serialize["tokens"].append((token.text, token.surface_form, token.pos_, token.ext))
-        with open(path, "w") as output:
-            json.dump(tokens_to_serialize, output)
+        with open(path, "wb") as output:
+            output.write(orjson.dumps(tokens_to_serialize))
 
     def load(self, path):
         """Load tokens from disk"""
         with open(path, "r") as input_file:
-            tokens = json.load(input_file)
+            data = input_file.read()
+        tokens = orjson.loads(data)
         self.metadata = tokens["metadata"]
         self.tokens = deque(Token(t[0], t[1], t[2], t[3]) for t in tokens["tokens"])
 
@@ -361,7 +361,7 @@ def __init__(
         hash_tokens: bool = False,
         workers: Optional[int] = None,
         post_processing_function: Optional[Callable] = None,
-        **extra_options,  # this is meant to make the constructor accept invalid keywords
+        **_,  # this is meant to make the constructor accept invalid keywords
     ):
         cls.language = language
         cls.is_philo_db = is_philo_db
@@ -502,7 +502,7 @@ def process_philo_text(cls, text: str, fetch_metadata: bool = True):
             open_file = open
         with open_file(text) as philo_db_text:
             for line in philo_db_text:
-                word_obj: Dict[str, Any] = rapidjson.loads(line.strip())
+                word_obj: Dict[str, Any] = orjson.loads(line.strip())
                 object_id = " ".join(word_obj["position"].split()[: PHILO_TEXT_OBJECT_TYPE[cls.text_object_type]])
                 if current_object_id == "":
                     current_object_id = object_id

text_preprocessing/spacy_helpers.py

@@ -13,25 +13,32 @@
 from modernizer import Modernizer
 
 
-# Updated as of 3/22/2021
+# Updated as of 8/23/2022
 SPACY_LANGUAGE_MODEL_MAP: Dict[str, List[str]] = {
+    "catalan": ["ca_core_news_sm", "ca_core_news_md", "ca_core_news_lg", "ca_core_news_trf"],
+    "chinese": ["zh_core_web_sm", "zh_core_web_md", "zh_core_web_lg", "zh_core_web_trf"],
+    "croation": ["hr_core_news_sm", "hr_core_news_md", "hr_core_news_lg"],
     "danish": ["da_core_news_sm", "da_core_news_md", "da_core_news_lg"],
+    "dutch": ["nl_core_news_sm", "nl_core_news_md", "nl_core_news_lg"],
+    "english": ["en_core_web_sm", "en_core_web_md", "en_core_web_lg", "en_core_web_trf"],
+    "finnish": ["fi_core_news_sm", "fi_core_news_md", "fi_core_news_lg"],
     "german": ["de_core_news_sm", "de_core_news_md", "de_core_news_lg", "de_dep_news_trf"],
     "greek": ["el_core_news_sm", "el_core_news_md", "el_core_news_lg"],
-    "english": ["en_core_web_sm", "en_core_web_md", "en_core_web_lg", "en_core_web_trf"],
-    "spanish": ["es_core_news_sm", "es_core_news_md", "es_core_news_lg", "es_dep_news_trf"],
     "french": ["fr_core_news_sm", "fr_core_news_md", "fr_core_news_lg", "fr_dep_news_trf"],
     "italian": ["it_core_news_sm", "it_core_news_md", "it_core_news_lg"],
     "japanese": ["ja_core_news_sm", "ja_core_news_md", "ja_core_news_lg"],
+    "korean": ["ko_core_news_sm", "ko_core_news_md", "ko_core_news_lg"],
     "lithuanian": ["lt_core_news_sm", "lt_core_news_md", "lt_core_news_lg"],
-    "norwegian bokmål": ["nb_core_news_sm", "nb_core_news_md", "nb_core_news_lg"],
-    "dutch": ["nl_core_news_sm", "nl_core_news_md", "nl_core_news_lg"],
+    "macedonian": ["mk_core_news_sm", "mk_core_news_md", "mk_core_news_lg"],
+    "norwegian": ["nb_core_news_sm", "nb_core_news_md", "nb_core_news_lg"],
     "polish": ["pl_core_news_sm", "pl_core_news_md", "pl_core_news_lg"],
     "portuguese": ["pt_core_news_sm", "pt_core_news_md", "pt_core_news_lg"],
     "romanian": ["ro_core_news_sm", "ro_core_news_md", "ro_core_news_lg"],
     "russian": ["ru_core_news_sm", "ru_core_news_md", "ru_core_news_lg"],
+    "spanish": ["es_core_news_sm", "es_core_news_md", "es_core_news_lg", "es_dep_news_trf"],
+    "swedish": ["sv_core_news_sm", "sv_core_news_md", "sv_core_news_lg"],
+    "ukrainian": ["uk_core_news_sm", "uk_core_news_md", "uk_core_news_lg"],
     "multi-language": ["xx_ent_wiki_sm", "xx_sent_ud_sm"],
-    "chinese": ["zh_core_web_sm", "zh_core_web_md", "zh_core_web_lg", "zh_core_web_trf"],
 }
 
 
@@ -343,13 +350,18 @@ def load_language_model(
     nlp.add_pipe("normalizer", config={"language": language, **normalizer_config})
     if ngram_config["ngram_window"] != 0:
         nlp.add_pipe("ngram_generator", config=ngram_config)
-    print(ngram_config, nlp.pipe_names)
     return nlp
 
 
 if __name__ == "__main__":
     nlp = load_language_model(
         "french",
+        {
+            "language": "french",
+            "modernize": Modernizer("french"),
+            "strip_tags": False,
+            "token_regex": re.compile(rf"(\w+)|([^\w+])"),
+        },
         {
             "convert_entities": True,
             "lowercase": True,
@@ -361,7 +373,9 @@ def load_language_model(
             "min_word_length": 1,
             "stopwords": None,
         },
-        filter_config={"pos_to_keep": ["NOUN", "ADJ"], "ents_to_keep": ["PER", "LOC"]},
+        {"pos_to_keep": ["NOUN", "ADJ"], "ents_to_keep": ["PER", "LOC"]},
+        {"ngram_window": 0, "ngram_word_order": True},
+        False,
     )
     s = """Comme pour « l’incident » survenu sur l’aérodrome de Saky, Kiev n’a pas revendiqué d’attaque sur Djankoï, un conseiller présidentiel, Mykhaïlo Podoliak, se contentant de confirmer l’explosion. Un responsable ukrainien a cependant affirmé au New York Times, sous couvert d’anonymat, qu’une unité militaire d’élite ukrainienne opérant derrière les lignes ennemies était à l’origine de l’attaque. Les responsables ukrainiens ont aussi prévenu mardi que la Crimée ne serait pas épargnée par les ravages de la guerre."""
     doc = nlp(s)