hojichar.filters.language_identification
1import hashlib 2import logging 3import os 4import time 5from os import PathLike 6from pathlib import Path 7from typing import Any 8 9try: 10 import requests 11 from fasttext import load_model # type: ignore 12 13 is_loaded_extras = True 14except ImportError: 15 is_loaded_extras = False 16 17from tqdm import tqdm 18 19from hojichar import Document, Filter 20 21logger = logging.getLogger(__name__) 22 23 24FASTTEXT_MODEL_URL = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin" 25MODEL_CHECKSUM = "01810bc59c6a3d2b79c79e6336612f65" 26 27 28def _download_with_progress_bar( 29 download_url: str, save_path: str | PathLike[str], retries: int = 3, delay: float = 1.0 30) -> None: 31 try: 32 Path(save_path).parent.mkdir(parents=True, exist_ok=True) 33 with requests.get(download_url, stream=True) as r: 34 r.raise_for_status() 35 total_size = int(r.headers.get("content-length", 0)) 36 with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: 37 with open(save_path, "wb") as f: 38 for chunk in r.iter_content(chunk_size=8192): 39 f.write(chunk) 40 pbar.update(len(chunk)) 41 except requests.RequestException as e: 42 if retries > 0: 43 logger.warning( 44 f"Download failed, retrying in {delay} seconds... ({retries} retries left)" 45 ) 46 time.sleep(delay) 47 _download_with_progress_bar(download_url, save_path, retries - 1, delay) 48 else: 49 logger.error(f"Download failed after retries: {e}") 50 raise 51 52 53def _get_md5_hash_of_file(file_path: str | PathLike[str]) -> str: 54 """ 55 Function to calculate the MD5 hash of a file. 56 57 Read the file in chunks to avoid loading large files into memory. 58 cf. https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file 59 """ 60 md5_hash = hashlib.md5() 61 with open(file_path, "rb") as file: 62 while chunk := file.read(8192): 63 md5_hash.update(chunk) 64 return md5_hash.hexdigest() 65 66 67def _download_fasttext_model(model_path: str | PathLike[str]) -> None: 68 logger.info(f"Downloading fasttext model from {FASTTEXT_MODEL_URL} to {model_path}...") 69 _download_with_progress_bar(FASTTEXT_MODEL_URL, model_path) 70 71 72class LanguageIdentificationByFastText(Filter): 73 """ 74 A filter that removes non-Japanese text 75 using Language IDentification (LID). 76 Download the fastText model for LID: https://fasttext.cc/docs/en/language-identification.html 77 The available languages are: 78 af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce ceb 79 ckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom 80 gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv 81 kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn 82 nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah 83 sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec 84 vep vi vls vo wa war wuu xal xmf yi yo yue zh 85 """ 86 87 def __init__( 88 self, 89 language: str, 90 lang_score_threshold: float = 0.50, 91 model_path: str | PathLike[str] | None = None, 92 *args: Any, 93 **kwargs: Any, 94 ) -> None: 95 """ 96 Args: 97 language: the language to be accepted. 98 lang_score_threshold: the document whose fasttext score is 99 below this threshold will be discarded. 100 A default value is 0.50, which is empirically very generous value, 101 i.e., almost all documents are accepted if the document 102 is classified as japanese text. 103 model_path: The directory path, which the model saved in. 104 If None, the model will be saved in the current directory. 105 *args: 106 **kwargs: 107 """ 108 super().__init__(*args, **kwargs) 109 if not is_loaded_extras: 110 raise ImportError( 111 "The `fasttext` package is required to use this filter. " 112 "Please install it by running `pip install hojichar[all]`" 113 "or `pip install fasttext requests`." 114 ) 115 116 self.lang_score_threshold = lang_score_threshold 117 self.language = language 118 119 self.model_path = Path(model_path) if model_path else Path(os.getcwd()) / "lid.176.bin" 120 121 if not self.model_path.exists(): 122 logger.info("Fasttext model file was not found.") 123 _download_fasttext_model(self.model_path) 124 125 assert _get_md5_hash_of_file(self.model_path) == MODEL_CHECKSUM, ( 126 f"Checksum of the downloaded model file does not match the expected value. " 127 f"Expected: {MODEL_CHECKSUM}, " 128 f"Actual: {_get_md5_hash_of_file(self.model_path)}" 129 ) 130 self.model = load_model(str(self.model_path)) 131 132 def _predict_language(self, text: str) -> tuple[str, float]: 133 # fasttext cannot handle multiline input 134 # so we must remove the newline character 135 text = text.strip().replace("\n", " ") 136 # The single-string path in fasttext 0.9.3 uses 137 # np.array(..., copy=False), which is incompatible with NumPy 2. 138 # The documented batch path avoids that conversion. 139 labels, scores = self.model.predict([text]) 140 pred_lang = labels[0][0].replace("__label__", "") 141 pred_score = scores[0][0] 142 return pred_lang, pred_score 143 144 def apply(self, doc: Document) -> Document: 145 pred_lang, score = self._predict_language(doc.text) 146 if not (pred_lang == self.language and score >= self.lang_score_threshold): 147 doc.is_rejected = True 148 return doc 149 150 151class AcceptJapaneseByFastText(LanguageIdentificationByFastText): 152 """ 153 A filter that removes non-Japanese text via Language Identification (LID) by FastText. 154 155 >>> AcceptJapaneseByFastText().apply(Document("This is English document")).is_rejected 156 True 157 >>> AcceptJapaneseByFastText().apply(Document("自然言語処理大好き!")).is_rejected 158 False 159 >>> AcceptJapaneseByFastText().apply(Document("快三手机投注平台代理")).is_rejected 160 True 161 """ 162 163 def __init__( 164 self, 165 lang_score_threshold: float = 0.50, 166 model_path: str | PathLike[str] | None = None, 167 *args: Any, 168 **kwargs: Any, 169 ) -> None: 170 super().__init__("ja", lang_score_threshold, model_path, *args, **kwargs)
73class LanguageIdentificationByFastText(Filter): 74 """ 75 A filter that removes non-Japanese text 76 using Language IDentification (LID). 77 Download the fastText model for LID: https://fasttext.cc/docs/en/language-identification.html 78 The available languages are: 79 af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce ceb 80 ckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom 81 gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv 82 kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn 83 nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah 84 sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec 85 vep vi vls vo wa war wuu xal xmf yi yo yue zh 86 """ 87 88 def __init__( 89 self, 90 language: str, 91 lang_score_threshold: float = 0.50, 92 model_path: str | PathLike[str] | None = None, 93 *args: Any, 94 **kwargs: Any, 95 ) -> None: 96 """ 97 Args: 98 language: the language to be accepted. 99 lang_score_threshold: the document whose fasttext score is 100 below this threshold will be discarded. 101 A default value is 0.50, which is empirically very generous value, 102 i.e., almost all documents are accepted if the document 103 is classified as japanese text. 104 model_path: The directory path, which the model saved in. 105 If None, the model will be saved in the current directory. 106 *args: 107 **kwargs: 108 """ 109 super().__init__(*args, **kwargs) 110 if not is_loaded_extras: 111 raise ImportError( 112 "The `fasttext` package is required to use this filter. " 113 "Please install it by running `pip install hojichar[all]`" 114 "or `pip install fasttext requests`." 115 ) 116 117 self.lang_score_threshold = lang_score_threshold 118 self.language = language 119 120 self.model_path = Path(model_path) if model_path else Path(os.getcwd()) / "lid.176.bin" 121 122 if not self.model_path.exists(): 123 logger.info("Fasttext model file was not found.") 124 _download_fasttext_model(self.model_path) 125 126 assert _get_md5_hash_of_file(self.model_path) == MODEL_CHECKSUM, ( 127 f"Checksum of the downloaded model file does not match the expected value. " 128 f"Expected: {MODEL_CHECKSUM}, " 129 f"Actual: {_get_md5_hash_of_file(self.model_path)}" 130 ) 131 self.model = load_model(str(self.model_path)) 132 133 def _predict_language(self, text: str) -> tuple[str, float]: 134 # fasttext cannot handle multiline input 135 # so we must remove the newline character 136 text = text.strip().replace("\n", " ") 137 # The single-string path in fasttext 0.9.3 uses 138 # np.array(..., copy=False), which is incompatible with NumPy 2. 139 # The documented batch path avoids that conversion. 140 labels, scores = self.model.predict([text]) 141 pred_lang = labels[0][0].replace("__label__", "") 142 pred_score = scores[0][0] 143 return pred_lang, pred_score 144 145 def apply(self, doc: Document) -> Document: 146 pred_lang, score = self._predict_language(doc.text) 147 if not (pred_lang == self.language and score >= self.lang_score_threshold): 148 doc.is_rejected = True 149 return doc
A filter that removes non-Japanese text using Language IDentification (LID). Download the fastText model for LID: https://fasttext.cc/docs/en/language-identification.html The available languages are: af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce ceb ckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec vep vi vls vo wa war wuu xal xmf yi yo yue zh
88 def __init__( 89 self, 90 language: str, 91 lang_score_threshold: float = 0.50, 92 model_path: str | PathLike[str] | None = None, 93 *args: Any, 94 **kwargs: Any, 95 ) -> None: 96 """ 97 Args: 98 language: the language to be accepted. 99 lang_score_threshold: the document whose fasttext score is 100 below this threshold will be discarded. 101 A default value is 0.50, which is empirically very generous value, 102 i.e., almost all documents are accepted if the document 103 is classified as japanese text. 104 model_path: The directory path, which the model saved in. 105 If None, the model will be saved in the current directory. 106 *args: 107 **kwargs: 108 """ 109 super().__init__(*args, **kwargs) 110 if not is_loaded_extras: 111 raise ImportError( 112 "The `fasttext` package is required to use this filter. " 113 "Please install it by running `pip install hojichar[all]`" 114 "or `pip install fasttext requests`." 115 ) 116 117 self.lang_score_threshold = lang_score_threshold 118 self.language = language 119 120 self.model_path = Path(model_path) if model_path else Path(os.getcwd()) / "lid.176.bin" 121 122 if not self.model_path.exists(): 123 logger.info("Fasttext model file was not found.") 124 _download_fasttext_model(self.model_path) 125 126 assert _get_md5_hash_of_file(self.model_path) == MODEL_CHECKSUM, ( 127 f"Checksum of the downloaded model file does not match the expected value. " 128 f"Expected: {MODEL_CHECKSUM}, " 129 f"Actual: {_get_md5_hash_of_file(self.model_path)}" 130 ) 131 self.model = load_model(str(self.model_path))
Args: language: the language to be accepted. lang_score_threshold: the document whose fasttext score is below this threshold will be discarded. A default value is 0.50, which is empirically very generous value, i.e., almost all documents are accepted if the document is classified as japanese text. model_path: The directory path, which the model saved in. If None, the model will be saved in the current directory. args: *kwargs:
145 def apply(self, doc: Document) -> Document: 146 pred_lang, score = self._predict_language(doc.text) 147 if not (pred_lang == self.language and score >= self.lang_score_threshold): 148 doc.is_rejected = True 149 return doc
Definition of filter behavior.
The document must have a protocol TextContent,
and mostly used hojichar.Document class.
In this method, the filter will modify document.text or
document.extras and set document.is_rejected = True to discard the document.
Parameters
document : Document Input document
Returns
Document Processed Document
152class AcceptJapaneseByFastText(LanguageIdentificationByFastText): 153 """ 154 A filter that removes non-Japanese text via Language Identification (LID) by FastText. 155 156 >>> AcceptJapaneseByFastText().apply(Document("This is English document")).is_rejected 157 True 158 >>> AcceptJapaneseByFastText().apply(Document("自然言語処理大好き!")).is_rejected 159 False 160 >>> AcceptJapaneseByFastText().apply(Document("快三手机投注平台代理")).is_rejected 161 True 162 """ 163 164 def __init__( 165 self, 166 lang_score_threshold: float = 0.50, 167 model_path: str | PathLike[str] | None = None, 168 *args: Any, 169 **kwargs: Any, 170 ) -> None: 171 super().__init__("ja", lang_score_threshold, model_path, *args, **kwargs)
A filter that removes non-Japanese text via Language Identification (LID) by FastText.
>>> AcceptJapaneseByFastText().apply(Document("This is English document")).is_rejected
True
>>> AcceptJapaneseByFastText().apply(Document("自然言語処理大好き!")).is_rejected
False
>>> AcceptJapaneseByFastText().apply(Document("快三手机投注平台代理")).is_rejected
True
164 def __init__( 165 self, 166 lang_score_threshold: float = 0.50, 167 model_path: str | PathLike[str] | None = None, 168 *args: Any, 169 **kwargs: Any, 170 ) -> None: 171 super().__init__("ja", lang_score_threshold, model_path, *args, **kwargs)
Args: language: the language to be accepted. lang_score_threshold: the document whose fasttext score is below this threshold will be discarded. A default value is 0.50, which is empirically very generous value, i.e., almost all documents are accepted if the document is classified as japanese text. model_path: The directory path, which the model saved in. If None, the model will be saved in the current directory. args: *kwargs: