hojichar.filters.language_identification

  1import hashlib
  2import logging
  3import os
  4import time
  5from os import PathLike
  6from pathlib import Path
  7from typing import Any
  8
  9try:
 10    import requests
 11    from fasttext import load_model  # type: ignore
 12
 13    is_loaded_extras = True
 14except ImportError:
 15    is_loaded_extras = False
 16
 17from tqdm import tqdm
 18
 19from hojichar import Document, Filter
 20
 21logger = logging.getLogger(__name__)
 22
 23
 24FASTTEXT_MODEL_URL = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"
 25MODEL_CHECKSUM = "01810bc59c6a3d2b79c79e6336612f65"
 26
 27
 28def _download_with_progress_bar(
 29    download_url: str, save_path: str | PathLike[str], retries: int = 3, delay: float = 1.0
 30) -> None:
 31    try:
 32        Path(save_path).parent.mkdir(parents=True, exist_ok=True)
 33        with requests.get(download_url, stream=True) as r:
 34            r.raise_for_status()
 35            total_size = int(r.headers.get("content-length", 0))
 36            with tqdm(total=total_size, unit="B", unit_scale=True) as pbar:
 37                with open(save_path, "wb") as f:
 38                    for chunk in r.iter_content(chunk_size=8192):
 39                        f.write(chunk)
 40                        pbar.update(len(chunk))
 41    except requests.RequestException as e:
 42        if retries > 0:
 43            logger.warning(
 44                f"Download failed, retrying in {delay} seconds... ({retries} retries left)"
 45            )
 46            time.sleep(delay)
 47            _download_with_progress_bar(download_url, save_path, retries - 1, delay)
 48        else:
 49            logger.error(f"Download failed after retries: {e}")
 50            raise
 51
 52
 53def _get_md5_hash_of_file(file_path: str | PathLike[str]) -> str:
 54    """
 55    Function to calculate the MD5 hash of a file.
 56
 57    Read the file in chunks to avoid loading large files into memory.
 58    cf. https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file
 59    """
 60    md5_hash = hashlib.md5()
 61    with open(file_path, "rb") as file:
 62        while chunk := file.read(8192):
 63            md5_hash.update(chunk)
 64    return md5_hash.hexdigest()
 65
 66
 67def _download_fasttext_model(model_path: str | PathLike[str]) -> None:
 68    logger.info(f"Downloading fasttext model from {FASTTEXT_MODEL_URL} to {model_path}...")
 69    _download_with_progress_bar(FASTTEXT_MODEL_URL, model_path)
 70
 71
 72class LanguageIdentificationByFastText(Filter):
 73    """
 74    A filter that removes non-Japanese text
 75    using Language IDentification (LID).
 76    Download the fastText model for LID: https://fasttext.cc/docs/en/language-identification.html
 77    The available languages are:
 78        af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce ceb
 79        ckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom
 80        gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv
 81        kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn
 82        nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah
 83        sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec
 84        vep vi vls vo wa war wuu xal xmf yi yo yue zh
 85    """
 86
 87    def __init__(
 88        self,
 89        language: str,
 90        lang_score_threshold: float = 0.50,
 91        model_path: str | PathLike[str] | None = None,
 92        *args: Any,
 93        **kwargs: Any,
 94    ) -> None:
 95        """
 96        Args:
 97            language: the language to be accepted.
 98            lang_score_threshold: the document whose fasttext score is
 99              below this threshold will be discarded.
100              A default value is 0.50, which is empirically very generous value,
101              i.e., almost all documents are accepted if the document
102              is classified as japanese text.
103            model_path: The directory path, which the model saved in.
104                If None, the model will be saved in the current directory.
105            *args:
106            **kwargs:
107        """
108        super().__init__(*args, **kwargs)
109        if not is_loaded_extras:
110            raise ImportError(
111                "The `fasttext` package is required to use this filter. "
112                "Please install it by running `pip install hojichar[all]`"
113                "or `pip install fasttext requests`."
114            )
115
116        self.lang_score_threshold = lang_score_threshold
117        self.language = language
118
119        self.model_path = Path(model_path) if model_path else Path(os.getcwd()) / "lid.176.bin"
120
121        if not self.model_path.exists():
122            logger.info("Fasttext model file was not found.")
123            _download_fasttext_model(self.model_path)
124
125        assert _get_md5_hash_of_file(self.model_path) == MODEL_CHECKSUM, (
126            f"Checksum of the downloaded model file does not match the expected value. "
127            f"Expected: {MODEL_CHECKSUM}, "
128            f"Actual: {_get_md5_hash_of_file(self.model_path)}"
129        )
130        self.model = load_model(str(self.model_path))
131
132    def _predict_language(self, text: str) -> tuple[str, float]:
133        # fasttext cannot handle multiline input
134        # so we must remove the newline character
135        text = text.strip().replace("\n", " ")
136        # The single-string path in fasttext 0.9.3 uses
137        # np.array(..., copy=False), which is incompatible with NumPy 2.
138        # The documented batch path avoids that conversion.
139        labels, scores = self.model.predict([text])
140        pred_lang = labels[0][0].replace("__label__", "")
141        pred_score = scores[0][0]
142        return pred_lang, pred_score
143
144    def apply(self, doc: Document) -> Document:
145        pred_lang, score = self._predict_language(doc.text)
146        if not (pred_lang == self.language and score >= self.lang_score_threshold):
147            doc.is_rejected = True
148        return doc
149
150
151class AcceptJapaneseByFastText(LanguageIdentificationByFastText):
152    """
153    A filter that removes non-Japanese text via Language Identification (LID) by FastText.
154
155    >>> AcceptJapaneseByFastText().apply(Document("This is English document")).is_rejected
156    True
157    >>> AcceptJapaneseByFastText().apply(Document("自然言語処理大好き!")).is_rejected
158    False
159    >>> AcceptJapaneseByFastText().apply(Document("快三手机投注平台代理")).is_rejected
160    True
161    """
162
163    def __init__(
164        self,
165        lang_score_threshold: float = 0.50,
166        model_path: str | PathLike[str] | None = None,
167        *args: Any,
168        **kwargs: Any,
169    ) -> None:
170        super().__init__("ja", lang_score_threshold, model_path, *args, **kwargs)
class LanguageIdentificationByFastText(hojichar.core.filter_interface.Filter):
 73class LanguageIdentificationByFastText(Filter):
 74    """
 75    A filter that removes non-Japanese text
 76    using Language IDentification (LID).
 77    Download the fastText model for LID: https://fasttext.cc/docs/en/language-identification.html
 78    The available languages are:
 79        af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce ceb
 80        ckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom
 81        gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv
 82        kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn
 83        nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah
 84        sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec
 85        vep vi vls vo wa war wuu xal xmf yi yo yue zh
 86    """
 87
 88    def __init__(
 89        self,
 90        language: str,
 91        lang_score_threshold: float = 0.50,
 92        model_path: str | PathLike[str] | None = None,
 93        *args: Any,
 94        **kwargs: Any,
 95    ) -> None:
 96        """
 97        Args:
 98            language: the language to be accepted.
 99            lang_score_threshold: the document whose fasttext score is
100              below this threshold will be discarded.
101              A default value is 0.50, which is empirically very generous value,
102              i.e., almost all documents are accepted if the document
103              is classified as japanese text.
104            model_path: The directory path, which the model saved in.
105                If None, the model will be saved in the current directory.
106            *args:
107            **kwargs:
108        """
109        super().__init__(*args, **kwargs)
110        if not is_loaded_extras:
111            raise ImportError(
112                "The `fasttext` package is required to use this filter. "
113                "Please install it by running `pip install hojichar[all]`"
114                "or `pip install fasttext requests`."
115            )
116
117        self.lang_score_threshold = lang_score_threshold
118        self.language = language
119
120        self.model_path = Path(model_path) if model_path else Path(os.getcwd()) / "lid.176.bin"
121
122        if not self.model_path.exists():
123            logger.info("Fasttext model file was not found.")
124            _download_fasttext_model(self.model_path)
125
126        assert _get_md5_hash_of_file(self.model_path) == MODEL_CHECKSUM, (
127            f"Checksum of the downloaded model file does not match the expected value. "
128            f"Expected: {MODEL_CHECKSUM}, "
129            f"Actual: {_get_md5_hash_of_file(self.model_path)}"
130        )
131        self.model = load_model(str(self.model_path))
132
133    def _predict_language(self, text: str) -> tuple[str, float]:
134        # fasttext cannot handle multiline input
135        # so we must remove the newline character
136        text = text.strip().replace("\n", " ")
137        # The single-string path in fasttext 0.9.3 uses
138        # np.array(..., copy=False), which is incompatible with NumPy 2.
139        # The documented batch path avoids that conversion.
140        labels, scores = self.model.predict([text])
141        pred_lang = labels[0][0].replace("__label__", "")
142        pred_score = scores[0][0]
143        return pred_lang, pred_score
144
145    def apply(self, doc: Document) -> Document:
146        pred_lang, score = self._predict_language(doc.text)
147        if not (pred_lang == self.language and score >= self.lang_score_threshold):
148            doc.is_rejected = True
149        return doc

A filter that removes non-Japanese text using Language IDentification (LID). Download the fastText model for LID: https://fasttext.cc/docs/en/language-identification.html The available languages are: af als am an ar arz as ast av az azb ba bar bcl be bg bh bn bo bpy br bs bxr ca cbk ce ceb ckb co cs cv cy da de diq dsb dty dv el eml en eo es et eu fa fi fr frr fy ga gd gl gn gom gu gv he hi hif hr hsb ht hu hy ia id ie ilo io is it ja jbo jv ka kk km kn ko krc ku kv kw ky la lb lez li lmo lo lrc lt lv mai mg mhr min mk ml mn mr mrj ms mt mwl my myv mzn nah nap nds ne new nl nn no oc or os pa pam pfl pl pms pnb ps pt qu rm ro ru rue sa sah sc scn sco sd sh si sk sl so sq sr su sv sw ta te tg th tk tl tr tt tyv ug uk ur uz vec vep vi vls vo wa war wuu xal xmf yi yo yue zh

LanguageIdentificationByFastText( language: str, lang_score_threshold: float = 0.5, model_path: str | os.PathLike[str] | None = None, *args: Any, **kwargs: Any)
 88    def __init__(
 89        self,
 90        language: str,
 91        lang_score_threshold: float = 0.50,
 92        model_path: str | PathLike[str] | None = None,
 93        *args: Any,
 94        **kwargs: Any,
 95    ) -> None:
 96        """
 97        Args:
 98            language: the language to be accepted.
 99            lang_score_threshold: the document whose fasttext score is
100              below this threshold will be discarded.
101              A default value is 0.50, which is empirically very generous value,
102              i.e., almost all documents are accepted if the document
103              is classified as japanese text.
104            model_path: The directory path, which the model saved in.
105                If None, the model will be saved in the current directory.
106            *args:
107            **kwargs:
108        """
109        super().__init__(*args, **kwargs)
110        if not is_loaded_extras:
111            raise ImportError(
112                "The `fasttext` package is required to use this filter. "
113                "Please install it by running `pip install hojichar[all]`"
114                "or `pip install fasttext requests`."
115            )
116
117        self.lang_score_threshold = lang_score_threshold
118        self.language = language
119
120        self.model_path = Path(model_path) if model_path else Path(os.getcwd()) / "lid.176.bin"
121
122        if not self.model_path.exists():
123            logger.info("Fasttext model file was not found.")
124            _download_fasttext_model(self.model_path)
125
126        assert _get_md5_hash_of_file(self.model_path) == MODEL_CHECKSUM, (
127            f"Checksum of the downloaded model file does not match the expected value. "
128            f"Expected: {MODEL_CHECKSUM}, "
129            f"Actual: {_get_md5_hash_of_file(self.model_path)}"
130        )
131        self.model = load_model(str(self.model_path))

Args: language: the language to be accepted. lang_score_threshold: the document whose fasttext score is below this threshold will be discarded. A default value is 0.50, which is empirically very generous value, i.e., almost all documents are accepted if the document is classified as japanese text. model_path: The directory path, which the model saved in. If None, the model will be saved in the current directory. args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
145    def apply(self, doc: Document) -> Document:
146        pred_lang, score = self._predict_language(doc.text)
147        if not (pred_lang == self.language and score >= self.lang_score_threshold):
148            doc.is_rejected = True
149        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class AcceptJapaneseByFastText(LanguageIdentificationByFastText):
152class AcceptJapaneseByFastText(LanguageIdentificationByFastText):
153    """
154    A filter that removes non-Japanese text via Language Identification (LID) by FastText.
155
156    >>> AcceptJapaneseByFastText().apply(Document("This is English document")).is_rejected
157    True
158    >>> AcceptJapaneseByFastText().apply(Document("自然言語処理大好き!")).is_rejected
159    False
160    >>> AcceptJapaneseByFastText().apply(Document("快三手机投注平台代理")).is_rejected
161    True
162    """
163
164    def __init__(
165        self,
166        lang_score_threshold: float = 0.50,
167        model_path: str | PathLike[str] | None = None,
168        *args: Any,
169        **kwargs: Any,
170    ) -> None:
171        super().__init__("ja", lang_score_threshold, model_path, *args, **kwargs)

A filter that removes non-Japanese text via Language Identification (LID) by FastText.

>>> AcceptJapaneseByFastText().apply(Document("This is English document")).is_rejected
True
>>> AcceptJapaneseByFastText().apply(Document("自然言語処理大好き!")).is_rejected
False
>>> AcceptJapaneseByFastText().apply(Document("快三手机投注平台代理")).is_rejected
True
AcceptJapaneseByFastText( lang_score_threshold: float = 0.5, model_path: str | os.PathLike[str] | None = None, *args: Any, **kwargs: Any)
164    def __init__(
165        self,
166        lang_score_threshold: float = 0.50,
167        model_path: str | PathLike[str] | None = None,
168        *args: Any,
169        **kwargs: Any,
170    ) -> None:
171        super().__init__("ja", lang_score_threshold, model_path, *args, **kwargs)

Args: language: the language to be accepted. lang_score_threshold: the document whose fasttext score is below this threshold will be discarded. A default value is 0.50, which is empirically very generous value, i.e., almost all documents are accepted if the document is classified as japanese text. model_path: The directory path, which the model saved in. If None, the model will be saved in the current directory. args: *kwargs: