hojichar.filters.document_filters

   1import json
   2import logging
   3import pathlib
   4import re
   5import string
   6import time
   7import unicodedata
   8from collections import Counter
   9from collections.abc import Iterable
  10from itertools import groupby
  11from os import PathLike
  12from typing import Any
  13
  14import numpy as np
  15
  16import hojichar
  17from hojichar.core.filter_interface import Filter
  18from hojichar.core.models import Document, Token
  19
  20try:
  21    import emoji
  22    from fugashi import Tagger  # type: ignore
  23
  24    is_loaded_extras = True
  25except ImportError:
  26    is_loaded_extras = False
  27
  28BASE_PATH = pathlib.Path(hojichar.__path__[0])
  29logger = logging.getLogger(__name__)
  30
  31
  32def _find_excessive_match(
  33    pattern: re.Pattern[str],
  34    text: str,
  35    max_allowed_num: int,
  36    min_matches_per_1k_chars: float,
  37) -> re.Match[str] | None:
  38    """
  39    Return the first match of `pattern` in `text` when the matches are more than
  40    `max_allowed_num` and at least `min_matches_per_1k_chars` per 1,000 characters,
  41    otherwise None. The scan stops as soon as both conditions hold.
  42    """
  43    min_matches = min_matches_per_1k_chars * len(text) / 1000
  44    first_match = None
  45    for num_matches, match in enumerate(pattern.finditer(text), start=1):
  46        if first_match is None:
  47            first_match = match
  48        if num_matches > max_allowed_num and num_matches >= min_matches:
  49            return first_match
  50    return None
  51
  52
  53class ExampleHojiChar(Filter):
  54    """基本的なフィルタの実装例です. 末尾に'<hojichar>'を追加します."""
  55
  56    def apply(self, document: Document) -> Document:
  57        """
  58        >>> ExampleHojiChar()("hello, world")
  59        'hello, world<hojichar>'
  60        """
  61        document.text += "<hojichar>"
  62        return document
  63
  64
  65class ExampleDiscardDocumentContainKeyword(Filter):
  66    """特定のキーワードを持つドキュメントを破棄するようなフィルタの実装例です."""
  67
  68    def __init__(self, keyword: str, *args: Any, **kwargs: Any) -> None:
  69        super().__init__(*args, **kwargs)
  70        self.keyword = keyword
  71
  72    def apply(self, document: Document) -> Document:
  73        """
  74        >>> ExampleDiscardDocumentContainKeyword("バカ").apply(Document("あいつはバカだ")).is_rejected
  75        True
  76        """
  77        if self.keyword in document.text:
  78            document.is_rejected = True
  79        return document
  80
  81
  82class Identity(Filter):
  83    """何も変化を加えないフィルタです. テスト・デバッグに用いられます."""
  84
  85    def apply(self, document: Document) -> Document:
  86        return document
  87
  88
  89class DiscardAll(Filter):
  90    """
  91    すべてのドキュメントを破棄するフィルタです.
  92    テスト・デバッグに用いられます.
  93    """
  94
  95    def apply(self, document: Document) -> Document:
  96        document.is_rejected = True
  97        return document
  98
  99
 100class ApplyDiscard(Filter):
 101    """
 102    上流フィルタで破棄された`Document`を空文字列にします.
 103
 104    `Document.is_rejected=True` の ドキュメントは無視されるため,
 105    このフィルタを `Compose` のコンストラクタに渡しても動作しません.
 106    このフィルタは主に`Compose` 内部や, `discard_filtered=False` を指定
 107    したデバッグ時などに利用されます.
 108    """
 109
 110    def __init__(self, *args: Any, **kwargs: Any) -> None:
 111        super().__init__(*args, **kwargs)
 112
 113    def apply(self, document: Document) -> Document:
 114        """
 115        >>> ApplyDiscard().apply(Document(text="hello", is_rejected=True)).text
 116        ''
 117        """
 118        if document.is_rejected:
 119            document.text = ""
 120
 121        return document
 122
 123
 124class Sleep(Filter):
 125    """
 126    デバッグ用のフィルタです. 指定秒スリープします.
 127    """
 128
 129    def __init__(self, time: float = 1.0, *args: Any, **kwargs: Any) -> None:
 130        super().__init__(*args, **kwargs)
 131        self.time = time
 132
 133    def apply(self, document: Document) -> Document:
 134        """
 135        >>> Sleep(0.1)('hello')  # After 0.1 seconds,
 136        'hello'
 137        """
 138        time.sleep(self.time)
 139        return document
 140
 141
 142class DocumentNormalizer(Filter):
 143    """
 144    Unicode の正規化をします.
 145    """
 146
 147    def __init__(self, *args: Any, **kwargs: Any) -> None:
 148        super().__init__(*args, **kwargs)
 149
 150    def apply(self, document: Document) -> Document:
 151        document.text = unicodedata.normalize("NFKC", document.text)
 152        return document
 153
 154
 155class JSONLoader(Filter):
 156    """
 157    テキストを Json として解釈し, `key` で指定した要素を文字列として
 158    doument に格納します.デフォルトの `key` は 'text' です.
 159
 160    Json の読み込み, あるいは `key` の読み込みに失敗した際には例外を送出します.
 161    これらを無視する場合は, `ignore=True` にします. その際, 読み込みに失敗
 162    したドキュメントは破棄されます.
 163
 164    入力 Json に `extras` キー(辞書形式)が含まれている場合, Document.extras に自動的にマージされます。
 165    さらに `extra_keys` でフィールドを指定すると, それらの値も Document.extras に追記され, 既存の extras
 166    を上書きせずに統合できます。
 167    """
 168
 169    def __init__(
 170        self,
 171        key: str = "text",
 172        ignore: bool = False,
 173        extra_keys: list[str] | None = None,
 174        *args: Any,
 175        **kwargs: Any,
 176    ) -> None:
 177        super().__init__(*args, **kwargs)
 178        self.key = key
 179        self.ignore = ignore
 180        self.extra_keys = extra_keys
 181
 182    def apply(self, document: Document) -> Document:
 183        """
 184        >>> JSONLoader()( '{"text": "hello, world", "words": 2}' )
 185        'hello, world'
 186
 187        >>> JSONLoader()( '{"text": hello, world ....' ) # Broken JSON
 188        Traceback (most recent call last):
 189            ...
 190        json.decoder.JSONDecodeError: Expecting value: line 1 column 10 (char 9)
 191
 192        >>> JSONLoader()( '{"words": 2}' )
 193        Traceback (most recent call last):
 194            ...
 195        KeyError: 'text'
 196
 197        >>> JSONLoader(ignore=True).apply(Document('{"text": hello, world ....' )).is_rejected
 198        True
 199        """
 200        try:
 201            data = json.loads(document.text)
 202            document.text = str(data[self.key])
 203            if "extras" in data and isinstance(data["extras"], dict):
 204                document.extras.update(data["extras"])
 205            if self.extra_keys is not None:
 206                for key in self.extra_keys:
 207                    if key not in data:
 208                        continue
 209                    if key == "extras" and isinstance(data[key], dict):
 210                        document.extras.update(data[key])
 211                    else:
 212                        document.extras[key] = data[key]
 213        except Exception as e:
 214            logger.error(f"Failed to parsing in JSONLoader. Input document: \n{document.text}")
 215            if self.ignore:
 216                document.is_rejected = True
 217                return document
 218            else:
 219                raise e
 220
 221        return document
 222
 223
 224class JSONDumper(Filter):
 225    """
 226    Document.text の文字列を json に変換します.
 227    必要に応じ Document のメタデータを付与します. これはドキュメントの破棄事由が含まれ、偽陽性の分析に有効です。
 228    デフォルトで `skip_rejected` が `False` にセットされており、Document の破棄フラグにかかわらず
 229    処理されます。
 230    """
 231
 232    def __init__(
 233        self,
 234        dump_reason: bool = False,
 235        p: float = 1,
 236        skip_rejected: bool = False,
 237        export_extras: bool = False,
 238        *args: Any,
 239        **kwargs: Any,
 240    ) -> None:
 241        """
 242        Args:
 243            dump_reason (bool, optional): `is_rejected`, `reason` エントリをダンプします. Defaults to False.
 244            p (float, optional): Apply probability. Defaults to 1.
 245            skip_rejected (bool, optional): 破棄済みサンプルを排除しません.
 246        """
 247        super().__init__(p, skip_rejected, *args, **kwargs)
 248        self.dump_reason = dump_reason
 249        self.export_extras = export_extras
 250
 251    def apply(self, document: Document) -> Document:
 252        """
 253        >>> JSONDumper()("hojichar")
 254        '{"text": "hojichar"}'
 255        """
 256        text = document.text
 257        if self.dump_reason:
 258            if self.export_extras:
 259                output_extras = dict(document.extras)
 260                document.text = json.dumps(
 261                    {
 262                        "text": text,
 263                        "is_rejected": document.is_rejected,
 264                        "reason": document.reject_reason,
 265                        "extras": output_extras,
 266                    },
 267                    ensure_ascii=False,
 268                )
 269            else:
 270                document.text = json.dumps(
 271                    {
 272                        "text": text,
 273                        "is_rejected": document.is_rejected,
 274                        "reason": document.reject_reason,
 275                    },
 276                    ensure_ascii=False,
 277                )
 278        else:
 279            if self.export_extras:
 280                output_extras = dict(document.extras)
 281                document.text = json.dumps(
 282                    {
 283                        "text": text,
 284                        "extras": output_extras,
 285                    },
 286                    ensure_ascii=False,
 287                )
 288            else:
 289                document.text = json.dumps({"text": text}, ensure_ascii=False)
 290        return document
 291
 292
 293class DocumentLengthFilter(Filter):
 294    """
 295    `min_doc_len`, `max_doc_len` で指定した上限・下限の範囲内にないドキュメントを破棄します.
 296    デフォルトでは 200字 以上 50000字以内のテキストが受理されます.
 297    """
 298
 299    def __init__(
 300        self,
 301        min_doc_len: int | None = None,
 302        max_doc_len: int | None = None,
 303        *args: Any,
 304        **kwargs: Any,
 305    ) -> None:
 306        super().__init__(*args, **kwargs)
 307
 308        self.min_doc_len = min_doc_len
 309        self.max_doc_len = max_doc_len
 310
 311    def apply(self, doc: Document) -> Document:
 312        """
 313        >>> DocumentLengthFilter(min_doc_len=5).apply(Document("1234")).is_rejected
 314        True
 315        """
 316        doc_len = len(doc.text)
 317        if self.min_doc_len is not None:
 318            if doc_len < self.min_doc_len:
 319                doc.is_rejected = True
 320        if self.max_doc_len is not None:
 321            if self.max_doc_len < doc_len:
 322                doc.is_rejected = True
 323        return doc
 324
 325
 326class NgWordsFilterJa(Filter):
 327    """
 328    日本語のNGワード(および不適切語)を含む文書を破棄します.
 329    `dict_path` で指定したファイルから, キーワードのリストを得ます.
 330    ファイルは単語が改行で羅列されたテキストファイルです.
 331
 332    `ignore_confused` を `True` にすると,
 333    偽陽性を軽減するために, カタカナのNGワードは前後にカタカナが無い場合のみNG判定されます.
 334    デフォルト値は `False` です.
 335
 336    デフォルトでは, NGワードに1回でもマッチした文書を破棄します. `max_allowed_num` と
 337    `min_matches_per_1k_chars` を指定すると, マッチ数が `max_allowed_num` 回より多く, かつ
 338    1,000 字あたりのマッチ数が `min_matches_per_1k_chars` 以上の文書だけを破棄します.
 339    NGワードが数回だけ現れる長い文書 (生物学の記事など) を残し, NGワードを並べたページを
 340    破棄する用途を想定しています. この2つの引数は, このクラスを継承する
 341    `DiscardAdultContentJa`, `DiscardDiscriminationContentJa`, `DiscardViolenceContentJa` でも
 342    使えます.
 343    """
 344
 345    def __init__(
 346        self,
 347        dict_path: str | PathLike[str],
 348        ignore_confused: bool = False,
 349        *args: Any,
 350        max_allowed_num: int = 0,
 351        min_matches_per_1k_chars: float = 0.0,
 352        **kwargs: Any,
 353    ) -> None:
 354        """
 355        Args:
 356            dict_path: Path to the NG word list, one word per line.
 357            ignore_confused: Match katakana words only where no katakana precedes or follows.
 358            *args:
 359            max_allowed_num: The document is removed if the words match more than this number
 360                of times. Defaults to 0, which removes the document on the first match.
 361            min_matches_per_1k_chars: The document is removed only if there are also at least
 362                this many matches per 1,000 characters. Defaults to 0.0, which decides by
 363                `max_allowed_num` alone.
 364            **kwargs:
 365        """
 366        super().__init__(*args, **kwargs)
 367        self.max_allowed_num = max_allowed_num
 368        self.min_matches_per_1k_chars = min_matches_per_1k_chars
 369
 370        with open(dict_path, encoding="utf-8") as fp:
 371            ng_words = fp.read().split("\n")
 372        ng_words = [w.strip() for w in ng_words if not len(w) == 0]
 373
 374        if ignore_confused:
 375            words_katakana = []
 376            words_not_katakana = []
 377            for w in ng_words:
 378                if re.fullmatch(r"[ァ-ヴー]+", w):
 379                    words_katakana.append(re.escape(w))
 380                else:
 381                    words_not_katakana.append(re.escape(w))
 382            katakana_pat = "|".join(words_katakana)
 383            katakana_pat = rf"(?<![ァ-ヴー])({katakana_pat})(?![ァ-ヴー])"
 384            pat = "|".join(words_not_katakana) + "|" + katakana_pat
 385            self.keyword_pat = re.compile(pat)
 386        else:
 387            ng_words = [re.escape(w) for w in ng_words]
 388            pat = "|".join(ng_words)
 389            self.keyword_pat = re.compile(pat)
 390
 391    def apply(self, doc: Document) -> Document:
 392        regex_match = _find_excessive_match(
 393            self.keyword_pat, doc.text, self.max_allowed_num, self.min_matches_per_1k_chars
 394        )
 395        if regex_match is not None:
 396            doc.is_rejected = True
 397            self.matched_text = regex_match.group()
 398            self.matched_text_neighbor = doc.text[
 399                regex_match.start() - 20 : regex_match.end() + 20
 400            ]
 401
 402        return doc
 403
 404
 405class NgWordsFilterEn(Filter):
 406    """
 407    英語のNGワード(および不適切語)を含む文書を破棄します.
 408    `dict_path` で指定したファイルから, キーワードのリストを得ます.
 409    ファイルは単語が改行で羅列されたテキストファイルです.
 410    """
 411
 412    def __init__(self, dict_path: str | PathLike[str], *args: Any, **kwargs: Any) -> None:
 413        super().__init__(*args, **kwargs)
 414
 415        with open(dict_path, encoding="utf-8") as fp:
 416            ng_words = fp.read().split("\n")
 417        ng_words = [re.escape(w.strip()) for w in ng_words if not len(w) == 0]
 418        pat = "|".join(ng_words)
 419        # 英語のパターンにマッチするようにしている, \s[単語]\s や [単語]. [単語], などにマッチ.
 420        self.keyword_pat = re.compile(rf"(?:^| )({pat})(?:( |,|\.)|$)", re.IGNORECASE)
 421
 422    def apply(self, doc: Document) -> Document:
 423        if self.keyword_pat.search(doc.text):
 424            doc.is_rejected = True
 425        return doc
 426
 427
 428class DiscardAdultContentJa(NgWordsFilterJa):
 429    """
 430    日本語のアダルトキーワード(および不適切語)を含む文書を破棄します.
 431    `dict_path` で指定したファイルから, キーワードのリストを得ます.
 432    ファイルは単語が改行で羅列されたテキストファイルです.
 433    デフォルトの`dict_path` は /hojichar/dict/adult_keywords_ja.txt です.
 434    """
 435
 436    def __init__(
 437        self,
 438        dict_path: str | PathLike[str] = BASE_PATH / "dict/adult_keywords_ja.txt",
 439        *args: Any,
 440        **kwargs: Any,
 441    ) -> None:
 442        super().__init__(dict_path, *args, **kwargs)
 443
 444    def apply(self, doc: Document) -> Document:
 445        """
 446        >>> DiscardAdultContentJa().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
 447        True
 448
 449        >>> DiscardAdultContentJa().apply(Document("ほうじ茶")).is_rejected
 450        False
 451
 452        挙動は正しいが誤検知しているケース. 他にも, サック in リュックサック,
 453        >>> DiscardAdultContentJa().apply(Document("アスパラガス")).is_rejected \
 454        # Matching with NG keyword "アス"
 455        True
 456        """
 457        return super().apply(doc)
 458
 459
 460class DiscardAdultContentEn(NgWordsFilterEn):
 461    """
 462    英語のアダルトキーワード(および不適切語)を含む文書を破棄します.
 463    `dict_path` で指定したファイルから, キーワードのリストを得ます.
 464    ファイルは単語が改行で羅列されたテキストファイルです.
 465    デフォルトの`dict_path` は /hojichar/dict/adult_keywords_en.txt です.
 466    """
 467
 468    def __init__(
 469        self,
 470        dict_path: str | PathLike[str] = BASE_PATH / "dict/adult_keywords_en.txt",
 471        *args: Any,
 472        **kwargs: Any,
 473    ) -> None:
 474        super().__init__(dict_path, *args, **kwargs)
 475
 476    def apply(self, doc: Document) -> Document:
 477        """
 478        >>> DiscardAdultContentEn().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
 479        True
 480
 481        >>> DiscardAdultContentEn().apply(Document("hojichar")).is_rejected
 482        False
 483        """
 484        return super().apply(doc)
 485
 486
 487class DiscardDiscriminationContentJa(NgWordsFilterJa):
 488    """
 489    日本語の差別キーワード(および不適切語)を含む文書を破棄します.
 490    `dict_path` で指定したファイルから, キーワードのリストを得ます.
 491    ファイルは単語が改行で羅列されたテキストファイルです.
 492    デフォルトの`dict_path` は /hojichar/dict/discrimination_keywords_ja.txt です.
 493    """
 494
 495    def __init__(
 496        self,
 497        dict_path: str | PathLike[str] = BASE_PATH / "dict/discrimination_keywords_ja.txt",
 498        *args: Any,
 499        **kwargs: Any,
 500    ):
 501        super().__init__(dict_path, *args, **kwargs)
 502
 503    def apply(self, doc: Document) -> Document:
 504        """
 505        >>> DiscardDiscriminationContentJa().\
 506            apply(Document("<TEST_STRING_OF_DISCRIMINATION_KEYWORD>")).is_rejected
 507        True
 508
 509        >>> DiscardDiscriminationContentJa().apply(Document("ほうじ茶")).is_rejected
 510        False
 511        """
 512        return super().apply(doc)
 513
 514
 515class DiscardViolenceContentJa(NgWordsFilterJa):
 516    """
 517    日本語の暴力・脅迫を示唆するキーワードを含む文書を破棄します.
 518    `dict_path` で指定したファイルから, キーワードのリストを得ます.
 519    ファイルは単語が改行で羅列されたテキストファイルです.
 520    デフォルトの`dict_path` は /hojichar/dict/violence_keywords_ja.txt です.
 521    """
 522
 523    def __init__(
 524        self,
 525        dict_path: str | PathLike[str] = BASE_PATH / "dict/violence_keywords_ja.txt",
 526        *args: Any,
 527        **kwargs: Any,
 528    ) -> None:
 529        super().__init__(dict_path, *args, **kwargs)
 530
 531    def apply(self, doc: Document) -> Document:
 532        """
 533        >>> DiscardViolenceContentJa()\
 534            .apply(Document("<TEST_STRING_OF_VIOLENCE_KEYWORD>")).is_rejected
 535        True
 536
 537        >>> DiscardViolenceContentJa().apply(Document("ほうじ茶")).is_rejected
 538        False
 539        """
 540        return super().apply(doc)
 541
 542
 543class DiscardBBSComments(Filter):
 544    """
 545    正規表現 "BBS Pattern" に `max_allowed_num` 回よりたくさんマッチする文書を破棄します.
 546    `max_allowed_num` のデフォルト値は14です.
 547
 548    マッチ数は文書が長いほど増えるため, 固定の回数だけで判定すると, 日付や「ページ」「質問」を
 549    含む長い文書 (学校だより・官公庁の PDF など) が長さだけを理由に破棄されます.
 550    `min_matches_per_1k_chars` を指定すると, 1,000 字あたりのマッチ数がこの値以上の場合にだけ
 551    破棄します. デフォルト値は 0.0 で, マッチ数だけで判定します.
 552
 553    正規表現 "BBS Pattern" は下記のリンクで検証可能です. ただし, リンク先は末尾に `|-` を含む
 554    以前のパターンです. この `-` は, Markdown の表の区切り行やハイフンでつないだ語に含まれる
 555    すべてのハイフンを1回のマッチとして数えていたため, 取り除きました.
 556    https://regex101.com/r/ybQvL2/1
 557    """
 558
 559    def __init__(
 560        self,
 561        max_allowed_num: int = 14,
 562        *args: Any,
 563        min_matches_per_1k_chars: float = 0.0,
 564        **kwargs: Any,
 565    ) -> None:
 566        """
 567        Args:
 568            max_allowed_num: The document is removed if the pattern matches more than this
 569                number of times.
 570            *args:
 571            min_matches_per_1k_chars: The document is removed only if there are also at least
 572                this many matches per 1,000 characters. Defaults to 0.0, which decides by
 573                `max_allowed_num` alone.
 574            **kwargs:
 575        """
 576        super().__init__(*args, **kwargs)
 577
 578        self.max_allowed_num = max_allowed_num
 579        self.min_matches_per_1k_chars = min_matches_per_1k_chars
 580        self.keyword_pat = re.compile(
 581            r"\d{4}[年\.\-\/][\ ]*\d{1,2}[月\.\-\/][\ ]*\d{1,2}[日]*|コメント|SOLD OUT|レビュー|投稿|ページ|\([月火水木金土日]\)|質問|\d+話|楽天市場"  # noqa
 582        )
 583
 584    def apply(self, doc: Document) -> Document:
 585        """
 586        >>> DiscardBBSComments().apply(Document("楽天市場 質問 投稿 コメント レビュー "*3)).is_rejected
 587        True
 588
 589        >>> DiscardBBSComments().apply(Document("鏡餅")).is_rejected
 590        False
 591
 592        >>> DiscardBBSComments().apply(Document("| a | b |\\n| --- | --- |\\n" * 5)).is_rejected
 593        False
 594        """
 595        match = _find_excessive_match(
 596            self.keyword_pat, doc.text, self.max_allowed_num, self.min_matches_per_1k_chars
 597        )
 598        if match is not None:
 599            doc.is_rejected = True
 600        return doc
 601
 602
 603class DiscardAds(Filter):
 604    """
 605    主に広告キーワードを`max_allowed_num`より多く含む文書を破棄します.
 606    デフォルトで`max_allowed_num` は14です.
 607    `dict_path` で指定したファイルから, 広告キーワードのリストを得ます.
 608    ファイルは単語が改行で羅列されたテキストファイルです.
 609    デフォルトの`dict_path` は /hojichar/dict/advertisement_keywords_ja.txt です.
 610
 611    `DiscardBBSComments` と同じく, `min_matches_per_1k_chars` を指定すると, 1,000 字あたりの
 612    マッチ数がこの値以上の場合にだけ破棄します. 「お問い合わせ」などの語を含む長い文書が
 613    長さだけを理由に破棄されるのを防ぎます. デフォルト値は 0.0 で, マッチ数だけで判定します.
 614    """
 615
 616    def __init__(
 617        self,
 618        dict_path: str | PathLike[str] = BASE_PATH / "dict/advertisement_keywords_ja.txt",
 619        max_allowed_num: int = 14,
 620        *args: Any,
 621        min_matches_per_1k_chars: float = 0.0,
 622        **kwargs: Any,
 623    ):
 624        """
 625        Args:
 626            dict_path: Path to the keyword list, one keyword per line.
 627            max_allowed_num: The document is removed if the keywords match more than this
 628                number of times.
 629            *args:
 630            min_matches_per_1k_chars: The document is removed only if there are also at least
 631                this many matches per 1,000 characters. Defaults to 0.0, which decides by
 632                `max_allowed_num` alone.
 633            **kwargs:
 634        """
 635        super().__init__(*args, **kwargs)
 636
 637        self.max_allow_num = max_allowed_num
 638        self.min_matches_per_1k_chars = min_matches_per_1k_chars
 639        with open(dict_path, encoding="utf-8") as fp:
 640            ng_words = fp.read().split("\n")
 641        ng_words = [re.escape(w.strip()) for w in ng_words if not len(w) == 0]
 642        pat = r"|".join(ng_words)
 643        self.keyword_pat = re.compile(pat)
 644
 645    def apply(self, doc: Document) -> Document:
 646        """
 647        >>> DiscardAds().apply(Document("お問い合わせください 営業時間 よくある質問"*5)).is_rejected
 648        True
 649
 650        >>> DiscardAds().apply(Document("おはよう")).is_rejected
 651        False
 652        """
 653        match = _find_excessive_match(
 654            self.keyword_pat, doc.text, self.max_allow_num, self.min_matches_per_1k_chars
 655        )
 656        if match is not None:
 657            doc.is_rejected = True
 658        return doc
 659
 660
 661class AcceptJapanese(Filter):
 662    """
 663    日本語でないドキュメントを破棄します. 日本語判定は次の手順で行われます.
 664        1. テキストを左から`lookup_size` (デフォルトで50字) 参照し,
 665        ひらがな・カタカナが存在すれば日本語と判定する.
 666    """
 667
 668    def __init__(self, lookup_size: int = 50, *args: Any, **kwargs: Any) -> None:
 669        super().__init__(*args, **kwargs)
 670
 671        self.lookup_size = lookup_size
 672        self.hiragana_katakana_pat = re.compile(r"[ぁ-んァ-ン]")
 673
 674    def apply(self, doc: Document) -> Document:
 675        """
 676        >>> AcceptJapanese().apply(Document("This is English document")).is_rejected
 677        True
 678
 679        >>> AcceptJapanese().apply(Document("a"*50 + "あ")).is_rejected
 680        True
 681
 682        >>> AcceptJapanese().apply(Document("ほうじ茶")).is_rejected
 683        False
 684        """
 685        if not self.hiragana_katakana_pat.search(doc.text[: self.lookup_size]):
 686            doc.is_rejected = True
 687        return doc
 688
 689
 690class DiscardRareKuten(Filter):
 691    """
 692    日本語でないドキュメントを破棄します. 日本語判定は次の手順で行われます
 693    ドキュメントを句点"。"で区切り, 平均文長が
 694    `max_avarage_sentence_length` より長い場合は破棄します.
 695    `max_avarage_sentence_length` のデフォルト値は100です.
 696    このフィルタは, 文章中の句点の割合が少なすぎるドキュメントを破棄します.
 697    """
 698
 699    def __init__(self, max_average_sentence_length: int = 100, *args: Any, **kwargs: Any) -> None:
 700        super().__init__(*args, **kwargs)
 701
 702        self.max_average_sentence_length = max_average_sentence_length
 703        self.kuten_pat = re.compile(r"。")
 704
 705    def apply(self, doc: Document) -> Document:
 706        """
 707        >>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよ。")).is_rejected
 708        False
 709        >>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよう。")).is_rejected
 710        True
 711        """
 712        kuten_lst = self.kuten_pat.findall(doc.text)
 713        min_kuten_num = len(doc.text) / self.max_average_sentence_length
 714        if len(kuten_lst) < min_kuten_num:
 715            doc.is_rejected = True
 716        return doc
 717
 718
 719class HeaderFooterTagsRemover(Filter):
 720    """
 721    ドキュメントの冒頭・末尾のトークンを調査し, ヘッダー・フッダー的な
 722    タグが存在していた場合, そのトークンを除去します.
 723
 724    このフィルタを通す前に, 事前にセンテンスレベルにトーカナイズしておいてください.
 725    このフィルタでは Document.token にのみ変更が加えられるので, 出力前 あるいは 下流フィルタで
 726    Document.text に変更を加える前にトークンをマージしておいてください.
 727    """
 728
 729    def __init__(
 730        self,
 731        dict_path: str | PathLike[str] = BASE_PATH / "dict/header_footer_keywords_ja.txt",
 732        *args: Any,
 733        **kwargs: Any,
 734    ) -> None:
 735        super().__init__(*args, **kwargs)
 736
 737        with open(dict_path) as fp:
 738            keywords = fp.read().split("\n")
 739        keywords = [re.escape(w.strip()) for w in keywords if not len(w) == 0]
 740        self.keyword_pat = re.compile(r"|".join(keywords))
 741
 742    def apply(self, doc: Document) -> Document:
 743        if len(doc.tokens) == 0:
 744            return doc
 745
 746        lookup_size = 0
 747        if 1 <= len(doc.tokens) < 4:
 748            lookup_size = 1
 749        elif 4 <= len(doc.tokens) < 6:
 750            lookup_size = 2
 751        elif 6 <= len(doc.tokens):
 752            lookup_size = 3
 753
 754        for i in range(lookup_size):
 755            if self.should_drop_token(doc.tokens[i]):
 756                doc.tokens[i].is_rejected = True
 757            if self.should_drop_token(doc.tokens[-(i + 1)]):
 758                doc.tokens[i].is_rejected = True
 759
 760        return doc
 761
 762    def should_drop_token(self, token: Token) -> bool:
 763        """
 764        >>> HeaderFooterTagsRemover().should_drop_token(Token("<TEST_STRING_OF_KEYWORD>"))
 765        True
 766
 767        >>> HeaderFooterTagsRemover().should_drop_token(Token("ほうじ茶"))
 768        False
 769
 770        Comment.
 771        Original legacy code removed a pattern r"« _ | Main | _ »" .
 772        In the pattern, "|" is not escaped, so **ANY** string was eliminated.
 773        It seems unintended behavior, so I fix this.
 774        """
 775        if self.keyword_pat.match(token.text):
 776            return True
 777        else:
 778            return False
 779
 780
 781class MaskPersonalInformation(Filter):
 782    """
 783    ドキュメントに含まれる電話番号・電子メールアドレスを一部マスキングします.
 784
 785    電子メールアドレスのローカル部 ("@" の前) は, RFC 5321 の上限である 64 字までを対象にします.
 786    "@" の前に英数字や記号が 64 字を超えて続く場合は, "@" の直前の 64 字からマスクします.
 787    以前は上限がなく, 英数字や記号が長く続くテキスト (特に "@" を多く含むもの) では,
 788    処理時間がその長さの2乗に比例していました.
 789    """
 790
 791    def __init__(self, *args: Any, **kwargs: Any) -> None:
 792        super().__init__(*args, **kwargs)
 793
 794        self.phone_pat = re.compile(
 795            r"((0|\+\d{1,3}[- ]?)(\d{2}[- ]?\d{4}[- ]?|\d[- ]?\d{4}[- ]?|\d{2}[- ]?\d{3}[- ]?|\d{3}[- ]?\d{2}[- ]?|\d{4}[- ]?\d{1}[- ]?))\d{4}"  # noqa
 796        )
 797        # The local part is bounded to 64 characters, the limit of RFC 5321. Unbounded, a long
 798        # run of these characters was rescanned from every position in it, which is quadratic
 799        # in the run length. With the bound, at most 64 starting positions reach each "@", so
 800        # the scan is linear in the text length.
 801        self.email_pat = re.compile(
 802            r"[a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~.]{1,64}@[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+(\.[A-Za-z0-9\-]+)"  # noqa
 803        )
 804
 805    def apply(self, doc: Document) -> Document:
 806        """
 807        >>> MaskPersonalInformation()('06-1234-5678')
 808        '06-1234-XXXX'
 809        >>> MaskPersonalInformation()('075-123-4567')
 810        '075-123-XXXX'
 811        >>> MaskPersonalInformation()('0166-12-3456')
 812        '0166-12-XXXX'
 813        >>> MaskPersonalInformation()('09808-1-2345')
 814        '09808-1-XXXX'
 815        >>> MaskPersonalInformation()('090-1234-5678')
 816        '090-1234-XXXX'
 817        >>> MaskPersonalInformation()('0751234567')
 818        '075123XXXX'
 819        >>> MaskPersonalInformation()('08012345678')
 820        '0801234XXXX'
 821        >>> MaskPersonalInformation()('連絡は075-123-4567 まで')
 822        '連絡は075-123-XXXX まで'
 823        >>> MaskPersonalInformation()('+81-80-1234-5678')
 824        '+81-80-1234-XXXX'
 825        >>> MaskPersonalInformation()('+818012345678')
 826        '+81801234XXXX'
 827        >>> MaskPersonalInformation()('hogehoge@example.com')
 828        'xxxx@yyy.com'
 829        >>> MaskPersonalInformation()('何かあれば hogehoge@example.ne.jp まで連絡')
 830        '何かあれば xxxx@yyy.jp まで連絡'
 831        >>> MaskPersonalInformation()('a' * 70 + '@example.com')
 832        'aaaaaaxxxx@yyy.com'
 833        """
 834        text = self.phone_pat.sub(r"\1XXXX", doc.text)
 835        if "@" in text:
 836            text = self.email_pat.sub(r"xxxx@yyy\1", text)
 837        doc.text = text
 838        return doc
 839
 840
 841class DiscardTooManyNouns(Filter):
 842    """
 843    [!CAUTION] This filter requires `fugashi` package. Please install it
 844    by `pip install 'hojichar[all]'`.
 845
 846    A filter that removes document with too many nouns in Japanese i.e.,
 847    documents such as advertisement, word salad, etc ...
 848    """
 849
 850    def __init__(
 851        self, threshold: float = 0.80, max_parse_chars: int = 100_000, *args: Any, **kwargs: Any
 852    ) -> None:
 853        """
 854        Args:
 855            threshold: document whose noun ratio is higher than this value will be discarded
 856            max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document.
 857            *args:
 858            **kwargs:
 859        """
 860        super().__init__(*args, **kwargs)
 861        assert is_loaded_extras, (
 862            "fugashi is required for this filter. Try pip install 'hojichar[all]'"
 863        )
 864
 865        self.threshold = threshold
 866        self.max_parse_chars = max_parse_chars
 867        self.tagger = Tagger("-Owakati")
 868        assert "unidic" in self.tagger.dictionary_info[0]["filename"], (
 869            "MeCab dictionary must be unidic"
 870        )
 871
 872    def _chunk_text(self, text: str) -> Iterable[str]:
 873        """Slice text into chunks of `max_parse_chars` length."""
 874        step = self.max_parse_chars
 875        for i in range(0, len(text), step):
 876            yield text[i : i + step]
 877
 878    def apply(self, doc: Document) -> Document:
 879        """
 880        >>> DiscardTooManyNouns().apply(Document("自然言語処理大好き!")).is_rejected
 881        False
 882        >>> DiscardTooManyNouns().apply(Document("リンゴ・オレンジ・ミカン・バナナ セール中")).is_rejected
 883        True
 884        >>> DiscardTooManyNouns().apply(Document("今日の仙台朝市ではリンゴがセール中")).is_rejected
 885        False
 886        """
 887        # remove "補助記号" from part-of-speech statistics
 888        # because they often decrease the noun ratio,
 889        # e.g., the sentence "リンゴ・オレンジ・バナナ・" has 補助記号 ratio of 0.5
 890        # however, we don't want such sentence
 891
 892        pos_count: Counter[str] = Counter()
 893        for chunk in self._chunk_text(doc.text):
 894            for word in self.tagger(chunk):
 895                if word.feature.pos1 != "補助記号":
 896                    pos_count[word.feature.pos1] += 1
 897
 898        try:
 899            noun_ratio = pos_count["名詞"] / sum(pos_count.values())
 900        except ZeroDivisionError:
 901            noun_ratio = 0.0
 902        if noun_ratio >= self.threshold:
 903            doc.is_rejected = True
 904        return doc
 905
 906
 907class CharRepetitionRatioFilter(Filter):
 908    """
 909    文字Ngramの重なり率(文書中で高頻度文字Ngramが占める割合)を計算して, 重なりの大きいものを除去します.
 910    名詞の連続からなるような広告テキストを取り除くのに有効です.
 911
 912    実装は, BigScience で採用されていた前処理を参考にしています.
 913    元実装: https://github.com/bigscience-workshop/data-preparation/blob/9d0588419073cc5bf0fb92b58f37f2a1016572c3/preprocessing/training/01b_oscar_cleaning_and_filtering/filtering.py#L425-L453  # noqa: E501
 914
 915    「高頻度文字Ngram」は、sqrt(ユニークなNgramの総数)によって求めていますが,
 916    これは文書長の影響を軽減するためだとされています.
 917
 918    掲示板のテキストが引っかかりやすい傾向があります.
 919    13: 名無しさん@実況で競馬板アウト 2019/08/18(日) 15:28:46.10 ID:eBvZg8h+0
 920    的なものが高頻度で登場するため、文字Ngramの重なり率も高くなってしまう
 921    """
 922
 923    def __init__(
 924        self, threshold: float = 0.33, ngram_size: int = 5, *args: Any, **kwargs: Any
 925    ) -> None:
 926        """
 927
 928        Args:
 929            threshold: document with character repetition ratio higher than this value will be discarded
 930            ngram_size: character ngram size. Larger value will decrease the false positive of long documents
 931            *args:
 932            **kwargs:
 933        """  # noqa: E501
 934
 935        super().__init__(*args, **kwargs)
 936        self.threshold = threshold
 937        self.ngram_size = ngram_size
 938
 939    def apply(self, doc: Document) -> Document:
 940        ratio = self.compute_character_repetition_ratio(doc.text, self.ngram_size)
 941        if ratio >= self.threshold:
 942            doc.is_rejected = True
 943        return doc
 944
 945    @staticmethod
 946    def compute_character_repetition_ratio(
 947        document: str, character_repetition_length: int
 948    ) -> float:
 949        def get_freq_character_ngrams(document: str, n: int) -> dict[str, int]:
 950            character_ngrams: list[str] = [
 951                document[i : i + n] for i in range(len(document) - n + 1)
 952            ]
 953            freq_character_ngrams_dict: dict[str, int] = {}
 954            for character_ngram in character_ngrams:
 955                freq_character_ngrams_dict[character_ngram] = (
 956                    freq_character_ngrams_dict.get(character_ngram, 0) + 1
 957                )
 958            return freq_character_ngrams_dict
 959
 960        freq_character_ngrams_dict = get_freq_character_ngrams(
 961            document, character_repetition_length
 962        )
 963        if len(freq_character_ngrams_dict) == 0:
 964            return 0.0
 965        freq_character_ngrams: list[int] = list(freq_character_ngrams_dict.values())
 966        freq_character_ngrams = sorted(freq_character_ngrams, reverse=True)
 967        val_one = len([el for el in freq_character_ngrams if el == 1])
 968        num_rep_character_ngrams = min(
 969            int(np.sqrt(len(freq_character_ngrams))),
 970            len(freq_character_ngrams) - val_one,
 971        )
 972        character_repetition_ratio = sum(freq_character_ngrams[:num_rep_character_ngrams]) / sum(
 973            freq_character_ngrams
 974        )
 975        return character_repetition_ratio
 976
 977
 978class WordRepetitionRatioFilter(Filter):
 979    """
 980    [!CAUTION] This filter requires `fugashi` package. Please install it
 981    by `pip install 'hojichar[all]'`.
 982
 983    単語Ngramの重なり率(文書中で重複する単語Ngramが占める割合)を計算して、重なりの大きいものを弾くためのフィルタ.
 984    BigScienceで採用されていた前処理を参考にしている.
 985
 986    名詞が連打されているような広告テキストを取り除くのに有効な様子
 987    まともな文書がたまたま2回繰り返されている場合もあり、これを取り除いて良いのかは分からない
 988    例:
 989    "ウェブ\n本文: ニコンの上昇率16%超える、今3月期は経常76%の大幅増益見込む(ニコン) 2013年05月10日[minkabu PRESS] - みんなの株式 (みんかぶ)\n2013/05/10(10:57)
 990    ニコン<7731.T>が急騰、寄り付き直後に前日比355円高の2537円まで買い上げ
 991    られ、上昇率は16%を超えた。外国為替市場で円が1ドル100円台、1ユーロ131円台に入るなど急速に円安が進み、輸出株が軒並み高になる
 992    なか、9日取引終了後に発表した前年3月期決算で、今3月期は2ケタ近い増収で大幅増益を見込んだことが買い気を強めさせた。連結売上
 993    高は前期比9.8%増の1兆1100億円、経常利益75.8%増の850億円を予想。前期は半導体、電子部品の低迷が足かせになり、2ケタ増収ながら
 994    経常46%の大幅減益になったが、レンズ交換式デジタルカメラの拡大や液晶ディスプレイの回復で収益が急回復する。ニコンの株価は10時
 995    56分現在2491円(△309円)出所:株経通信(株式会社みんかぶ)\n2013/05/10 - ニコン(7731) の関連ニュース。 ニコン<7731.T>が急騰、寄
 996    り付き直後に前日比355円高の2537円まで買い上げられ、上昇率は16%を超えた。外国為替市場で円が1ドル100円台、1ユーロ131円台に入
 997    るなど急速に円安が進み、輸出株が軒並み高になるなか、9日取引終了後に発表した前年3月期決算で、今3月期は2ケタ近い増収で大幅増
 998    益を見込んだことが買い気を強めさせた。連結売上高は前期比9.8%増の1兆1100億円、経常利益75.8%増の850億円を予想。前期は半導体、
 999    電子部品の低迷が足かせになり、2ケタ増収ながら経常46%の大幅減益になったが、レンズ交換式デジタルカメラの拡大や液晶ディスプレ
1000    イの回復で収益が急回"
1001    """  # noqa: E501
1002
1003    def __init__(
1004        self,
1005        threshold: float = 0.40,
1006        ngram_size: int = 7,
1007        max_parse_chars: int = 100_000,
1008        *args: Any,
1009        **kwargs: Any,
1010    ) -> None:
1011        """
1012
1013        Args:
1014            threshold: document whose character repetition ratio is higher than this value will be discarded
1015            ngram_size: character ngram size. Larger value will decrease the false positive of long documents
1016            max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document.
1017            *args:
1018            **kwargs:
1019        """  # noqa: E501
1020        super().__init__(*args, **kwargs)
1021        assert is_loaded_extras, (
1022            "fugashi is required for this filter. Try pip install 'hojichar[all]'"
1023        )
1024
1025        self.threshold = threshold
1026        self.ngram_size = ngram_size
1027        self.max_parse_chars = max_parse_chars
1028        self.tagger = Tagger("-Owakati")
1029
1030    def _chunk_text(self, text: str) -> Iterable[str]:
1031        """Split text into chunks of `max_parse_chars` length."""
1032        step = self.max_parse_chars
1033        for i in range(0, len(text), step):
1034            yield text[i : i + step]
1035
1036    def _get_freq_word_ngrams(self, words: list[str], n: int) -> dict[str, int]:
1037        freq: dict[str, int] = {}
1038        if n <= 0 or len(words) < n:
1039            return freq
1040        for i in range(len(words) - n + 1):
1041            key = " ".join(words[i : i + n])
1042            freq[key] = freq.get(key, 0) + 1
1043        return freq
1044
1045    def apply(self, doc: Document) -> Document:
1046        ratio = self.compute_word_repetition_ratio(doc.text, self.ngram_size)
1047        if ratio >= self.threshold:
1048            doc.is_rejected = True
1049        return doc
1050
1051    def compute_word_repetition_ratio(self, document: str, n: int) -> float:
1052        total_counter: Counter[str] = Counter()
1053
1054        for chunk in self._chunk_text(document):
1055            words = [w.surface for w in self.tagger(chunk)]
1056            total_counter.update(self._get_freq_word_ngrams(words, n))
1057
1058        if not total_counter:
1059            return 0.0
1060
1061        total = sum(total_counter.values())
1062        repeated = sum(v for v in total_counter.values() if v > 1)
1063        return repeated / total
1064
1065
1066class DiscardTooManySpecialToken(Filter):
1067    """
1068    [!CAUTION] This filter requires `emoji` package. Please install it
1069    by `pip install 'hojichar[all]'`.
1070
1071    句読点を含む記号、空白、絵文字、その他特殊な文字を一定の割合以上含むような文書を取り除くためのフィルタ
1072    元実装: BigScience https://github.com/bigscience-workshop/data-preparation/blob/9d0588419073cc5bf0fb92b58f37f2a1016572c3/preprocessing/training/01b_oscar_cleaning_and_filtering/parameters_filtering.py#L5-L16  # noqa: E501
1073    """
1074
1075    def __init__(self, threshold: float = 0.4, *args: Any, **kwargs: Any) -> None:
1076        """
1077
1078        Args:
1079            threshold: document whose special token ratio is higher than this value will be discarded
1080            *args:
1081            **kwargs:
1082        """  # noqa: E501
1083        super().__init__(*args, **kwargs)
1084
1085        # digits are not regarded as special tokens
1086        # otherwise many false positives are made, i.e., good documents discarded
1087        main_special_characters = string.punctuation + string.whitespace  # + string.digits
1088        other_special_characters = (
1089            "’ “— ™ – •‘œ    ˜ ‚ƒ„’“”–▬…✦�­£​•€«»°·═"
1090            "×士^˘⇓()§″′´¿−±∈¢ø‚„½¼¾¹²³―⁃,ˌ¸‹›ʺˈʻ¦‐⠀‰……‑≤≥‖"
1091            "◆●■►▼▲▴∆▻¡★☆✱ːº。¯˜¥ɪ≈†:⁄♡✓⊕․.⋅÷1‟;،、¨ाাी्े◦˚"
1092            "゜ʼ≖ʼ¤℃√!?【】‿∞➤~πه۩☛₨➩☻๑٪♥ıॽ《‘©﴿٬?▷Г♫∟™ª₪®「—❖"
1093            "」﴾》�"
1094        )
1095
1096        en_emoji = emoji.EMOJI_DATA.keys()
1097
1098        special_characters_default = set(main_special_characters + other_special_characters)
1099        special_characters_default.update(en_emoji)
1100        self.special_characters = special_characters_default
1101
1102        self.threshold = threshold
1103
1104    def _compute_special_characters_ratio(self, text: str) -> float:
1105        if len(text) == 0:
1106            return 0
1107
1108        special_characters_ratio = len(
1109            [char for char in text if char in self.special_characters]
1110        ) / len(text)
1111        return special_characters_ratio
1112
1113    def apply(self, doc: Document) -> Document:
1114        special_characters_ratio = self._compute_special_characters_ratio(doc.text)
1115
1116        if special_characters_ratio > self.threshold:
1117            doc.is_rejected = True
1118        return doc
1119
1120
1121class SingleCharacterRepetitionFilter(Filter):
1122    """
1123    単一文字が大量に繰り返されているような文書を取り除くためのフィルタ
1124    そのような文書はノイズである可能性が高いため
1125    参考: BigScienceプロジェクトによると、oscarデータセットの中にバックスラッシュだけを2M個含むような文書が含まれていたらしい
1126    https://github.com/bigscience-workshop/bigscience/blob/master/train/tr8-104B-wide/chronicles.md#2m-backslash-only-samples-in-our-dataset  # noqa: E501
1127
1128    デフォルトでは文書の長さにかかわらず, `threshold` 文字以上の繰り返しが1か所でもあれば破棄します.
1129    そのため, 長い文書 (例えば 200,000 字のエージェントの実行ログ) にテスト結果の区切り線 "=====" が
1130    1行あるだけでも破棄されます. `min_ratio` を指定すると, 繰り返しの長さが文書長に対して
1131    `min_ratio` 以上の割合を占める場合にだけ破棄します.
1132
1133    >>> doc = "テストの実行ログです。\\n" * 2000 + "=" * 200
1134    >>> SingleCharacterRepetitionFilter().apply(Document(doc)).is_rejected
1135    True
1136    >>> SingleCharacterRepetitionFilter(min_ratio=0.1).apply(Document(doc)).is_rejected
1137    False
1138    """
1139
1140    def __init__(
1141        self,
1142        threshold: int = 200,
1143        *args: Any,
1144        min_ratio: float = 0.0,
1145        **kwargs: Any,
1146    ) -> None:
1147        """
1148        Args:
1149            threshold: The document is removed if character is repeated for this value or more
1150            *args:
1151            min_ratio: The document is removed only if the repeated run is also at least this
1152                fraction of the document length. Defaults to 0.0, which decides by `threshold`
1153                alone.
1154            **kwargs:
1155        """
1156        super().__init__(*args, **kwargs)
1157        self.threshold = threshold
1158        self.min_ratio = min_ratio
1159
1160    def _is_repeat_contained(self, text: str) -> bool:
1161        min_run = max(self.threshold, self.min_ratio * len(text))
1162        groups = groupby(text)
1163        is_repeat_contained = any(sum(1 for _ in group) >= min_run for _, group in groups)
1164        return is_repeat_contained
1165
1166    def apply(self, doc: Document) -> Document:
1167        if self._is_repeat_contained(doc.text):
1168            doc.is_rejected = True
1169        return doc
1170
1171
1172class DiscardTooManyEndingEllipsis(Filter):
1173    """
1174    ellipsisで終わるような行が大量に含まれるような文書を取り除くためのフィルタです.
1175    ellipsisとしては ... と … を用いている
1176    同様のフィルタが RedPajama v2で用いられています.
1177
1178    例として, 以下のような文書を検知します.
1179    ```
1180    ペアーズは女性、という驚愕の過食が出ているのをごアラサーですか。時代から付...
1181    バツイチアラフォー 婚活ち女性の特徴と子持な付...
1182    ```
1183
1184    デフォルトではしきい値を0.7としているが, これはC4から0.1%を削るような設定であり、
1185    precisionを重視した設定です.
1186    """
1187
1188    def __init__(
1189        self,
1190        threshold: float = 0.7,
1191        *args: Any,
1192        **kwargs: Any,
1193    ) -> None:
1194        """
1195        Args:
1196            threshold: The document is removed if ratio of lines ending with ellipsis is higher than this value
1197            *args:
1198            **kwargs:
1199        """  # noqa: E501
1200        super().__init__(*args, **kwargs)
1201        self.threshold = threshold
1202        self.ellipsis_pattern = re.compile(r"(\.{3}|…)\n")  # matches ...\n and …\n
1203
1204    def apply(self, doc: Document) -> Document:
1205        ellipsis_count = len(self.ellipsis_pattern.findall(doc.text))
1206        newline_count = max(doc.text.count("\n"), 1)  # avoid zero division
1207        ellipsis_ratio = ellipsis_count / newline_count
1208
1209        if ellipsis_ratio > self.threshold:
1210            doc.is_rejected = True
1211        return doc
1212
1213
1214class DiscardTooShortLines(Filter):
1215    """
1216    短い行を大量に含む文書を捨てるためのフィルタです.
1217
1218    メニューバーやパンくずリストのような要素を大量に含む文書を取り除くのに有効です.
1219    """
1220
1221    def __init__(self, threshold: float = 0.5, *args: Any, **kwargs: Any) -> None:
1222        """
1223        Args:
1224            threshold: The document is removed if the ratio of short (<10 chars) lines are more than this value.
1225            *args:
1226            **kwargs:
1227        """  # noqa: E501
1228        super().__init__(*args, **kwargs)
1229        self.threshold = threshold
1230        # この値は適当に決め打ち
1231        self.minimum_line_length = 10
1232
1233    def apply(self, doc: Document) -> Document:
1234        lines = [len(x) for x in doc.text.split("\n")]
1235        short_lines = [x for x in lines if x <= self.minimum_line_length]
1236        if (len(short_lines) / len(lines)) > self.threshold:
1237            doc.is_rejected = True
1238        return doc
1239
1240
1241class CollapseRepeatedDocument(Filter):
1242    """
1243    文書全体が同じブロックの2回または3回の繰り返しになっている場合に, 1回分だけを残します.
1244
1245    テキスト抽出器がページを2回続けて出力した文書などを想定しています.
1246    内容は1回分なら問題ないため, このフィルタは文書を破棄せず, テキストを書き換えます.
1247
1248    コピーの間には `max_separator_len` 字までの区切りを許し, 区切りの中身は比べません.
1249    ブロックの前後の空白を除いて比べるため, 空白や改行だけの区切りは, これより長くても
1250    畳まれる場合があります. 短い定型句の繰り返しを畳まないように, 残すブロックは
1251    `min_block_len` 字以上とします. 文書の先頭と末尾 (3回の場合は中央も) を比べるだけなので,
1252    計算量は文書長に比例します. 畳んだ場合は, 文書の前後の空白も取り除かれます.
1253
1254    >>> block = "HojiChar はテキストの前処理パイプラインを組み立てるための Python ライブラリです。"
1255    >>> CollapseRepeatedDocument()(block + "\\n" + block) == block
1256    True
1257    >>> CollapseRepeatedDocument()(block + "\\n\\n" + block + "\\n\\n" + block + "\\n") == block
1258    True
1259    >>> CollapseRepeatedDocument()("短い文です。" * 2)
1260    '短い文です。短い文です。'
1261    """
1262
1263    def __init__(
1264        self,
1265        min_block_len: int = 40,
1266        max_separator_len: int = 3,
1267        *args: Any,
1268        **kwargs: Any,
1269    ) -> None:
1270        """
1271        Args:
1272            min_block_len: Minimum length of the block to keep. Shorter repeats are left as
1273                they are.
1274            max_separator_len: Maximum length of the text between copies. Its content is not
1275                compared.
1276            *args:
1277            **kwargs:
1278        """
1279        super().__init__(*args, **kwargs)
1280        self.min_block_len = min_block_len
1281        self.max_separator_len = max_separator_len
1282
1283    def apply(self, doc: Document) -> Document:
1284        text = doc.text.strip()
1285        n = len(text)
1286        if n < 2 * self.min_block_len:
1287            return doc
1288        for num_copies in (2, 3):
1289            for total_separator_len in range(self.max_separator_len * (num_copies - 1) + 1):
1290                block_len, remainder = divmod(n - total_separator_len, num_copies)
1291                if remainder or block_len < self.min_block_len:
1292                    continue
1293                block = text[:block_len].rstrip()
1294                if len(block) < self.min_block_len or block != text[n - block_len :].lstrip():
1295                    continue
1296                if num_copies == 3:
1297                    # Each separator has its own length; the middle copy need not be centered.
1298                    min_first_len = max(0, total_separator_len - self.max_separator_len)
1299                    max_first_len = min(self.max_separator_len, total_separator_len)
1300                    for first_len in range(min_first_len, max_first_len + 1):
1301                        middle_start = block_len + first_len
1302                        if block == text[middle_start : middle_start + block_len].strip():
1303                            break
1304                    else:
1305                        continue
1306                doc.text = block
1307                return doc
1308        return doc
class ExampleHojiChar(hojichar.core.filter_interface.Filter):
54class ExampleHojiChar(Filter):
55    """基本的なフィルタの実装例です. 末尾に'<hojichar>'を追加します."""
56
57    def apply(self, document: Document) -> Document:
58        """
59        >>> ExampleHojiChar()("hello, world")
60        'hello, world<hojichar>'
61        """
62        document.text += "<hojichar>"
63        return document

基本的なフィルタの実装例です. 末尾に''を追加します.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
57    def apply(self, document: Document) -> Document:
58        """
59        >>> ExampleHojiChar()("hello, world")
60        'hello, world<hojichar>'
61        """
62        document.text += "<hojichar>"
63        return document
>>> ExampleHojiChar()("hello, world")
'hello, world<hojichar>'
class ExampleDiscardDocumentContainKeyword(hojichar.core.filter_interface.Filter):
66class ExampleDiscardDocumentContainKeyword(Filter):
67    """特定のキーワードを持つドキュメントを破棄するようなフィルタの実装例です."""
68
69    def __init__(self, keyword: str, *args: Any, **kwargs: Any) -> None:
70        super().__init__(*args, **kwargs)
71        self.keyword = keyword
72
73    def apply(self, document: Document) -> Document:
74        """
75        >>> ExampleDiscardDocumentContainKeyword("バカ").apply(Document("あいつはバカだ")).is_rejected
76        True
77        """
78        if self.keyword in document.text:
79            document.is_rejected = True
80        return document

特定のキーワードを持つドキュメントを破棄するようなフィルタの実装例です.

ExampleDiscardDocumentContainKeyword(keyword: str, *args: Any, **kwargs: Any)
69    def __init__(self, keyword: str, *args: Any, **kwargs: Any) -> None:
70        super().__init__(*args, **kwargs)
71        self.keyword = keyword

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
73    def apply(self, document: Document) -> Document:
74        """
75        >>> ExampleDiscardDocumentContainKeyword("バカ").apply(Document("あいつはバカだ")).is_rejected
76        True
77        """
78        if self.keyword in document.text:
79            document.is_rejected = True
80        return document
>>> ExampleDiscardDocumentContainKeyword("バカ").apply(Document("あいつはバカだ")).is_rejected
True
class Identity(hojichar.core.filter_interface.Filter):
83class Identity(Filter):
84    """何も変化を加えないフィルタです. テスト・デバッグに用いられます."""
85
86    def apply(self, document: Document) -> Document:
87        return document

何も変化を加えないフィルタです. テスト・デバッグに用いられます.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
86    def apply(self, document: Document) -> Document:
87        return document

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class DiscardAll(hojichar.core.filter_interface.Filter):
90class DiscardAll(Filter):
91    """
92    すべてのドキュメントを破棄するフィルタです.
93    テスト・デバッグに用いられます.
94    """
95
96    def apply(self, document: Document) -> Document:
97        document.is_rejected = True
98        return document

すべてのドキュメントを破棄するフィルタです. テスト・デバッグに用いられます.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
96    def apply(self, document: Document) -> Document:
97        document.is_rejected = True
98        return document

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class ApplyDiscard(hojichar.core.filter_interface.Filter):
101class ApplyDiscard(Filter):
102    """
103    上流フィルタで破棄された`Document`を空文字列にします.
104
105    `Document.is_rejected=True` の ドキュメントは無視されるため,
106    このフィルタを `Compose` のコンストラクタに渡しても動作しません.
107    このフィルタは主に`Compose` 内部や, `discard_filtered=False` を指定
108    したデバッグ時などに利用されます.
109    """
110
111    def __init__(self, *args: Any, **kwargs: Any) -> None:
112        super().__init__(*args, **kwargs)
113
114    def apply(self, document: Document) -> Document:
115        """
116        >>> ApplyDiscard().apply(Document(text="hello", is_rejected=True)).text
117        ''
118        """
119        if document.is_rejected:
120            document.text = ""
121
122        return document

上流フィルタで破棄されたDocumentを空文字列にします.

Document.is_rejected=True の ドキュメントは無視されるため, このフィルタを Compose のコンストラクタに渡しても動作しません. このフィルタは主にCompose 内部や, discard_filtered=False を指定 したデバッグ時などに利用されます.

ApplyDiscard(*args: Any, **kwargs: Any)
111    def __init__(self, *args: Any, **kwargs: Any) -> None:
112        super().__init__(*args, **kwargs)

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
114    def apply(self, document: Document) -> Document:
115        """
116        >>> ApplyDiscard().apply(Document(text="hello", is_rejected=True)).text
117        ''
118        """
119        if document.is_rejected:
120            document.text = ""
121
122        return document
>>> ApplyDiscard().apply(Document(text="hello", is_rejected=True)).text
''
class Sleep(hojichar.core.filter_interface.Filter):
125class Sleep(Filter):
126    """
127    デバッグ用のフィルタです. 指定秒スリープします.
128    """
129
130    def __init__(self, time: float = 1.0, *args: Any, **kwargs: Any) -> None:
131        super().__init__(*args, **kwargs)
132        self.time = time
133
134    def apply(self, document: Document) -> Document:
135        """
136        >>> Sleep(0.1)('hello')  # After 0.1 seconds,
137        'hello'
138        """
139        time.sleep(self.time)
140        return document

デバッグ用のフィルタです. 指定秒スリープします.

Sleep(time: float = 1.0, *args: Any, **kwargs: Any)
130    def __init__(self, time: float = 1.0, *args: Any, **kwargs: Any) -> None:
131        super().__init__(*args, **kwargs)
132        self.time = time

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
134    def apply(self, document: Document) -> Document:
135        """
136        >>> Sleep(0.1)('hello')  # After 0.1 seconds,
137        'hello'
138        """
139        time.sleep(self.time)
140        return document
>>> Sleep(0.1)('hello')  # After 0.1 seconds,
'hello'
class DocumentNormalizer(hojichar.core.filter_interface.Filter):
143class DocumentNormalizer(Filter):
144    """
145    Unicode の正規化をします.
146    """
147
148    def __init__(self, *args: Any, **kwargs: Any) -> None:
149        super().__init__(*args, **kwargs)
150
151    def apply(self, document: Document) -> Document:
152        document.text = unicodedata.normalize("NFKC", document.text)
153        return document

Unicode の正規化をします.

DocumentNormalizer(*args: Any, **kwargs: Any)
148    def __init__(self, *args: Any, **kwargs: Any) -> None:
149        super().__init__(*args, **kwargs)

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
151    def apply(self, document: Document) -> Document:
152        document.text = unicodedata.normalize("NFKC", document.text)
153        return document

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class JSONLoader(hojichar.core.filter_interface.Filter):
156class JSONLoader(Filter):
157    """
158    テキストを Json として解釈し, `key` で指定した要素を文字列として
159    doument に格納します.デフォルトの `key` は 'text' です.
160
161    Json の読み込み, あるいは `key` の読み込みに失敗した際には例外を送出します.
162    これらを無視する場合は, `ignore=True` にします. その際, 読み込みに失敗
163    したドキュメントは破棄されます.
164
165    入力 Json に `extras` キー(辞書形式)が含まれている場合, Document.extras に自動的にマージされます。
166    さらに `extra_keys` でフィールドを指定すると, それらの値も Document.extras に追記され, 既存の extras
167    を上書きせずに統合できます。
168    """
169
170    def __init__(
171        self,
172        key: str = "text",
173        ignore: bool = False,
174        extra_keys: list[str] | None = None,
175        *args: Any,
176        **kwargs: Any,
177    ) -> None:
178        super().__init__(*args, **kwargs)
179        self.key = key
180        self.ignore = ignore
181        self.extra_keys = extra_keys
182
183    def apply(self, document: Document) -> Document:
184        """
185        >>> JSONLoader()( '{"text": "hello, world", "words": 2}' )
186        'hello, world'
187
188        >>> JSONLoader()( '{"text": hello, world ....' ) # Broken JSON
189        Traceback (most recent call last):
190            ...
191        json.decoder.JSONDecodeError: Expecting value: line 1 column 10 (char 9)
192
193        >>> JSONLoader()( '{"words": 2}' )
194        Traceback (most recent call last):
195            ...
196        KeyError: 'text'
197
198        >>> JSONLoader(ignore=True).apply(Document('{"text": hello, world ....' )).is_rejected
199        True
200        """
201        try:
202            data = json.loads(document.text)
203            document.text = str(data[self.key])
204            if "extras" in data and isinstance(data["extras"], dict):
205                document.extras.update(data["extras"])
206            if self.extra_keys is not None:
207                for key in self.extra_keys:
208                    if key not in data:
209                        continue
210                    if key == "extras" and isinstance(data[key], dict):
211                        document.extras.update(data[key])
212                    else:
213                        document.extras[key] = data[key]
214        except Exception as e:
215            logger.error(f"Failed to parsing in JSONLoader. Input document: \n{document.text}")
216            if self.ignore:
217                document.is_rejected = True
218                return document
219            else:
220                raise e
221
222        return document

テキストを Json として解釈し, key で指定した要素を文字列として doument に格納します.デフォルトの key は 'text' です.

Json の読み込み, あるいは key の読み込みに失敗した際には例外を送出します. これらを無視する場合は, ignore=True にします. その際, 読み込みに失敗 したドキュメントは破棄されます.

入力 Json に extras キー(辞書形式)が含まれている場合, Document.extras に自動的にマージされます。 さらに extra_keys でフィールドを指定すると, それらの値も Document.extras に追記され, 既存の extras を上書きせずに統合できます。

JSONLoader( key: str = 'text', ignore: bool = False, extra_keys: list[str] | None = None, *args: Any, **kwargs: Any)
170    def __init__(
171        self,
172        key: str = "text",
173        ignore: bool = False,
174        extra_keys: list[str] | None = None,
175        *args: Any,
176        **kwargs: Any,
177    ) -> None:
178        super().__init__(*args, **kwargs)
179        self.key = key
180        self.ignore = ignore
181        self.extra_keys = extra_keys

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
183    def apply(self, document: Document) -> Document:
184        """
185        >>> JSONLoader()( '{"text": "hello, world", "words": 2}' )
186        'hello, world'
187
188        >>> JSONLoader()( '{"text": hello, world ....' ) # Broken JSON
189        Traceback (most recent call last):
190            ...
191        json.decoder.JSONDecodeError: Expecting value: line 1 column 10 (char 9)
192
193        >>> JSONLoader()( '{"words": 2}' )
194        Traceback (most recent call last):
195            ...
196        KeyError: 'text'
197
198        >>> JSONLoader(ignore=True).apply(Document('{"text": hello, world ....' )).is_rejected
199        True
200        """
201        try:
202            data = json.loads(document.text)
203            document.text = str(data[self.key])
204            if "extras" in data and isinstance(data["extras"], dict):
205                document.extras.update(data["extras"])
206            if self.extra_keys is not None:
207                for key in self.extra_keys:
208                    if key not in data:
209                        continue
210                    if key == "extras" and isinstance(data[key], dict):
211                        document.extras.update(data[key])
212                    else:
213                        document.extras[key] = data[key]
214        except Exception as e:
215            logger.error(f"Failed to parsing in JSONLoader. Input document: \n{document.text}")
216            if self.ignore:
217                document.is_rejected = True
218                return document
219            else:
220                raise e
221
222        return document
>>> JSONLoader()( '{"text": "hello, world", "words": 2}' )
'hello, world'
>>> JSONLoader()( '{"text": hello, world ....' ) # Broken JSON
Traceback (most recent call last):
    ...
json.decoder.JSONDecodeError: Expecting value: line 1 column 10 (char 9)
>>> JSONLoader()( '{"words": 2}' )
Traceback (most recent call last):
    ...
KeyError: 'text'
>>> JSONLoader(ignore=True).apply(Document('{"text": hello, world ....' )).is_rejected
True
class JSONDumper(hojichar.core.filter_interface.Filter):
225class JSONDumper(Filter):
226    """
227    Document.text の文字列を json に変換します.
228    必要に応じ Document のメタデータを付与します. これはドキュメントの破棄事由が含まれ、偽陽性の分析に有効です。
229    デフォルトで `skip_rejected` が `False` にセットされており、Document の破棄フラグにかかわらず
230    処理されます。
231    """
232
233    def __init__(
234        self,
235        dump_reason: bool = False,
236        p: float = 1,
237        skip_rejected: bool = False,
238        export_extras: bool = False,
239        *args: Any,
240        **kwargs: Any,
241    ) -> None:
242        """
243        Args:
244            dump_reason (bool, optional): `is_rejected`, `reason` エントリをダンプします. Defaults to False.
245            p (float, optional): Apply probability. Defaults to 1.
246            skip_rejected (bool, optional): 破棄済みサンプルを排除しません.
247        """
248        super().__init__(p, skip_rejected, *args, **kwargs)
249        self.dump_reason = dump_reason
250        self.export_extras = export_extras
251
252    def apply(self, document: Document) -> Document:
253        """
254        >>> JSONDumper()("hojichar")
255        '{"text": "hojichar"}'
256        """
257        text = document.text
258        if self.dump_reason:
259            if self.export_extras:
260                output_extras = dict(document.extras)
261                document.text = json.dumps(
262                    {
263                        "text": text,
264                        "is_rejected": document.is_rejected,
265                        "reason": document.reject_reason,
266                        "extras": output_extras,
267                    },
268                    ensure_ascii=False,
269                )
270            else:
271                document.text = json.dumps(
272                    {
273                        "text": text,
274                        "is_rejected": document.is_rejected,
275                        "reason": document.reject_reason,
276                    },
277                    ensure_ascii=False,
278                )
279        else:
280            if self.export_extras:
281                output_extras = dict(document.extras)
282                document.text = json.dumps(
283                    {
284                        "text": text,
285                        "extras": output_extras,
286                    },
287                    ensure_ascii=False,
288                )
289            else:
290                document.text = json.dumps({"text": text}, ensure_ascii=False)
291        return document

Document.text の文字列を json に変換します. 必要に応じ Document のメタデータを付与します. これはドキュメントの破棄事由が含まれ、偽陽性の分析に有効です。 デフォルトで skip_rejected が False にセットされており、Document の破棄フラグにかかわらず 処理されます。

JSONDumper( dump_reason: bool = False, p: float = 1, skip_rejected: bool = False, export_extras: bool = False, *args: Any, **kwargs: Any)
233    def __init__(
234        self,
235        dump_reason: bool = False,
236        p: float = 1,
237        skip_rejected: bool = False,
238        export_extras: bool = False,
239        *args: Any,
240        **kwargs: Any,
241    ) -> None:
242        """
243        Args:
244            dump_reason (bool, optional): `is_rejected`, `reason` エントリをダンプします. Defaults to False.
245            p (float, optional): Apply probability. Defaults to 1.
246            skip_rejected (bool, optional): 破棄済みサンプルを排除しません.
247        """
248        super().__init__(p, skip_rejected, *args, **kwargs)
249        self.dump_reason = dump_reason
250        self.export_extras = export_extras

Args: dump_reason (bool, optional): is_rejected, reason エントリをダンプします. Defaults to False. p (float, optional): Apply probability. Defaults to 1. skip_rejected (bool, optional): 破棄済みサンプルを排除しません.

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
252    def apply(self, document: Document) -> Document:
253        """
254        >>> JSONDumper()("hojichar")
255        '{"text": "hojichar"}'
256        """
257        text = document.text
258        if self.dump_reason:
259            if self.export_extras:
260                output_extras = dict(document.extras)
261                document.text = json.dumps(
262                    {
263                        "text": text,
264                        "is_rejected": document.is_rejected,
265                        "reason": document.reject_reason,
266                        "extras": output_extras,
267                    },
268                    ensure_ascii=False,
269                )
270            else:
271                document.text = json.dumps(
272                    {
273                        "text": text,
274                        "is_rejected": document.is_rejected,
275                        "reason": document.reject_reason,
276                    },
277                    ensure_ascii=False,
278                )
279        else:
280            if self.export_extras:
281                output_extras = dict(document.extras)
282                document.text = json.dumps(
283                    {
284                        "text": text,
285                        "extras": output_extras,
286                    },
287                    ensure_ascii=False,
288                )
289            else:
290                document.text = json.dumps({"text": text}, ensure_ascii=False)
291        return document
>>> JSONDumper()("hojichar")
'{"text": "hojichar"}'
class DocumentLengthFilter(hojichar.core.filter_interface.Filter):
294class DocumentLengthFilter(Filter):
295    """
296    `min_doc_len`, `max_doc_len` で指定した上限・下限の範囲内にないドキュメントを破棄します.
297    デフォルトでは 200字 以上 50000字以内のテキストが受理されます.
298    """
299
300    def __init__(
301        self,
302        min_doc_len: int | None = None,
303        max_doc_len: int | None = None,
304        *args: Any,
305        **kwargs: Any,
306    ) -> None:
307        super().__init__(*args, **kwargs)
308
309        self.min_doc_len = min_doc_len
310        self.max_doc_len = max_doc_len
311
312    def apply(self, doc: Document) -> Document:
313        """
314        >>> DocumentLengthFilter(min_doc_len=5).apply(Document("1234")).is_rejected
315        True
316        """
317        doc_len = len(doc.text)
318        if self.min_doc_len is not None:
319            if doc_len < self.min_doc_len:
320                doc.is_rejected = True
321        if self.max_doc_len is not None:
322            if self.max_doc_len < doc_len:
323                doc.is_rejected = True
324        return doc

min_doc_len, max_doc_len で指定した上限・下限の範囲内にないドキュメントを破棄します. デフォルトでは 200字 以上 50000字以内のテキストが受理されます.

DocumentLengthFilter( min_doc_len: int | None = None, max_doc_len: int | None = None, *args: Any, **kwargs: Any)
300    def __init__(
301        self,
302        min_doc_len: int | None = None,
303        max_doc_len: int | None = None,
304        *args: Any,
305        **kwargs: Any,
306    ) -> None:
307        super().__init__(*args, **kwargs)
308
309        self.min_doc_len = min_doc_len
310        self.max_doc_len = max_doc_len

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
312    def apply(self, doc: Document) -> Document:
313        """
314        >>> DocumentLengthFilter(min_doc_len=5).apply(Document("1234")).is_rejected
315        True
316        """
317        doc_len = len(doc.text)
318        if self.min_doc_len is not None:
319            if doc_len < self.min_doc_len:
320                doc.is_rejected = True
321        if self.max_doc_len is not None:
322            if self.max_doc_len < doc_len:
323                doc.is_rejected = True
324        return doc
>>> DocumentLengthFilter(min_doc_len=5).apply(Document("1234")).is_rejected
True
class NgWordsFilterJa(hojichar.core.filter_interface.Filter):
327class NgWordsFilterJa(Filter):
328    """
329    日本語のNGワード(および不適切語)を含む文書を破棄します.
330    `dict_path` で指定したファイルから, キーワードのリストを得ます.
331    ファイルは単語が改行で羅列されたテキストファイルです.
332
333    `ignore_confused` を `True` にすると,
334    偽陽性を軽減するために, カタカナのNGワードは前後にカタカナが無い場合のみNG判定されます.
335    デフォルト値は `False` です.
336
337    デフォルトでは, NGワードに1回でもマッチした文書を破棄します. `max_allowed_num` と
338    `min_matches_per_1k_chars` を指定すると, マッチ数が `max_allowed_num` 回より多く, かつ
339    1,000 字あたりのマッチ数が `min_matches_per_1k_chars` 以上の文書だけを破棄します.
340    NGワードが数回だけ現れる長い文書 (生物学の記事など) を残し, NGワードを並べたページを
341    破棄する用途を想定しています. この2つの引数は, このクラスを継承する
342    `DiscardAdultContentJa`, `DiscardDiscriminationContentJa`, `DiscardViolenceContentJa` でも
343    使えます.
344    """
345
346    def __init__(
347        self,
348        dict_path: str | PathLike[str],
349        ignore_confused: bool = False,
350        *args: Any,
351        max_allowed_num: int = 0,
352        min_matches_per_1k_chars: float = 0.0,
353        **kwargs: Any,
354    ) -> None:
355        """
356        Args:
357            dict_path: Path to the NG word list, one word per line.
358            ignore_confused: Match katakana words only where no katakana precedes or follows.
359            *args:
360            max_allowed_num: The document is removed if the words match more than this number
361                of times. Defaults to 0, which removes the document on the first match.
362            min_matches_per_1k_chars: The document is removed only if there are also at least
363                this many matches per 1,000 characters. Defaults to 0.0, which decides by
364                `max_allowed_num` alone.
365            **kwargs:
366        """
367        super().__init__(*args, **kwargs)
368        self.max_allowed_num = max_allowed_num
369        self.min_matches_per_1k_chars = min_matches_per_1k_chars
370
371        with open(dict_path, encoding="utf-8") as fp:
372            ng_words = fp.read().split("\n")
373        ng_words = [w.strip() for w in ng_words if not len(w) == 0]
374
375        if ignore_confused:
376            words_katakana = []
377            words_not_katakana = []
378            for w in ng_words:
379                if re.fullmatch(r"[ァ-ヴー]+", w):
380                    words_katakana.append(re.escape(w))
381                else:
382                    words_not_katakana.append(re.escape(w))
383            katakana_pat = "|".join(words_katakana)
384            katakana_pat = rf"(?<![ァ-ヴー])({katakana_pat})(?![ァ-ヴー])"
385            pat = "|".join(words_not_katakana) + "|" + katakana_pat
386            self.keyword_pat = re.compile(pat)
387        else:
388            ng_words = [re.escape(w) for w in ng_words]
389            pat = "|".join(ng_words)
390            self.keyword_pat = re.compile(pat)
391
392    def apply(self, doc: Document) -> Document:
393        regex_match = _find_excessive_match(
394            self.keyword_pat, doc.text, self.max_allowed_num, self.min_matches_per_1k_chars
395        )
396        if regex_match is not None:
397            doc.is_rejected = True
398            self.matched_text = regex_match.group()
399            self.matched_text_neighbor = doc.text[
400                regex_match.start() - 20 : regex_match.end() + 20
401            ]
402
403        return doc

日本語のNGワード(および不適切語)を含む文書を破棄します. dict_path で指定したファイルから, キーワードのリストを得ます. ファイルは単語が改行で羅列されたテキストファイルです.

ignore_confused を True にすると, 偽陽性を軽減するために, カタカナのNGワードは前後にカタカナが無い場合のみNG判定されます. デフォルト値は False です.

デフォルトでは, NGワードに1回でもマッチした文書を破棄します. max_allowed_num と min_matches_per_1k_chars を指定すると, マッチ数が max_allowed_num 回より多く, かつ 1,000 字あたりのマッチ数が min_matches_per_1k_chars 以上の文書だけを破棄します. NGワードが数回だけ現れる長い文書 (生物学の記事など) を残し, NGワードを並べたページを 破棄する用途を想定しています. この2つの引数は, このクラスを継承する DiscardAdultContentJa, DiscardDiscriminationContentJa, DiscardViolenceContentJa でも 使えます.

NgWordsFilterJa( dict_path: str | os.PathLike[str], ignore_confused: bool = False, *args: Any, max_allowed_num: int = 0, min_matches_per_1k_chars: float = 0.0, **kwargs: Any)
346    def __init__(
347        self,
348        dict_path: str | PathLike[str],
349        ignore_confused: bool = False,
350        *args: Any,
351        max_allowed_num: int = 0,
352        min_matches_per_1k_chars: float = 0.0,
353        **kwargs: Any,
354    ) -> None:
355        """
356        Args:
357            dict_path: Path to the NG word list, one word per line.
358            ignore_confused: Match katakana words only where no katakana precedes or follows.
359            *args:
360            max_allowed_num: The document is removed if the words match more than this number
361                of times. Defaults to 0, which removes the document on the first match.
362            min_matches_per_1k_chars: The document is removed only if there are also at least
363                this many matches per 1,000 characters. Defaults to 0.0, which decides by
364                `max_allowed_num` alone.
365            **kwargs:
366        """
367        super().__init__(*args, **kwargs)
368        self.max_allowed_num = max_allowed_num
369        self.min_matches_per_1k_chars = min_matches_per_1k_chars
370
371        with open(dict_path, encoding="utf-8") as fp:
372            ng_words = fp.read().split("\n")
373        ng_words = [w.strip() for w in ng_words if not len(w) == 0]
374
375        if ignore_confused:
376            words_katakana = []
377            words_not_katakana = []
378            for w in ng_words:
379                if re.fullmatch(r"[ァ-ヴー]+", w):
380                    words_katakana.append(re.escape(w))
381                else:
382                    words_not_katakana.append(re.escape(w))
383            katakana_pat = "|".join(words_katakana)
384            katakana_pat = rf"(?<![ァ-ヴー])({katakana_pat})(?![ァ-ヴー])"
385            pat = "|".join(words_not_katakana) + "|" + katakana_pat
386            self.keyword_pat = re.compile(pat)
387        else:
388            ng_words = [re.escape(w) for w in ng_words]
389            pat = "|".join(ng_words)
390            self.keyword_pat = re.compile(pat)

Args: dict_path: Path to the NG word list, one word per line. ignore_confused: Match katakana words only where no katakana precedes or follows. args: max_allowed_num: The document is removed if the words match more than this number of times. Defaults to 0, which removes the document on the first match. min_matches_per_1k_chars: The document is removed only if there are also at least this many matches per 1,000 characters. Defaults to 0.0, which decides by max_allowed_num alone. *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
392    def apply(self, doc: Document) -> Document:
393        regex_match = _find_excessive_match(
394            self.keyword_pat, doc.text, self.max_allowed_num, self.min_matches_per_1k_chars
395        )
396        if regex_match is not None:
397            doc.is_rejected = True
398            self.matched_text = regex_match.group()
399            self.matched_text_neighbor = doc.text[
400                regex_match.start() - 20 : regex_match.end() + 20
401            ]
402
403        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class NgWordsFilterEn(hojichar.core.filter_interface.Filter):
406class NgWordsFilterEn(Filter):
407    """
408    英語のNGワード(および不適切語)を含む文書を破棄します.
409    `dict_path` で指定したファイルから, キーワードのリストを得ます.
410    ファイルは単語が改行で羅列されたテキストファイルです.
411    """
412
413    def __init__(self, dict_path: str | PathLike[str], *args: Any, **kwargs: Any) -> None:
414        super().__init__(*args, **kwargs)
415
416        with open(dict_path, encoding="utf-8") as fp:
417            ng_words = fp.read().split("\n")
418        ng_words = [re.escape(w.strip()) for w in ng_words if not len(w) == 0]
419        pat = "|".join(ng_words)
420        # 英語のパターンにマッチするようにしている, \s[単語]\s や [単語]. [単語], などにマッチ.
421        self.keyword_pat = re.compile(rf"(?:^| )({pat})(?:( |,|\.)|$)", re.IGNORECASE)
422
423    def apply(self, doc: Document) -> Document:
424        if self.keyword_pat.search(doc.text):
425            doc.is_rejected = True
426        return doc

英語のNGワード(および不適切語)を含む文書を破棄します. dict_path で指定したファイルから, キーワードのリストを得ます. ファイルは単語が改行で羅列されたテキストファイルです.

NgWordsFilterEn(dict_path: str | os.PathLike[str], *args: Any, **kwargs: Any)
413    def __init__(self, dict_path: str | PathLike[str], *args: Any, **kwargs: Any) -> None:
414        super().__init__(*args, **kwargs)
415
416        with open(dict_path, encoding="utf-8") as fp:
417            ng_words = fp.read().split("\n")
418        ng_words = [re.escape(w.strip()) for w in ng_words if not len(w) == 0]
419        pat = "|".join(ng_words)
420        # 英語のパターンにマッチするようにしている, \s[単語]\s や [単語]. [単語], などにマッチ.
421        self.keyword_pat = re.compile(rf"(?:^| )({pat})(?:( |,|\.)|$)", re.IGNORECASE)

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
423    def apply(self, doc: Document) -> Document:
424        if self.keyword_pat.search(doc.text):
425            doc.is_rejected = True
426        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class DiscardAdultContentJa(NgWordsFilterJa):
429class DiscardAdultContentJa(NgWordsFilterJa):
430    """
431    日本語のアダルトキーワード(および不適切語)を含む文書を破棄します.
432    `dict_path` で指定したファイルから, キーワードのリストを得ます.
433    ファイルは単語が改行で羅列されたテキストファイルです.
434    デフォルトの`dict_path` は /hojichar/dict/adult_keywords_ja.txt です.
435    """
436
437    def __init__(
438        self,
439        dict_path: str | PathLike[str] = BASE_PATH / "dict/adult_keywords_ja.txt",
440        *args: Any,
441        **kwargs: Any,
442    ) -> None:
443        super().__init__(dict_path, *args, **kwargs)
444
445    def apply(self, doc: Document) -> Document:
446        """
447        >>> DiscardAdultContentJa().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
448        True
449
450        >>> DiscardAdultContentJa().apply(Document("ほうじ茶")).is_rejected
451        False
452
453        挙動は正しいが誤検知しているケース. 他にも, サック in リュックサック,
454        >>> DiscardAdultContentJa().apply(Document("アスパラガス")).is_rejected \
455        # Matching with NG keyword "アス"
456        True
457        """
458        return super().apply(doc)

日本語のアダルトキーワード(および不適切語)を含む文書を破棄します. dict_path で指定したファイルから, キーワードのリストを得ます. ファイルは単語が改行で羅列されたテキストファイルです. デフォルトのdict_path は /hojichar/dict/adult_keywords_ja.txt です.

DiscardAdultContentJa( dict_path: str | os.PathLike[str] = PosixPath('/home/runner/work/HojiChar/HojiChar/hojichar/dict/adult_keywords_ja.txt'), *args: Any, **kwargs: Any)
437    def __init__(
438        self,
439        dict_path: str | PathLike[str] = BASE_PATH / "dict/adult_keywords_ja.txt",
440        *args: Any,
441        **kwargs: Any,
442    ) -> None:
443        super().__init__(dict_path, *args, **kwargs)

Args: dict_path: Path to the NG word list, one word per line. ignore_confused: Match katakana words only where no katakana precedes or follows. args: max_allowed_num: The document is removed if the words match more than this number of times. Defaults to 0, which removes the document on the first match. min_matches_per_1k_chars: The document is removed only if there are also at least this many matches per 1,000 characters. Defaults to 0.0, which decides by max_allowed_num alone. *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
445    def apply(self, doc: Document) -> Document:
446        """
447        >>> DiscardAdultContentJa().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
448        True
449
450        >>> DiscardAdultContentJa().apply(Document("ほうじ茶")).is_rejected
451        False
452
453        挙動は正しいが誤検知しているケース. 他にも, サック in リュックサック,
454        >>> DiscardAdultContentJa().apply(Document("アスパラガス")).is_rejected \
455        # Matching with NG keyword "アス"
456        True
457        """
458        return super().apply(doc)
>>> DiscardAdultContentJa().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
True
>>> DiscardAdultContentJa().apply(Document("ほうじ茶")).is_rejected
False

挙動は正しいが誤検知しているケース. 他にも, サック in リュックサック,

>>> DiscardAdultContentJa().apply(Document("アスパラガス")).is_rejected         # Matching with NG keyword "アス"
True
class DiscardAdultContentEn(NgWordsFilterEn):
461class DiscardAdultContentEn(NgWordsFilterEn):
462    """
463    英語のアダルトキーワード(および不適切語)を含む文書を破棄します.
464    `dict_path` で指定したファイルから, キーワードのリストを得ます.
465    ファイルは単語が改行で羅列されたテキストファイルです.
466    デフォルトの`dict_path` は /hojichar/dict/adult_keywords_en.txt です.
467    """
468
469    def __init__(
470        self,
471        dict_path: str | PathLike[str] = BASE_PATH / "dict/adult_keywords_en.txt",
472        *args: Any,
473        **kwargs: Any,
474    ) -> None:
475        super().__init__(dict_path, *args, **kwargs)
476
477    def apply(self, doc: Document) -> Document:
478        """
479        >>> DiscardAdultContentEn().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
480        True
481
482        >>> DiscardAdultContentEn().apply(Document("hojichar")).is_rejected
483        False
484        """
485        return super().apply(doc)

英語のアダルトキーワード(および不適切語)を含む文書を破棄します. dict_path で指定したファイルから, キーワードのリストを得ます. ファイルは単語が改行で羅列されたテキストファイルです. デフォルトのdict_path は /hojichar/dict/adult_keywords_en.txt です.

DiscardAdultContentEn( dict_path: str | os.PathLike[str] = PosixPath('/home/runner/work/HojiChar/HojiChar/hojichar/dict/adult_keywords_en.txt'), *args: Any, **kwargs: Any)
469    def __init__(
470        self,
471        dict_path: str | PathLike[str] = BASE_PATH / "dict/adult_keywords_en.txt",
472        *args: Any,
473        **kwargs: Any,
474    ) -> None:
475        super().__init__(dict_path, *args, **kwargs)

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
477    def apply(self, doc: Document) -> Document:
478        """
479        >>> DiscardAdultContentEn().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
480        True
481
482        >>> DiscardAdultContentEn().apply(Document("hojichar")).is_rejected
483        False
484        """
485        return super().apply(doc)
>>> DiscardAdultContentEn().apply(Document("<TEST_STRING_OF_ADULT_KEYWORD>")).is_rejected
True
>>> DiscardAdultContentEn().apply(Document("hojichar")).is_rejected
False
class DiscardDiscriminationContentJa(NgWordsFilterJa):
488class DiscardDiscriminationContentJa(NgWordsFilterJa):
489    """
490    日本語の差別キーワード(および不適切語)を含む文書を破棄します.
491    `dict_path` で指定したファイルから, キーワードのリストを得ます.
492    ファイルは単語が改行で羅列されたテキストファイルです.
493    デフォルトの`dict_path` は /hojichar/dict/discrimination_keywords_ja.txt です.
494    """
495
496    def __init__(
497        self,
498        dict_path: str | PathLike[str] = BASE_PATH / "dict/discrimination_keywords_ja.txt",
499        *args: Any,
500        **kwargs: Any,
501    ):
502        super().__init__(dict_path, *args, **kwargs)
503
504    def apply(self, doc: Document) -> Document:
505        """
506        >>> DiscardDiscriminationContentJa().\
507            apply(Document("<TEST_STRING_OF_DISCRIMINATION_KEYWORD>")).is_rejected
508        True
509
510        >>> DiscardDiscriminationContentJa().apply(Document("ほうじ茶")).is_rejected
511        False
512        """
513        return super().apply(doc)

日本語の差別キーワード(および不適切語)を含む文書を破棄します. dict_path で指定したファイルから, キーワードのリストを得ます. ファイルは単語が改行で羅列されたテキストファイルです. デフォルトのdict_path は /hojichar/dict/discrimination_keywords_ja.txt です.

DiscardDiscriminationContentJa( dict_path: str | os.PathLike[str] = PosixPath('/home/runner/work/HojiChar/HojiChar/hojichar/dict/discrimination_keywords_ja.txt'), *args: Any, **kwargs: Any)
496    def __init__(
497        self,
498        dict_path: str | PathLike[str] = BASE_PATH / "dict/discrimination_keywords_ja.txt",
499        *args: Any,
500        **kwargs: Any,
501    ):
502        super().__init__(dict_path, *args, **kwargs)

Args: dict_path: Path to the NG word list, one word per line. ignore_confused: Match katakana words only where no katakana precedes or follows. args: max_allowed_num: The document is removed if the words match more than this number of times. Defaults to 0, which removes the document on the first match. min_matches_per_1k_chars: The document is removed only if there are also at least this many matches per 1,000 characters. Defaults to 0.0, which decides by max_allowed_num alone. *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
504    def apply(self, doc: Document) -> Document:
505        """
506        >>> DiscardDiscriminationContentJa().\
507            apply(Document("<TEST_STRING_OF_DISCRIMINATION_KEYWORD>")).is_rejected
508        True
509
510        >>> DiscardDiscriminationContentJa().apply(Document("ほうじ茶")).is_rejected
511        False
512        """
513        return super().apply(doc)
>>> DiscardDiscriminationContentJa().            apply(Document("<TEST_STRING_OF_DISCRIMINATION_KEYWORD>")).is_rejected
True
>>> DiscardDiscriminationContentJa().apply(Document("ほうじ茶")).is_rejected
False
class DiscardViolenceContentJa(NgWordsFilterJa):
516class DiscardViolenceContentJa(NgWordsFilterJa):
517    """
518    日本語の暴力・脅迫を示唆するキーワードを含む文書を破棄します.
519    `dict_path` で指定したファイルから, キーワードのリストを得ます.
520    ファイルは単語が改行で羅列されたテキストファイルです.
521    デフォルトの`dict_path` は /hojichar/dict/violence_keywords_ja.txt です.
522    """
523
524    def __init__(
525        self,
526        dict_path: str | PathLike[str] = BASE_PATH / "dict/violence_keywords_ja.txt",
527        *args: Any,
528        **kwargs: Any,
529    ) -> None:
530        super().__init__(dict_path, *args, **kwargs)
531
532    def apply(self, doc: Document) -> Document:
533        """
534        >>> DiscardViolenceContentJa()\
535            .apply(Document("<TEST_STRING_OF_VIOLENCE_KEYWORD>")).is_rejected
536        True
537
538        >>> DiscardViolenceContentJa().apply(Document("ほうじ茶")).is_rejected
539        False
540        """
541        return super().apply(doc)

日本語の暴力・脅迫を示唆するキーワードを含む文書を破棄します. dict_path で指定したファイルから, キーワードのリストを得ます. ファイルは単語が改行で羅列されたテキストファイルです. デフォルトのdict_path は /hojichar/dict/violence_keywords_ja.txt です.

DiscardViolenceContentJa( dict_path: str | os.PathLike[str] = PosixPath('/home/runner/work/HojiChar/HojiChar/hojichar/dict/violence_keywords_ja.txt'), *args: Any, **kwargs: Any)
524    def __init__(
525        self,
526        dict_path: str | PathLike[str] = BASE_PATH / "dict/violence_keywords_ja.txt",
527        *args: Any,
528        **kwargs: Any,
529    ) -> None:
530        super().__init__(dict_path, *args, **kwargs)

Args: dict_path: Path to the NG word list, one word per line. ignore_confused: Match katakana words only where no katakana precedes or follows. args: max_allowed_num: The document is removed if the words match more than this number of times. Defaults to 0, which removes the document on the first match. min_matches_per_1k_chars: The document is removed only if there are also at least this many matches per 1,000 characters. Defaults to 0.0, which decides by max_allowed_num alone. *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
532    def apply(self, doc: Document) -> Document:
533        """
534        >>> DiscardViolenceContentJa()\
535            .apply(Document("<TEST_STRING_OF_VIOLENCE_KEYWORD>")).is_rejected
536        True
537
538        >>> DiscardViolenceContentJa().apply(Document("ほうじ茶")).is_rejected
539        False
540        """
541        return super().apply(doc)
>>> DiscardViolenceContentJa()            .apply(Document("<TEST_STRING_OF_VIOLENCE_KEYWORD>")).is_rejected
True
>>> DiscardViolenceContentJa().apply(Document("ほうじ茶")).is_rejected
False
class DiscardBBSComments(hojichar.core.filter_interface.Filter):
544class DiscardBBSComments(Filter):
545    """
546    正規表現 "BBS Pattern" に `max_allowed_num` 回よりたくさんマッチする文書を破棄します.
547    `max_allowed_num` のデフォルト値は14です.
548
549    マッチ数は文書が長いほど増えるため, 固定の回数だけで判定すると, 日付や「ページ」「質問」を
550    含む長い文書 (学校だより・官公庁の PDF など) が長さだけを理由に破棄されます.
551    `min_matches_per_1k_chars` を指定すると, 1,000 字あたりのマッチ数がこの値以上の場合にだけ
552    破棄します. デフォルト値は 0.0 で, マッチ数だけで判定します.
553
554    正規表現 "BBS Pattern" は下記のリンクで検証可能です. ただし, リンク先は末尾に `|-` を含む
555    以前のパターンです. この `-` は, Markdown の表の区切り行やハイフンでつないだ語に含まれる
556    すべてのハイフンを1回のマッチとして数えていたため, 取り除きました.
557    https://regex101.com/r/ybQvL2/1
558    """
559
560    def __init__(
561        self,
562        max_allowed_num: int = 14,
563        *args: Any,
564        min_matches_per_1k_chars: float = 0.0,
565        **kwargs: Any,
566    ) -> None:
567        """
568        Args:
569            max_allowed_num: The document is removed if the pattern matches more than this
570                number of times.
571            *args:
572            min_matches_per_1k_chars: The document is removed only if there are also at least
573                this many matches per 1,000 characters. Defaults to 0.0, which decides by
574                `max_allowed_num` alone.
575            **kwargs:
576        """
577        super().__init__(*args, **kwargs)
578
579        self.max_allowed_num = max_allowed_num
580        self.min_matches_per_1k_chars = min_matches_per_1k_chars
581        self.keyword_pat = re.compile(
582            r"\d{4}[年\.\-\/][\ ]*\d{1,2}[月\.\-\/][\ ]*\d{1,2}[日]*|コメント|SOLD OUT|レビュー|投稿|ページ|\([月火水木金土日]\)|質問|\d+話|楽天市場"  # noqa
583        )
584
585    def apply(self, doc: Document) -> Document:
586        """
587        >>> DiscardBBSComments().apply(Document("楽天市場 質問 投稿 コメント レビュー "*3)).is_rejected
588        True
589
590        >>> DiscardBBSComments().apply(Document("鏡餅")).is_rejected
591        False
592
593        >>> DiscardBBSComments().apply(Document("| a | b |\\n| --- | --- |\\n" * 5)).is_rejected
594        False
595        """
596        match = _find_excessive_match(
597            self.keyword_pat, doc.text, self.max_allowed_num, self.min_matches_per_1k_chars
598        )
599        if match is not None:
600            doc.is_rejected = True
601        return doc

正規表現 "BBS Pattern" に max_allowed_num 回よりたくさんマッチする文書を破棄します. max_allowed_num のデフォルト値は14です.

マッチ数は文書が長いほど増えるため, 固定の回数だけで判定すると, 日付や「ページ」「質問」を 含む長い文書 (学校だより・官公庁の PDF など) が長さだけを理由に破棄されます. min_matches_per_1k_chars を指定すると, 1,000 字あたりのマッチ数がこの値以上の場合にだけ 破棄します. デフォルト値は 0.0 で, マッチ数だけで判定します.

正規表現 "BBS Pattern" は下記のリンクで検証可能です. ただし, リンク先は末尾に |- を含む 以前のパターンです. この - は, Markdown の表の区切り行やハイフンでつないだ語に含まれる すべてのハイフンを1回のマッチとして数えていたため, 取り除きました. https://regex101.com/r/ybQvL2/1

DiscardBBSComments( max_allowed_num: int = 14, *args: Any, min_matches_per_1k_chars: float = 0.0, **kwargs: Any)
560    def __init__(
561        self,
562        max_allowed_num: int = 14,
563        *args: Any,
564        min_matches_per_1k_chars: float = 0.0,
565        **kwargs: Any,
566    ) -> None:
567        """
568        Args:
569            max_allowed_num: The document is removed if the pattern matches more than this
570                number of times.
571            *args:
572            min_matches_per_1k_chars: The document is removed only if there are also at least
573                this many matches per 1,000 characters. Defaults to 0.0, which decides by
574                `max_allowed_num` alone.
575            **kwargs:
576        """
577        super().__init__(*args, **kwargs)
578
579        self.max_allowed_num = max_allowed_num
580        self.min_matches_per_1k_chars = min_matches_per_1k_chars
581        self.keyword_pat = re.compile(
582            r"\d{4}[年\.\-\/][\ ]*\d{1,2}[月\.\-\/][\ ]*\d{1,2}[日]*|コメント|SOLD OUT|レビュー|投稿|ページ|\([月火水木金土日]\)|質問|\d+話|楽天市場"  # noqa
583        )

Args: max_allowed_num: The document is removed if the pattern matches more than this number of times. args: min_matches_per_1k_chars: The document is removed only if there are also at least this many matches per 1,000 characters. Defaults to 0.0, which decides by max_allowed_num alone. *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
585    def apply(self, doc: Document) -> Document:
586        """
587        >>> DiscardBBSComments().apply(Document("楽天市場 質問 投稿 コメント レビュー "*3)).is_rejected
588        True
589
590        >>> DiscardBBSComments().apply(Document("鏡餅")).is_rejected
591        False
592
593        >>> DiscardBBSComments().apply(Document("| a | b |\\n| --- | --- |\\n" * 5)).is_rejected
594        False
595        """
596        match = _find_excessive_match(
597            self.keyword_pat, doc.text, self.max_allowed_num, self.min_matches_per_1k_chars
598        )
599        if match is not None:
600            doc.is_rejected = True
601        return doc
>>> DiscardBBSComments().apply(Document("楽天市場 質問 投稿 コメント レビュー "*3)).is_rejected
True
>>> DiscardBBSComments().apply(Document("鏡餅")).is_rejected
False
>>> DiscardBBSComments().apply(Document("| a | b |\n| --- | --- |\n" * 5)).is_rejected
False
class DiscardAds(hojichar.core.filter_interface.Filter):
604class DiscardAds(Filter):
605    """
606    主に広告キーワードを`max_allowed_num`より多く含む文書を破棄します.
607    デフォルトで`max_allowed_num` は14です.
608    `dict_path` で指定したファイルから, 広告キーワードのリストを得ます.
609    ファイルは単語が改行で羅列されたテキストファイルです.
610    デフォルトの`dict_path` は /hojichar/dict/advertisement_keywords_ja.txt です.
611
612    `DiscardBBSComments` と同じく, `min_matches_per_1k_chars` を指定すると, 1,000 字あたりの
613    マッチ数がこの値以上の場合にだけ破棄します. 「お問い合わせ」などの語を含む長い文書が
614    長さだけを理由に破棄されるのを防ぎます. デフォルト値は 0.0 で, マッチ数だけで判定します.
615    """
616
617    def __init__(
618        self,
619        dict_path: str | PathLike[str] = BASE_PATH / "dict/advertisement_keywords_ja.txt",
620        max_allowed_num: int = 14,
621        *args: Any,
622        min_matches_per_1k_chars: float = 0.0,
623        **kwargs: Any,
624    ):
625        """
626        Args:
627            dict_path: Path to the keyword list, one keyword per line.
628            max_allowed_num: The document is removed if the keywords match more than this
629                number of times.
630            *args:
631            min_matches_per_1k_chars: The document is removed only if there are also at least
632                this many matches per 1,000 characters. Defaults to 0.0, which decides by
633                `max_allowed_num` alone.
634            **kwargs:
635        """
636        super().__init__(*args, **kwargs)
637
638        self.max_allow_num = max_allowed_num
639        self.min_matches_per_1k_chars = min_matches_per_1k_chars
640        with open(dict_path, encoding="utf-8") as fp:
641            ng_words = fp.read().split("\n")
642        ng_words = [re.escape(w.strip()) for w in ng_words if not len(w) == 0]
643        pat = r"|".join(ng_words)
644        self.keyword_pat = re.compile(pat)
645
646    def apply(self, doc: Document) -> Document:
647        """
648        >>> DiscardAds().apply(Document("お問い合わせください 営業時間 よくある質問"*5)).is_rejected
649        True
650
651        >>> DiscardAds().apply(Document("おはよう")).is_rejected
652        False
653        """
654        match = _find_excessive_match(
655            self.keyword_pat, doc.text, self.max_allow_num, self.min_matches_per_1k_chars
656        )
657        if match is not None:
658            doc.is_rejected = True
659        return doc

主に広告キーワードをmax_allowed_numより多く含む文書を破棄します. デフォルトでmax_allowed_num は14です. dict_path で指定したファイルから, 広告キーワードのリストを得ます. ファイルは単語が改行で羅列されたテキストファイルです. デフォルトのdict_path は /hojichar/dict/advertisement_keywords_ja.txt です.

DiscardBBSComments と同じく, min_matches_per_1k_chars を指定すると, 1,000 字あたりの マッチ数がこの値以上の場合にだけ破棄します. 「お問い合わせ」などの語を含む長い文書が 長さだけを理由に破棄されるのを防ぎます. デフォルト値は 0.0 で, マッチ数だけで判定します.

DiscardAds( dict_path: str | os.PathLike[str] = PosixPath('/home/runner/work/HojiChar/HojiChar/hojichar/dict/advertisement_keywords_ja.txt'), max_allowed_num: int = 14, *args: Any, min_matches_per_1k_chars: float = 0.0, **kwargs: Any)
617    def __init__(
618        self,
619        dict_path: str | PathLike[str] = BASE_PATH / "dict/advertisement_keywords_ja.txt",
620        max_allowed_num: int = 14,
621        *args: Any,
622        min_matches_per_1k_chars: float = 0.0,
623        **kwargs: Any,
624    ):
625        """
626        Args:
627            dict_path: Path to the keyword list, one keyword per line.
628            max_allowed_num: The document is removed if the keywords match more than this
629                number of times.
630            *args:
631            min_matches_per_1k_chars: The document is removed only if there are also at least
632                this many matches per 1,000 characters. Defaults to 0.0, which decides by
633                `max_allowed_num` alone.
634            **kwargs:
635        """
636        super().__init__(*args, **kwargs)
637
638        self.max_allow_num = max_allowed_num
639        self.min_matches_per_1k_chars = min_matches_per_1k_chars
640        with open(dict_path, encoding="utf-8") as fp:
641            ng_words = fp.read().split("\n")
642        ng_words = [re.escape(w.strip()) for w in ng_words if not len(w) == 0]
643        pat = r"|".join(ng_words)
644        self.keyword_pat = re.compile(pat)

Args: dict_path: Path to the keyword list, one keyword per line. max_allowed_num: The document is removed if the keywords match more than this number of times. args: min_matches_per_1k_chars: The document is removed only if there are also at least this many matches per 1,000 characters. Defaults to 0.0, which decides by max_allowed_num alone. *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
646    def apply(self, doc: Document) -> Document:
647        """
648        >>> DiscardAds().apply(Document("お問い合わせください 営業時間 よくある質問"*5)).is_rejected
649        True
650
651        >>> DiscardAds().apply(Document("おはよう")).is_rejected
652        False
653        """
654        match = _find_excessive_match(
655            self.keyword_pat, doc.text, self.max_allow_num, self.min_matches_per_1k_chars
656        )
657        if match is not None:
658            doc.is_rejected = True
659        return doc
>>> DiscardAds().apply(Document("お問い合わせください 営業時間 よくある質問"*5)).is_rejected
True
>>> DiscardAds().apply(Document("おはよう")).is_rejected
False
class AcceptJapanese(hojichar.core.filter_interface.Filter):
662class AcceptJapanese(Filter):
663    """
664    日本語でないドキュメントを破棄します. 日本語判定は次の手順で行われます.
665        1. テキストを左から`lookup_size` (デフォルトで50字) 参照し,
666        ひらがな・カタカナが存在すれば日本語と判定する.
667    """
668
669    def __init__(self, lookup_size: int = 50, *args: Any, **kwargs: Any) -> None:
670        super().__init__(*args, **kwargs)
671
672        self.lookup_size = lookup_size
673        self.hiragana_katakana_pat = re.compile(r"[ぁ-んァ-ン]")
674
675    def apply(self, doc: Document) -> Document:
676        """
677        >>> AcceptJapanese().apply(Document("This is English document")).is_rejected
678        True
679
680        >>> AcceptJapanese().apply(Document("a"*50 + "あ")).is_rejected
681        True
682
683        >>> AcceptJapanese().apply(Document("ほうじ茶")).is_rejected
684        False
685        """
686        if not self.hiragana_katakana_pat.search(doc.text[: self.lookup_size]):
687            doc.is_rejected = True
688        return doc

日本語でないドキュメントを破棄します. 日本語判定は次の手順で行われます. 1. テキストを左からlookup_size (デフォルトで50字) 参照し, ひらがな・カタカナが存在すれば日本語と判定する.

AcceptJapanese(lookup_size: int = 50, *args: Any, **kwargs: Any)
669    def __init__(self, lookup_size: int = 50, *args: Any, **kwargs: Any) -> None:
670        super().__init__(*args, **kwargs)
671
672        self.lookup_size = lookup_size
673        self.hiragana_katakana_pat = re.compile(r"[ぁ-んァ-ン]")

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
675    def apply(self, doc: Document) -> Document:
676        """
677        >>> AcceptJapanese().apply(Document("This is English document")).is_rejected
678        True
679
680        >>> AcceptJapanese().apply(Document("a"*50 + "あ")).is_rejected
681        True
682
683        >>> AcceptJapanese().apply(Document("ほうじ茶")).is_rejected
684        False
685        """
686        if not self.hiragana_katakana_pat.search(doc.text[: self.lookup_size]):
687            doc.is_rejected = True
688        return doc
>>> AcceptJapanese().apply(Document("This is English document")).is_rejected
True
>>> AcceptJapanese().apply(Document("a"*50 + "あ")).is_rejected
True
>>> AcceptJapanese().apply(Document("ほうじ茶")).is_rejected
False
class DiscardRareKuten(hojichar.core.filter_interface.Filter):
691class DiscardRareKuten(Filter):
692    """
693    日本語でないドキュメントを破棄します. 日本語判定は次の手順で行われます
694    ドキュメントを句点"。"で区切り, 平均文長が
695    `max_avarage_sentence_length` より長い場合は破棄します.
696    `max_avarage_sentence_length` のデフォルト値は100です.
697    このフィルタは, 文章中の句点の割合が少なすぎるドキュメントを破棄します.
698    """
699
700    def __init__(self, max_average_sentence_length: int = 100, *args: Any, **kwargs: Any) -> None:
701        super().__init__(*args, **kwargs)
702
703        self.max_average_sentence_length = max_average_sentence_length
704        self.kuten_pat = re.compile(r"。")
705
706    def apply(self, doc: Document) -> Document:
707        """
708        >>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよ。")).is_rejected
709        False
710        >>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよう。")).is_rejected
711        True
712        """
713        kuten_lst = self.kuten_pat.findall(doc.text)
714        min_kuten_num = len(doc.text) / self.max_average_sentence_length
715        if len(kuten_lst) < min_kuten_num:
716            doc.is_rejected = True
717        return doc

日本語でないドキュメントを破棄します. 日本語判定は次の手順で行われます ドキュメントを句点"。"で区切り, 平均文長が max_avarage_sentence_length より長い場合は破棄します. max_avarage_sentence_length のデフォルト値は100です. このフィルタは, 文章中の句点の割合が少なすぎるドキュメントを破棄します.

DiscardRareKuten(max_average_sentence_length: int = 100, *args: Any, **kwargs: Any)
700    def __init__(self, max_average_sentence_length: int = 100, *args: Any, **kwargs: Any) -> None:
701        super().__init__(*args, **kwargs)
702
703        self.max_average_sentence_length = max_average_sentence_length
704        self.kuten_pat = re.compile(r"。")

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
706    def apply(self, doc: Document) -> Document:
707        """
708        >>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよ。")).is_rejected
709        False
710        >>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよう。")).is_rejected
711        True
712        """
713        kuten_lst = self.kuten_pat.findall(doc.text)
714        min_kuten_num = len(doc.text) / self.max_average_sentence_length
715        if len(kuten_lst) < min_kuten_num:
716            doc.is_rejected = True
717        return doc
>>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよ。")).is_rejected
False
>>> DiscardRareKuten(max_average_sentence_length=4).apply(Document("おはよう。")).is_rejected
True
class HeaderFooterTagsRemover(hojichar.core.filter_interface.Filter):
720class HeaderFooterTagsRemover(Filter):
721    """
722    ドキュメントの冒頭・末尾のトークンを調査し, ヘッダー・フッダー的な
723    タグが存在していた場合, そのトークンを除去します.
724
725    このフィルタを通す前に, 事前にセンテンスレベルにトーカナイズしておいてください.
726    このフィルタでは Document.token にのみ変更が加えられるので, 出力前 あるいは 下流フィルタで
727    Document.text に変更を加える前にトークンをマージしておいてください.
728    """
729
730    def __init__(
731        self,
732        dict_path: str | PathLike[str] = BASE_PATH / "dict/header_footer_keywords_ja.txt",
733        *args: Any,
734        **kwargs: Any,
735    ) -> None:
736        super().__init__(*args, **kwargs)
737
738        with open(dict_path) as fp:
739            keywords = fp.read().split("\n")
740        keywords = [re.escape(w.strip()) for w in keywords if not len(w) == 0]
741        self.keyword_pat = re.compile(r"|".join(keywords))
742
743    def apply(self, doc: Document) -> Document:
744        if len(doc.tokens) == 0:
745            return doc
746
747        lookup_size = 0
748        if 1 <= len(doc.tokens) < 4:
749            lookup_size = 1
750        elif 4 <= len(doc.tokens) < 6:
751            lookup_size = 2
752        elif 6 <= len(doc.tokens):
753            lookup_size = 3
754
755        for i in range(lookup_size):
756            if self.should_drop_token(doc.tokens[i]):
757                doc.tokens[i].is_rejected = True
758            if self.should_drop_token(doc.tokens[-(i + 1)]):
759                doc.tokens[i].is_rejected = True
760
761        return doc
762
763    def should_drop_token(self, token: Token) -> bool:
764        """
765        >>> HeaderFooterTagsRemover().should_drop_token(Token("<TEST_STRING_OF_KEYWORD>"))
766        True
767
768        >>> HeaderFooterTagsRemover().should_drop_token(Token("ほうじ茶"))
769        False
770
771        Comment.
772        Original legacy code removed a pattern r"« _ | Main | _ »" .
773        In the pattern, "|" is not escaped, so **ANY** string was eliminated.
774        It seems unintended behavior, so I fix this.
775        """
776        if self.keyword_pat.match(token.text):
777            return True
778        else:
779            return False

ドキュメントの冒頭・末尾のトークンを調査し, ヘッダー・フッダー的な タグが存在していた場合, そのトークンを除去します.

このフィルタを通す前に, 事前にセンテンスレベルにトーカナイズしておいてください. このフィルタでは Document.token にのみ変更が加えられるので, 出力前 あるいは 下流フィルタで Document.text に変更を加える前にトークンをマージしておいてください.

HeaderFooterTagsRemover( dict_path: str | os.PathLike[str] = PosixPath('/home/runner/work/HojiChar/HojiChar/hojichar/dict/header_footer_keywords_ja.txt'), *args: Any, **kwargs: Any)
730    def __init__(
731        self,
732        dict_path: str | PathLike[str] = BASE_PATH / "dict/header_footer_keywords_ja.txt",
733        *args: Any,
734        **kwargs: Any,
735    ) -> None:
736        super().__init__(*args, **kwargs)
737
738        with open(dict_path) as fp:
739            keywords = fp.read().split("\n")
740        keywords = [re.escape(w.strip()) for w in keywords if not len(w) == 0]
741        self.keyword_pat = re.compile(r"|".join(keywords))

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
743    def apply(self, doc: Document) -> Document:
744        if len(doc.tokens) == 0:
745            return doc
746
747        lookup_size = 0
748        if 1 <= len(doc.tokens) < 4:
749            lookup_size = 1
750        elif 4 <= len(doc.tokens) < 6:
751            lookup_size = 2
752        elif 6 <= len(doc.tokens):
753            lookup_size = 3
754
755        for i in range(lookup_size):
756            if self.should_drop_token(doc.tokens[i]):
757                doc.tokens[i].is_rejected = True
758            if self.should_drop_token(doc.tokens[-(i + 1)]):
759                doc.tokens[i].is_rejected = True
760
761        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

def should_drop_token(self, token: hojichar.core.models.Token) -> bool:
763    def should_drop_token(self, token: Token) -> bool:
764        """
765        >>> HeaderFooterTagsRemover().should_drop_token(Token("<TEST_STRING_OF_KEYWORD>"))
766        True
767
768        >>> HeaderFooterTagsRemover().should_drop_token(Token("ほうじ茶"))
769        False
770
771        Comment.
772        Original legacy code removed a pattern r"« _ | Main | _ »" .
773        In the pattern, "|" is not escaped, so **ANY** string was eliminated.
774        It seems unintended behavior, so I fix this.
775        """
776        if self.keyword_pat.match(token.text):
777            return True
778        else:
779            return False
>>> HeaderFooterTagsRemover().should_drop_token(Token("<TEST_STRING_OF_KEYWORD>"))
True
>>> HeaderFooterTagsRemover().should_drop_token(Token("ほうじ茶"))
False

Comment. Original legacy code removed a pattern r"« _ | Main | _ »" . In the pattern, "|" is not escaped, so ANY string was eliminated. It seems unintended behavior, so I fix this.

class MaskPersonalInformation(hojichar.core.filter_interface.Filter):
782class MaskPersonalInformation(Filter):
783    """
784    ドキュメントに含まれる電話番号・電子メールアドレスを一部マスキングします.
785
786    電子メールアドレスのローカル部 ("@" の前) は, RFC 5321 の上限である 64 字までを対象にします.
787    "@" の前に英数字や記号が 64 字を超えて続く場合は, "@" の直前の 64 字からマスクします.
788    以前は上限がなく, 英数字や記号が長く続くテキスト (特に "@" を多く含むもの) では,
789    処理時間がその長さの2乗に比例していました.
790    """
791
792    def __init__(self, *args: Any, **kwargs: Any) -> None:
793        super().__init__(*args, **kwargs)
794
795        self.phone_pat = re.compile(
796            r"((0|\+\d{1,3}[- ]?)(\d{2}[- ]?\d{4}[- ]?|\d[- ]?\d{4}[- ]?|\d{2}[- ]?\d{3}[- ]?|\d{3}[- ]?\d{2}[- ]?|\d{4}[- ]?\d{1}[- ]?))\d{4}"  # noqa
797        )
798        # The local part is bounded to 64 characters, the limit of RFC 5321. Unbounded, a long
799        # run of these characters was rescanned from every position in it, which is quadratic
800        # in the run length. With the bound, at most 64 starting positions reach each "@", so
801        # the scan is linear in the text length.
802        self.email_pat = re.compile(
803            r"[a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~.]{1,64}@[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+(\.[A-Za-z0-9\-]+)"  # noqa
804        )
805
806    def apply(self, doc: Document) -> Document:
807        """
808        >>> MaskPersonalInformation()('06-1234-5678')
809        '06-1234-XXXX'
810        >>> MaskPersonalInformation()('075-123-4567')
811        '075-123-XXXX'
812        >>> MaskPersonalInformation()('0166-12-3456')
813        '0166-12-XXXX'
814        >>> MaskPersonalInformation()('09808-1-2345')
815        '09808-1-XXXX'
816        >>> MaskPersonalInformation()('090-1234-5678')
817        '090-1234-XXXX'
818        >>> MaskPersonalInformation()('0751234567')
819        '075123XXXX'
820        >>> MaskPersonalInformation()('08012345678')
821        '0801234XXXX'
822        >>> MaskPersonalInformation()('連絡は075-123-4567 まで')
823        '連絡は075-123-XXXX まで'
824        >>> MaskPersonalInformation()('+81-80-1234-5678')
825        '+81-80-1234-XXXX'
826        >>> MaskPersonalInformation()('+818012345678')
827        '+81801234XXXX'
828        >>> MaskPersonalInformation()('hogehoge@example.com')
829        'xxxx@yyy.com'
830        >>> MaskPersonalInformation()('何かあれば hogehoge@example.ne.jp まで連絡')
831        '何かあれば xxxx@yyy.jp まで連絡'
832        >>> MaskPersonalInformation()('a' * 70 + '@example.com')
833        'aaaaaaxxxx@yyy.com'
834        """
835        text = self.phone_pat.sub(r"\1XXXX", doc.text)
836        if "@" in text:
837            text = self.email_pat.sub(r"xxxx@yyy\1", text)
838        doc.text = text
839        return doc

ドキュメントに含まれる電話番号・電子メールアドレスを一部マスキングします.

電子メールアドレスのローカル部 ("@" の前) は, RFC 5321 の上限である 64 字までを対象にします. "@" の前に英数字や記号が 64 字を超えて続く場合は, "@" の直前の 64 字からマスクします. 以前は上限がなく, 英数字や記号が長く続くテキスト (特に "@" を多く含むもの) では, 処理時間がその長さの2乗に比例していました.

MaskPersonalInformation(*args: Any, **kwargs: Any)
792    def __init__(self, *args: Any, **kwargs: Any) -> None:
793        super().__init__(*args, **kwargs)
794
795        self.phone_pat = re.compile(
796            r"((0|\+\d{1,3}[- ]?)(\d{2}[- ]?\d{4}[- ]?|\d[- ]?\d{4}[- ]?|\d{2}[- ]?\d{3}[- ]?|\d{3}[- ]?\d{2}[- ]?|\d{4}[- ]?\d{1}[- ]?))\d{4}"  # noqa
797        )
798        # The local part is bounded to 64 characters, the limit of RFC 5321. Unbounded, a long
799        # run of these characters was rescanned from every position in it, which is quadratic
800        # in the run length. With the bound, at most 64 starting positions reach each "@", so
801        # the scan is linear in the text length.
802        self.email_pat = re.compile(
803            r"[a-zA-Z0-9!#$%&'*+\-/=?^_`{|}~.]{1,64}@[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+(\.[A-Za-z0-9\-]+)"  # noqa
804        )

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
806    def apply(self, doc: Document) -> Document:
807        """
808        >>> MaskPersonalInformation()('06-1234-5678')
809        '06-1234-XXXX'
810        >>> MaskPersonalInformation()('075-123-4567')
811        '075-123-XXXX'
812        >>> MaskPersonalInformation()('0166-12-3456')
813        '0166-12-XXXX'
814        >>> MaskPersonalInformation()('09808-1-2345')
815        '09808-1-XXXX'
816        >>> MaskPersonalInformation()('090-1234-5678')
817        '090-1234-XXXX'
818        >>> MaskPersonalInformation()('0751234567')
819        '075123XXXX'
820        >>> MaskPersonalInformation()('08012345678')
821        '0801234XXXX'
822        >>> MaskPersonalInformation()('連絡は075-123-4567 まで')
823        '連絡は075-123-XXXX まで'
824        >>> MaskPersonalInformation()('+81-80-1234-5678')
825        '+81-80-1234-XXXX'
826        >>> MaskPersonalInformation()('+818012345678')
827        '+81801234XXXX'
828        >>> MaskPersonalInformation()('hogehoge@example.com')
829        'xxxx@yyy.com'
830        >>> MaskPersonalInformation()('何かあれば hogehoge@example.ne.jp まで連絡')
831        '何かあれば xxxx@yyy.jp まで連絡'
832        >>> MaskPersonalInformation()('a' * 70 + '@example.com')
833        'aaaaaaxxxx@yyy.com'
834        """
835        text = self.phone_pat.sub(r"\1XXXX", doc.text)
836        if "@" in text:
837            text = self.email_pat.sub(r"xxxx@yyy\1", text)
838        doc.text = text
839        return doc
>>> MaskPersonalInformation()('06-1234-5678')
'06-1234-XXXX'
>>> MaskPersonalInformation()('075-123-4567')
'075-123-XXXX'
>>> MaskPersonalInformation()('0166-12-3456')
'0166-12-XXXX'
>>> MaskPersonalInformation()('09808-1-2345')
'09808-1-XXXX'
>>> MaskPersonalInformation()('090-1234-5678')
'090-1234-XXXX'
>>> MaskPersonalInformation()('0751234567')
'075123XXXX'
>>> MaskPersonalInformation()('08012345678')
'0801234XXXX'
>>> MaskPersonalInformation()('連絡は075-123-4567 まで')
'連絡は075-123-XXXX まで'
>>> MaskPersonalInformation()('+81-80-1234-5678')
'+81-80-1234-XXXX'
>>> MaskPersonalInformation()('+818012345678')
'+81801234XXXX'
>>> MaskPersonalInformation()('hogehoge@example.com')
'xxxx@yyy.com'
>>> MaskPersonalInformation()('何かあれば hogehoge@example.ne.jp まで連絡')
'何かあれば xxxx@yyy.jp まで連絡'
>>> MaskPersonalInformation()('a' * 70 + '@example.com')
'aaaaaaxxxx@yyy.com'
class DiscardTooManyNouns(hojichar.core.filter_interface.Filter):
842class DiscardTooManyNouns(Filter):
843    """
844    [!CAUTION] This filter requires `fugashi` package. Please install it
845    by `pip install 'hojichar[all]'`.
846
847    A filter that removes document with too many nouns in Japanese i.e.,
848    documents such as advertisement, word salad, etc ...
849    """
850
851    def __init__(
852        self, threshold: float = 0.80, max_parse_chars: int = 100_000, *args: Any, **kwargs: Any
853    ) -> None:
854        """
855        Args:
856            threshold: document whose noun ratio is higher than this value will be discarded
857            max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document.
858            *args:
859            **kwargs:
860        """
861        super().__init__(*args, **kwargs)
862        assert is_loaded_extras, (
863            "fugashi is required for this filter. Try pip install 'hojichar[all]'"
864        )
865
866        self.threshold = threshold
867        self.max_parse_chars = max_parse_chars
868        self.tagger = Tagger("-Owakati")
869        assert "unidic" in self.tagger.dictionary_info[0]["filename"], (
870            "MeCab dictionary must be unidic"
871        )
872
873    def _chunk_text(self, text: str) -> Iterable[str]:
874        """Slice text into chunks of `max_parse_chars` length."""
875        step = self.max_parse_chars
876        for i in range(0, len(text), step):
877            yield text[i : i + step]
878
879    def apply(self, doc: Document) -> Document:
880        """
881        >>> DiscardTooManyNouns().apply(Document("自然言語処理大好き!")).is_rejected
882        False
883        >>> DiscardTooManyNouns().apply(Document("リンゴ・オレンジ・ミカン・バナナ セール中")).is_rejected
884        True
885        >>> DiscardTooManyNouns().apply(Document("今日の仙台朝市ではリンゴがセール中")).is_rejected
886        False
887        """
888        # remove "補助記号" from part-of-speech statistics
889        # because they often decrease the noun ratio,
890        # e.g., the sentence "リンゴ・オレンジ・バナナ・" has 補助記号 ratio of 0.5
891        # however, we don't want such sentence
892
893        pos_count: Counter[str] = Counter()
894        for chunk in self._chunk_text(doc.text):
895            for word in self.tagger(chunk):
896                if word.feature.pos1 != "補助記号":
897                    pos_count[word.feature.pos1] += 1
898
899        try:
900            noun_ratio = pos_count["名詞"] / sum(pos_count.values())
901        except ZeroDivisionError:
902            noun_ratio = 0.0
903        if noun_ratio >= self.threshold:
904            doc.is_rejected = True
905        return doc

[!CAUTION] This filter requires fugashi package. Please install it by pip install 'hojichar[all]'.

A filter that removes document with too many nouns in Japanese i.e., documents such as advertisement, word salad, etc ...

DiscardTooManyNouns( threshold: float = 0.8, max_parse_chars: int = 100000, *args: Any, **kwargs: Any)
851    def __init__(
852        self, threshold: float = 0.80, max_parse_chars: int = 100_000, *args: Any, **kwargs: Any
853    ) -> None:
854        """
855        Args:
856            threshold: document whose noun ratio is higher than this value will be discarded
857            max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document.
858            *args:
859            **kwargs:
860        """
861        super().__init__(*args, **kwargs)
862        assert is_loaded_extras, (
863            "fugashi is required for this filter. Try pip install 'hojichar[all]'"
864        )
865
866        self.threshold = threshold
867        self.max_parse_chars = max_parse_chars
868        self.tagger = Tagger("-Owakati")
869        assert "unidic" in self.tagger.dictionary_info[0]["filename"], (
870            "MeCab dictionary must be unidic"
871        )

Args: threshold: document whose noun ratio is higher than this value will be discarded max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document. args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
879    def apply(self, doc: Document) -> Document:
880        """
881        >>> DiscardTooManyNouns().apply(Document("自然言語処理大好き!")).is_rejected
882        False
883        >>> DiscardTooManyNouns().apply(Document("リンゴ・オレンジ・ミカン・バナナ セール中")).is_rejected
884        True
885        >>> DiscardTooManyNouns().apply(Document("今日の仙台朝市ではリンゴがセール中")).is_rejected
886        False
887        """
888        # remove "補助記号" from part-of-speech statistics
889        # because they often decrease the noun ratio,
890        # e.g., the sentence "リンゴ・オレンジ・バナナ・" has 補助記号 ratio of 0.5
891        # however, we don't want such sentence
892
893        pos_count: Counter[str] = Counter()
894        for chunk in self._chunk_text(doc.text):
895            for word in self.tagger(chunk):
896                if word.feature.pos1 != "補助記号":
897                    pos_count[word.feature.pos1] += 1
898
899        try:
900            noun_ratio = pos_count["名詞"] / sum(pos_count.values())
901        except ZeroDivisionError:
902            noun_ratio = 0.0
903        if noun_ratio >= self.threshold:
904            doc.is_rejected = True
905        return doc
>>> DiscardTooManyNouns().apply(Document("自然言語処理大好き!")).is_rejected
False
>>> DiscardTooManyNouns().apply(Document("リンゴ・オレンジ・ミカン・バナナ セール中")).is_rejected
True
>>> DiscardTooManyNouns().apply(Document("今日の仙台朝市ではリンゴがセール中")).is_rejected
False
class CharRepetitionRatioFilter(hojichar.core.filter_interface.Filter):
908class CharRepetitionRatioFilter(Filter):
909    """
910    文字Ngramの重なり率(文書中で高頻度文字Ngramが占める割合)を計算して, 重なりの大きいものを除去します.
911    名詞の連続からなるような広告テキストを取り除くのに有効です.
912
913    実装は, BigScience で採用されていた前処理を参考にしています.
914    元実装: https://github.com/bigscience-workshop/data-preparation/blob/9d0588419073cc5bf0fb92b58f37f2a1016572c3/preprocessing/training/01b_oscar_cleaning_and_filtering/filtering.py#L425-L453  # noqa: E501
915
916    「高頻度文字Ngram」は、sqrt(ユニークなNgramの総数)によって求めていますが,
917    これは文書長の影響を軽減するためだとされています.
918
919    掲示板のテキストが引っかかりやすい傾向があります.
920    13: 名無しさん@実況で競馬板アウト 2019/08/18(日) 15:28:46.10 ID:eBvZg8h+0
921    的なものが高頻度で登場するため、文字Ngramの重なり率も高くなってしまう
922    """
923
924    def __init__(
925        self, threshold: float = 0.33, ngram_size: int = 5, *args: Any, **kwargs: Any
926    ) -> None:
927        """
928
929        Args:
930            threshold: document with character repetition ratio higher than this value will be discarded
931            ngram_size: character ngram size. Larger value will decrease the false positive of long documents
932            *args:
933            **kwargs:
934        """  # noqa: E501
935
936        super().__init__(*args, **kwargs)
937        self.threshold = threshold
938        self.ngram_size = ngram_size
939
940    def apply(self, doc: Document) -> Document:
941        ratio = self.compute_character_repetition_ratio(doc.text, self.ngram_size)
942        if ratio >= self.threshold:
943            doc.is_rejected = True
944        return doc
945
946    @staticmethod
947    def compute_character_repetition_ratio(
948        document: str, character_repetition_length: int
949    ) -> float:
950        def get_freq_character_ngrams(document: str, n: int) -> dict[str, int]:
951            character_ngrams: list[str] = [
952                document[i : i + n] for i in range(len(document) - n + 1)
953            ]
954            freq_character_ngrams_dict: dict[str, int] = {}
955            for character_ngram in character_ngrams:
956                freq_character_ngrams_dict[character_ngram] = (
957                    freq_character_ngrams_dict.get(character_ngram, 0) + 1
958                )
959            return freq_character_ngrams_dict
960
961        freq_character_ngrams_dict = get_freq_character_ngrams(
962            document, character_repetition_length
963        )
964        if len(freq_character_ngrams_dict) == 0:
965            return 0.0
966        freq_character_ngrams: list[int] = list(freq_character_ngrams_dict.values())
967        freq_character_ngrams = sorted(freq_character_ngrams, reverse=True)
968        val_one = len([el for el in freq_character_ngrams if el == 1])
969        num_rep_character_ngrams = min(
970            int(np.sqrt(len(freq_character_ngrams))),
971            len(freq_character_ngrams) - val_one,
972        )
973        character_repetition_ratio = sum(freq_character_ngrams[:num_rep_character_ngrams]) / sum(
974            freq_character_ngrams
975        )
976        return character_repetition_ratio

文字Ngramの重なり率(文書中で高頻度文字Ngramが占める割合)を計算して, 重なりの大きいものを除去します. 名詞の連続からなるような広告テキストを取り除くのに有効です.

実装は, BigScience で採用されていた前処理を参考にしています. 元実装: https://github.com/bigscience-workshop/data-preparation/blob/9d0588419073cc5bf0fb92b58f37f2a1016572c3/preprocessing/training/01b_oscar_cleaning_and_filtering/filtering.py#L425-L453 # noqa: E501

「高頻度文字Ngram」は、sqrt(ユニークなNgramの総数)によって求めていますが, これは文書長の影響を軽減するためだとされています.

掲示板のテキストが引っかかりやすい傾向があります. 13: 名無しさん@実況で競馬板アウト 2019/08/18(日) 15:28:46.10 ID:eBvZg8h+0 的なものが高頻度で登場するため、文字Ngramの重なり率も高くなってしまう

CharRepetitionRatioFilter( threshold: float = 0.33, ngram_size: int = 5, *args: Any, **kwargs: Any)
924    def __init__(
925        self, threshold: float = 0.33, ngram_size: int = 5, *args: Any, **kwargs: Any
926    ) -> None:
927        """
928
929        Args:
930            threshold: document with character repetition ratio higher than this value will be discarded
931            ngram_size: character ngram size. Larger value will decrease the false positive of long documents
932            *args:
933            **kwargs:
934        """  # noqa: E501
935
936        super().__init__(*args, **kwargs)
937        self.threshold = threshold
938        self.ngram_size = ngram_size

Args: threshold: document with character repetition ratio higher than this value will be discarded ngram_size: character ngram size. Larger value will decrease the false positive of long documents args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
940    def apply(self, doc: Document) -> Document:
941        ratio = self.compute_character_repetition_ratio(doc.text, self.ngram_size)
942        if ratio >= self.threshold:
943            doc.is_rejected = True
944        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

@staticmethod
def compute_character_repetition_ratio(document: str, character_repetition_length: int) -> float:
946    @staticmethod
947    def compute_character_repetition_ratio(
948        document: str, character_repetition_length: int
949    ) -> float:
950        def get_freq_character_ngrams(document: str, n: int) -> dict[str, int]:
951            character_ngrams: list[str] = [
952                document[i : i + n] for i in range(len(document) - n + 1)
953            ]
954            freq_character_ngrams_dict: dict[str, int] = {}
955            for character_ngram in character_ngrams:
956                freq_character_ngrams_dict[character_ngram] = (
957                    freq_character_ngrams_dict.get(character_ngram, 0) + 1
958                )
959            return freq_character_ngrams_dict
960
961        freq_character_ngrams_dict = get_freq_character_ngrams(
962            document, character_repetition_length
963        )
964        if len(freq_character_ngrams_dict) == 0:
965            return 0.0
966        freq_character_ngrams: list[int] = list(freq_character_ngrams_dict.values())
967        freq_character_ngrams = sorted(freq_character_ngrams, reverse=True)
968        val_one = len([el for el in freq_character_ngrams if el == 1])
969        num_rep_character_ngrams = min(
970            int(np.sqrt(len(freq_character_ngrams))),
971            len(freq_character_ngrams) - val_one,
972        )
973        character_repetition_ratio = sum(freq_character_ngrams[:num_rep_character_ngrams]) / sum(
974            freq_character_ngrams
975        )
976        return character_repetition_ratio
class WordRepetitionRatioFilter(hojichar.core.filter_interface.Filter):
 979class WordRepetitionRatioFilter(Filter):
 980    """
 981    [!CAUTION] This filter requires `fugashi` package. Please install it
 982    by `pip install 'hojichar[all]'`.
 983
 984    単語Ngramの重なり率(文書中で重複する単語Ngramが占める割合)を計算して、重なりの大きいものを弾くためのフィルタ.
 985    BigScienceで採用されていた前処理を参考にしている.
 986
 987    名詞が連打されているような広告テキストを取り除くのに有効な様子
 988    まともな文書がたまたま2回繰り返されている場合もあり、これを取り除いて良いのかは分からない
 989    例:
 990    "ウェブ\n本文: ニコンの上昇率16%超える、今3月期は経常76%の大幅増益見込む(ニコン) 2013年05月10日[minkabu PRESS] - みんなの株式 (みんかぶ)\n2013/05/10(10:57)
 991    ニコン<7731.T>が急騰、寄り付き直後に前日比355円高の2537円まで買い上げ
 992    られ、上昇率は16%を超えた。外国為替市場で円が1ドル100円台、1ユーロ131円台に入るなど急速に円安が進み、輸出株が軒並み高になる
 993    なか、9日取引終了後に発表した前年3月期決算で、今3月期は2ケタ近い増収で大幅増益を見込んだことが買い気を強めさせた。連結売上
 994    高は前期比9.8%増の1兆1100億円、経常利益75.8%増の850億円を予想。前期は半導体、電子部品の低迷が足かせになり、2ケタ増収ながら
 995    経常46%の大幅減益になったが、レンズ交換式デジタルカメラの拡大や液晶ディスプレイの回復で収益が急回復する。ニコンの株価は10時
 996    56分現在2491円(△309円)出所:株経通信(株式会社みんかぶ)\n2013/05/10 - ニコン(7731) の関連ニュース。 ニコン<7731.T>が急騰、寄
 997    り付き直後に前日比355円高の2537円まで買い上げられ、上昇率は16%を超えた。外国為替市場で円が1ドル100円台、1ユーロ131円台に入
 998    るなど急速に円安が進み、輸出株が軒並み高になるなか、9日取引終了後に発表した前年3月期決算で、今3月期は2ケタ近い増収で大幅増
 999    益を見込んだことが買い気を強めさせた。連結売上高は前期比9.8%増の1兆1100億円、経常利益75.8%増の850億円を予想。前期は半導体、
1000    電子部品の低迷が足かせになり、2ケタ増収ながら経常46%の大幅減益になったが、レンズ交換式デジタルカメラの拡大や液晶ディスプレ
1001    イの回復で収益が急回"
1002    """  # noqa: E501
1003
1004    def __init__(
1005        self,
1006        threshold: float = 0.40,
1007        ngram_size: int = 7,
1008        max_parse_chars: int = 100_000,
1009        *args: Any,
1010        **kwargs: Any,
1011    ) -> None:
1012        """
1013
1014        Args:
1015            threshold: document whose character repetition ratio is higher than this value will be discarded
1016            ngram_size: character ngram size. Larger value will decrease the false positive of long documents
1017            max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document.
1018            *args:
1019            **kwargs:
1020        """  # noqa: E501
1021        super().__init__(*args, **kwargs)
1022        assert is_loaded_extras, (
1023            "fugashi is required for this filter. Try pip install 'hojichar[all]'"
1024        )
1025
1026        self.threshold = threshold
1027        self.ngram_size = ngram_size
1028        self.max_parse_chars = max_parse_chars
1029        self.tagger = Tagger("-Owakati")
1030
1031    def _chunk_text(self, text: str) -> Iterable[str]:
1032        """Split text into chunks of `max_parse_chars` length."""
1033        step = self.max_parse_chars
1034        for i in range(0, len(text), step):
1035            yield text[i : i + step]
1036
1037    def _get_freq_word_ngrams(self, words: list[str], n: int) -> dict[str, int]:
1038        freq: dict[str, int] = {}
1039        if n <= 0 or len(words) < n:
1040            return freq
1041        for i in range(len(words) - n + 1):
1042            key = " ".join(words[i : i + n])
1043            freq[key] = freq.get(key, 0) + 1
1044        return freq
1045
1046    def apply(self, doc: Document) -> Document:
1047        ratio = self.compute_word_repetition_ratio(doc.text, self.ngram_size)
1048        if ratio >= self.threshold:
1049            doc.is_rejected = True
1050        return doc
1051
1052    def compute_word_repetition_ratio(self, document: str, n: int) -> float:
1053        total_counter: Counter[str] = Counter()
1054
1055        for chunk in self._chunk_text(document):
1056            words = [w.surface for w in self.tagger(chunk)]
1057            total_counter.update(self._get_freq_word_ngrams(words, n))
1058
1059        if not total_counter:
1060            return 0.0
1061
1062        total = sum(total_counter.values())
1063        repeated = sum(v for v in total_counter.values() if v > 1)
1064        return repeated / total

[!CAUTION] This filter requires fugashi package. Please install it by pip install 'hojichar[all]'.

単語Ngramの重なり率(文書中で重複する単語Ngramが占める割合)を計算して、重なりの大きいものを弾くためのフィルタ.
BigScienceで採用されていた前処理を参考にしている.

名詞が連打されているような広告テキストを取り除くのに有効な様子
まともな文書がたまたま2回繰り返されている場合もあり、これを取り除いて良いのかは分からない
例:
"ウェブ

本文: ニコンの上昇率16%超える、今3月期は経常76%の大幅増益見込む(ニコン) 2013年05月10日[minkabu PRESS] - みんなの株式 (みんかぶ) 2013/05/10(10:57) ニコン<7731.T>が急騰、寄り付き直後に前日比355円高の2537円まで買い上げ られ、上昇率は16%を超えた。外国為替市場で円が1ドル100円台、1ユーロ131円台に入るなど急速に円安が進み、輸出株が軒並み高になる なか、9日取引終了後に発表した前年3月期決算で、今3月期は2ケタ近い増収で大幅増益を見込んだことが買い気を強めさせた。連結売上 高は前期比9.8%増の1兆1100億円、経常利益75.8%増の850億円を予想。前期は半導体、電子部品の低迷が足かせになり、2ケタ増収ながら 経常46%の大幅減益になったが、レンズ交換式デジタルカメラの拡大や液晶ディスプレイの回復で収益が急回復する。ニコンの株価は10時 56分現在2491円(△309円)出所:株経通信(株式会社みんかぶ) 2013/05/10 - ニコン(7731) の関連ニュース。 ニコン<7731.T>が急騰、寄 り付き直後に前日比355円高の2537円まで買い上げられ、上昇率は16%を超えた。外国為替市場で円が1ドル100円台、1ユーロ131円台に入 るなど急速に円安が進み、輸出株が軒並み高になるなか、9日取引終了後に発表した前年3月期決算で、今3月期は2ケタ近い増収で大幅増 益を見込んだことが買い気を強めさせた。連結売上高は前期比9.8%増の1兆1100億円、経常利益75.8%増の850億円を予想。前期は半導体、 電子部品の低迷が足かせになり、2ケタ増収ながら経常46%の大幅減益になったが、レンズ交換式デジタルカメラの拡大や液晶ディスプレ イの回復で収益が急回"

WordRepetitionRatioFilter( threshold: float = 0.4, ngram_size: int = 7, max_parse_chars: int = 100000, *args: Any, **kwargs: Any)
1004    def __init__(
1005        self,
1006        threshold: float = 0.40,
1007        ngram_size: int = 7,
1008        max_parse_chars: int = 100_000,
1009        *args: Any,
1010        **kwargs: Any,
1011    ) -> None:
1012        """
1013
1014        Args:
1015            threshold: document whose character repetition ratio is higher than this value will be discarded
1016            ngram_size: character ngram size. Larger value will decrease the false positive of long documents
1017            max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document.
1018            *args:
1019            **kwargs:
1020        """  # noqa: E501
1021        super().__init__(*args, **kwargs)
1022        assert is_loaded_extras, (
1023            "fugashi is required for this filter. Try pip install 'hojichar[all]'"
1024        )
1025
1026        self.threshold = threshold
1027        self.ngram_size = ngram_size
1028        self.max_parse_chars = max_parse_chars
1029        self.tagger = Tagger("-Owakati")

Args: threshold: document whose character repetition ratio is higher than this value will be discarded ngram_size: character ngram size. Larger value will decrease the false positive of long documents max_parse_chars: maximum number of characters to parse in the document. Too large value may cause segmentation fault parsing the document. args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
1046    def apply(self, doc: Document) -> Document:
1047        ratio = self.compute_word_repetition_ratio(doc.text, self.ngram_size)
1048        if ratio >= self.threshold:
1049            doc.is_rejected = True
1050        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

def compute_word_repetition_ratio(self, document: str, n: int) -> float:
1052    def compute_word_repetition_ratio(self, document: str, n: int) -> float:
1053        total_counter: Counter[str] = Counter()
1054
1055        for chunk in self._chunk_text(document):
1056            words = [w.surface for w in self.tagger(chunk)]
1057            total_counter.update(self._get_freq_word_ngrams(words, n))
1058
1059        if not total_counter:
1060            return 0.0
1061
1062        total = sum(total_counter.values())
1063        repeated = sum(v for v in total_counter.values() if v > 1)
1064        return repeated / total
class DiscardTooManySpecialToken(hojichar.core.filter_interface.Filter):
1067class DiscardTooManySpecialToken(Filter):
1068    """
1069    [!CAUTION] This filter requires `emoji` package. Please install it
1070    by `pip install 'hojichar[all]'`.
1071
1072    句読点を含む記号、空白、絵文字、その他特殊な文字を一定の割合以上含むような文書を取り除くためのフィルタ
1073    元実装: BigScience https://github.com/bigscience-workshop/data-preparation/blob/9d0588419073cc5bf0fb92b58f37f2a1016572c3/preprocessing/training/01b_oscar_cleaning_and_filtering/parameters_filtering.py#L5-L16  # noqa: E501
1074    """
1075
1076    def __init__(self, threshold: float = 0.4, *args: Any, **kwargs: Any) -> None:
1077        """
1078
1079        Args:
1080            threshold: document whose special token ratio is higher than this value will be discarded
1081            *args:
1082            **kwargs:
1083        """  # noqa: E501
1084        super().__init__(*args, **kwargs)
1085
1086        # digits are not regarded as special tokens
1087        # otherwise many false positives are made, i.e., good documents discarded
1088        main_special_characters = string.punctuation + string.whitespace  # + string.digits
1089        other_special_characters = (
1090            "’ “— ™ – •‘œ    ˜ ‚ƒ„’“”–▬…✦�­£​•€«»°·═"
1091            "×士^˘⇓()§″′´¿−±∈¢ø‚„½¼¾¹²³―⁃,ˌ¸‹›ʺˈʻ¦‐⠀‰……‑≤≥‖"
1092            "◆●■►▼▲▴∆▻¡★☆✱ːº。¯˜¥ɪ≈†:⁄♡✓⊕․.⋅÷1‟;،、¨ाাी्े◦˚"
1093            "゜ʼ≖ʼ¤℃√!?【】‿∞➤~πه۩☛₨➩☻๑٪♥ıॽ《‘©﴿٬?▷Г♫∟™ª₪®「—❖"
1094            "」﴾》�"
1095        )
1096
1097        en_emoji = emoji.EMOJI_DATA.keys()
1098
1099        special_characters_default = set(main_special_characters + other_special_characters)
1100        special_characters_default.update(en_emoji)
1101        self.special_characters = special_characters_default
1102
1103        self.threshold = threshold
1104
1105    def _compute_special_characters_ratio(self, text: str) -> float:
1106        if len(text) == 0:
1107            return 0
1108
1109        special_characters_ratio = len(
1110            [char for char in text if char in self.special_characters]
1111        ) / len(text)
1112        return special_characters_ratio
1113
1114    def apply(self, doc: Document) -> Document:
1115        special_characters_ratio = self._compute_special_characters_ratio(doc.text)
1116
1117        if special_characters_ratio > self.threshold:
1118            doc.is_rejected = True
1119        return doc

[!CAUTION] This filter requires emoji package. Please install it by pip install 'hojichar[all]'.

句読点を含む記号、空白、絵文字、その他特殊な文字を一定の割合以上含むような文書を取り除くためのフィルタ 元実装: BigScience https://github.com/bigscience-workshop/data-preparation/blob/9d0588419073cc5bf0fb92b58f37f2a1016572c3/preprocessing/training/01b_oscar_cleaning_and_filtering/parameters_filtering.py#L5-L16 # noqa: E501

DiscardTooManySpecialToken(threshold: float = 0.4, *args: Any, **kwargs: Any)
1076    def __init__(self, threshold: float = 0.4, *args: Any, **kwargs: Any) -> None:
1077        """
1078
1079        Args:
1080            threshold: document whose special token ratio is higher than this value will be discarded
1081            *args:
1082            **kwargs:
1083        """  # noqa: E501
1084        super().__init__(*args, **kwargs)
1085
1086        # digits are not regarded as special tokens
1087        # otherwise many false positives are made, i.e., good documents discarded
1088        main_special_characters = string.punctuation + string.whitespace  # + string.digits
1089        other_special_characters = (
1090            "’ “— ™ – •‘œ    ˜ ‚ƒ„’“”–▬…✦�­£​•€«»°·═"
1091            "×士^˘⇓()§″′´¿−±∈¢ø‚„½¼¾¹²³―⁃,ˌ¸‹›ʺˈʻ¦‐⠀‰……‑≤≥‖"
1092            "◆●■►▼▲▴∆▻¡★☆✱ːº。¯˜¥ɪ≈†:⁄♡✓⊕․.⋅÷1‟;،、¨ाাी्े◦˚"
1093            "゜ʼ≖ʼ¤℃√!?【】‿∞➤~πه۩☛₨➩☻๑٪♥ıॽ《‘©﴿٬?▷Г♫∟™ª₪®「—❖"
1094            "」﴾》�"
1095        )
1096
1097        en_emoji = emoji.EMOJI_DATA.keys()
1098
1099        special_characters_default = set(main_special_characters + other_special_characters)
1100        special_characters_default.update(en_emoji)
1101        self.special_characters = special_characters_default
1102
1103        self.threshold = threshold

Args: threshold: document whose special token ratio is higher than this value will be discarded args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
1114    def apply(self, doc: Document) -> Document:
1115        special_characters_ratio = self._compute_special_characters_ratio(doc.text)
1116
1117        if special_characters_ratio > self.threshold:
1118            doc.is_rejected = True
1119        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class SingleCharacterRepetitionFilter(hojichar.core.filter_interface.Filter):
1122class SingleCharacterRepetitionFilter(Filter):
1123    """
1124    単一文字が大量に繰り返されているような文書を取り除くためのフィルタ
1125    そのような文書はノイズである可能性が高いため
1126    参考: BigScienceプロジェクトによると、oscarデータセットの中にバックスラッシュだけを2M個含むような文書が含まれていたらしい
1127    https://github.com/bigscience-workshop/bigscience/blob/master/train/tr8-104B-wide/chronicles.md#2m-backslash-only-samples-in-our-dataset  # noqa: E501
1128
1129    デフォルトでは文書の長さにかかわらず, `threshold` 文字以上の繰り返しが1か所でもあれば破棄します.
1130    そのため, 長い文書 (例えば 200,000 字のエージェントの実行ログ) にテスト結果の区切り線 "=====" が
1131    1行あるだけでも破棄されます. `min_ratio` を指定すると, 繰り返しの長さが文書長に対して
1132    `min_ratio` 以上の割合を占める場合にだけ破棄します.
1133
1134    >>> doc = "テストの実行ログです。\\n" * 2000 + "=" * 200
1135    >>> SingleCharacterRepetitionFilter().apply(Document(doc)).is_rejected
1136    True
1137    >>> SingleCharacterRepetitionFilter(min_ratio=0.1).apply(Document(doc)).is_rejected
1138    False
1139    """
1140
1141    def __init__(
1142        self,
1143        threshold: int = 200,
1144        *args: Any,
1145        min_ratio: float = 0.0,
1146        **kwargs: Any,
1147    ) -> None:
1148        """
1149        Args:
1150            threshold: The document is removed if character is repeated for this value or more
1151            *args:
1152            min_ratio: The document is removed only if the repeated run is also at least this
1153                fraction of the document length. Defaults to 0.0, which decides by `threshold`
1154                alone.
1155            **kwargs:
1156        """
1157        super().__init__(*args, **kwargs)
1158        self.threshold = threshold
1159        self.min_ratio = min_ratio
1160
1161    def _is_repeat_contained(self, text: str) -> bool:
1162        min_run = max(self.threshold, self.min_ratio * len(text))
1163        groups = groupby(text)
1164        is_repeat_contained = any(sum(1 for _ in group) >= min_run for _, group in groups)
1165        return is_repeat_contained
1166
1167    def apply(self, doc: Document) -> Document:
1168        if self._is_repeat_contained(doc.text):
1169            doc.is_rejected = True
1170        return doc

単一文字が大量に繰り返されているような文書を取り除くためのフィルタ そのような文書はノイズである可能性が高いため 参考: BigScienceプロジェクトによると、oscarデータセットの中にバックスラッシュだけを2M個含むような文書が含まれていたらしい https://github.com/bigscience-workshop/bigscience/blob/master/train/tr8-104B-wide/chronicles.md#2m-backslash-only-samples-in-our-dataset # noqa: E501

デフォルトでは文書の長さにかかわらず, threshold 文字以上の繰り返しが1か所でもあれば破棄します. そのため, 長い文書 (例えば 200,000 字のエージェントの実行ログ) にテスト結果の区切り線 "=====" が 1行あるだけでも破棄されます. min_ratio を指定すると, 繰り返しの長さが文書長に対して min_ratio 以上の割合を占める場合にだけ破棄します.

>>> doc = "テストの実行ログです。\n" * 2000 + "=" * 200
>>> SingleCharacterRepetitionFilter().apply(Document(doc)).is_rejected
True
>>> SingleCharacterRepetitionFilter(min_ratio=0.1).apply(Document(doc)).is_rejected
False
SingleCharacterRepetitionFilter( threshold: int = 200, *args: Any, min_ratio: float = 0.0, **kwargs: Any)
1141    def __init__(
1142        self,
1143        threshold: int = 200,
1144        *args: Any,
1145        min_ratio: float = 0.0,
1146        **kwargs: Any,
1147    ) -> None:
1148        """
1149        Args:
1150            threshold: The document is removed if character is repeated for this value or more
1151            *args:
1152            min_ratio: The document is removed only if the repeated run is also at least this
1153                fraction of the document length. Defaults to 0.0, which decides by `threshold`
1154                alone.
1155            **kwargs:
1156        """
1157        super().__init__(*args, **kwargs)
1158        self.threshold = threshold
1159        self.min_ratio = min_ratio

Args: threshold: The document is removed if character is repeated for this value or more args: min_ratio: The document is removed only if the repeated run is also at least this fraction of the document length. Defaults to 0.0, which decides by threshold alone. *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
1167    def apply(self, doc: Document) -> Document:
1168        if self._is_repeat_contained(doc.text):
1169            doc.is_rejected = True
1170        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class DiscardTooManyEndingEllipsis(hojichar.core.filter_interface.Filter):
1173class DiscardTooManyEndingEllipsis(Filter):
1174    """
1175    ellipsisで終わるような行が大量に含まれるような文書を取り除くためのフィルタです.
1176    ellipsisとしては ... と … を用いている
1177    同様のフィルタが RedPajama v2で用いられています.
1178
1179    例として, 以下のような文書を検知します.
1180    ```
1181    ペアーズは女性、という驚愕の過食が出ているのをごアラサーですか。時代から付...
1182    バツイチアラフォー 婚活ち女性の特徴と子持な付...
1183    ```
1184
1185    デフォルトではしきい値を0.7としているが, これはC4から0.1%を削るような設定であり、
1186    precisionを重視した設定です.
1187    """
1188
1189    def __init__(
1190        self,
1191        threshold: float = 0.7,
1192        *args: Any,
1193        **kwargs: Any,
1194    ) -> None:
1195        """
1196        Args:
1197            threshold: The document is removed if ratio of lines ending with ellipsis is higher than this value
1198            *args:
1199            **kwargs:
1200        """  # noqa: E501
1201        super().__init__(*args, **kwargs)
1202        self.threshold = threshold
1203        self.ellipsis_pattern = re.compile(r"(\.{3}|…)\n")  # matches ...\n and …\n
1204
1205    def apply(self, doc: Document) -> Document:
1206        ellipsis_count = len(self.ellipsis_pattern.findall(doc.text))
1207        newline_count = max(doc.text.count("\n"), 1)  # avoid zero division
1208        ellipsis_ratio = ellipsis_count / newline_count
1209
1210        if ellipsis_ratio > self.threshold:
1211            doc.is_rejected = True
1212        return doc

ellipsisで終わるような行が大量に含まれるような文書を取り除くためのフィルタです. ellipsisとしては ... と … を用いている 同様のフィルタが RedPajama v2で用いられています.

例として, 以下のような文書を検知します.

ペアーズは女性、という驚愕の過食が出ているのをごアラサーですか。時代から付...
バツイチアラフォー 婚活ち女性の特徴と子持な付...

デフォルトではしきい値を0.7としているが, これはC4から0.1%を削るような設定であり、 precisionを重視した設定です.

DiscardTooManyEndingEllipsis(threshold: float = 0.7, *args: Any, **kwargs: Any)
1189    def __init__(
1190        self,
1191        threshold: float = 0.7,
1192        *args: Any,
1193        **kwargs: Any,
1194    ) -> None:
1195        """
1196        Args:
1197            threshold: The document is removed if ratio of lines ending with ellipsis is higher than this value
1198            *args:
1199            **kwargs:
1200        """  # noqa: E501
1201        super().__init__(*args, **kwargs)
1202        self.threshold = threshold
1203        self.ellipsis_pattern = re.compile(r"(\.{3}|…)\n")  # matches ...\n and …\n

Args: threshold: The document is removed if ratio of lines ending with ellipsis is higher than this value args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
1205    def apply(self, doc: Document) -> Document:
1206        ellipsis_count = len(self.ellipsis_pattern.findall(doc.text))
1207        newline_count = max(doc.text.count("\n"), 1)  # avoid zero division
1208        ellipsis_ratio = ellipsis_count / newline_count
1209
1210        if ellipsis_ratio > self.threshold:
1211            doc.is_rejected = True
1212        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class DiscardTooShortLines(hojichar.core.filter_interface.Filter):
1215class DiscardTooShortLines(Filter):
1216    """
1217    短い行を大量に含む文書を捨てるためのフィルタです.
1218
1219    メニューバーやパンくずリストのような要素を大量に含む文書を取り除くのに有効です.
1220    """
1221
1222    def __init__(self, threshold: float = 0.5, *args: Any, **kwargs: Any) -> None:
1223        """
1224        Args:
1225            threshold: The document is removed if the ratio of short (<10 chars) lines are more than this value.
1226            *args:
1227            **kwargs:
1228        """  # noqa: E501
1229        super().__init__(*args, **kwargs)
1230        self.threshold = threshold
1231        # この値は適当に決め打ち
1232        self.minimum_line_length = 10
1233
1234    def apply(self, doc: Document) -> Document:
1235        lines = [len(x) for x in doc.text.split("\n")]
1236        short_lines = [x for x in lines if x <= self.minimum_line_length]
1237        if (len(short_lines) / len(lines)) > self.threshold:
1238            doc.is_rejected = True
1239        return doc

短い行を大量に含む文書を捨てるためのフィルタです.

メニューバーやパンくずリストのような要素を大量に含む文書を取り除くのに有効です.

DiscardTooShortLines(threshold: float = 0.5, *args: Any, **kwargs: Any)
1222    def __init__(self, threshold: float = 0.5, *args: Any, **kwargs: Any) -> None:
1223        """
1224        Args:
1225            threshold: The document is removed if the ratio of short (<10 chars) lines are more than this value.
1226            *args:
1227            **kwargs:
1228        """  # noqa: E501
1229        super().__init__(*args, **kwargs)
1230        self.threshold = threshold
1231        # この値は適当に決め打ち
1232        self.minimum_line_length = 10

Args: threshold: The document is removed if the ratio of short (<10 chars) lines are more than this value. args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
1234    def apply(self, doc: Document) -> Document:
1235        lines = [len(x) for x in doc.text.split("\n")]
1236        short_lines = [x for x in lines if x <= self.minimum_line_length]
1237        if (len(short_lines) / len(lines)) > self.threshold:
1238            doc.is_rejected = True
1239        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

class CollapseRepeatedDocument(hojichar.core.filter_interface.Filter):
1242class CollapseRepeatedDocument(Filter):
1243    """
1244    文書全体が同じブロックの2回または3回の繰り返しになっている場合に, 1回分だけを残します.
1245
1246    テキスト抽出器がページを2回続けて出力した文書などを想定しています.
1247    内容は1回分なら問題ないため, このフィルタは文書を破棄せず, テキストを書き換えます.
1248
1249    コピーの間には `max_separator_len` 字までの区切りを許し, 区切りの中身は比べません.
1250    ブロックの前後の空白を除いて比べるため, 空白や改行だけの区切りは, これより長くても
1251    畳まれる場合があります. 短い定型句の繰り返しを畳まないように, 残すブロックは
1252    `min_block_len` 字以上とします. 文書の先頭と末尾 (3回の場合は中央も) を比べるだけなので,
1253    計算量は文書長に比例します. 畳んだ場合は, 文書の前後の空白も取り除かれます.
1254
1255    >>> block = "HojiChar はテキストの前処理パイプラインを組み立てるための Python ライブラリです。"
1256    >>> CollapseRepeatedDocument()(block + "\\n" + block) == block
1257    True
1258    >>> CollapseRepeatedDocument()(block + "\\n\\n" + block + "\\n\\n" + block + "\\n") == block
1259    True
1260    >>> CollapseRepeatedDocument()("短い文です。" * 2)
1261    '短い文です。短い文です。'
1262    """
1263
1264    def __init__(
1265        self,
1266        min_block_len: int = 40,
1267        max_separator_len: int = 3,
1268        *args: Any,
1269        **kwargs: Any,
1270    ) -> None:
1271        """
1272        Args:
1273            min_block_len: Minimum length of the block to keep. Shorter repeats are left as
1274                they are.
1275            max_separator_len: Maximum length of the text between copies. Its content is not
1276                compared.
1277            *args:
1278            **kwargs:
1279        """
1280        super().__init__(*args, **kwargs)
1281        self.min_block_len = min_block_len
1282        self.max_separator_len = max_separator_len
1283
1284    def apply(self, doc: Document) -> Document:
1285        text = doc.text.strip()
1286        n = len(text)
1287        if n < 2 * self.min_block_len:
1288            return doc
1289        for num_copies in (2, 3):
1290            for total_separator_len in range(self.max_separator_len * (num_copies - 1) + 1):
1291                block_len, remainder = divmod(n - total_separator_len, num_copies)
1292                if remainder or block_len < self.min_block_len:
1293                    continue
1294                block = text[:block_len].rstrip()
1295                if len(block) < self.min_block_len or block != text[n - block_len :].lstrip():
1296                    continue
1297                if num_copies == 3:
1298                    # Each separator has its own length; the middle copy need not be centered.
1299                    min_first_len = max(0, total_separator_len - self.max_separator_len)
1300                    max_first_len = min(self.max_separator_len, total_separator_len)
1301                    for first_len in range(min_first_len, max_first_len + 1):
1302                        middle_start = block_len + first_len
1303                        if block == text[middle_start : middle_start + block_len].strip():
1304                            break
1305                    else:
1306                        continue
1307                doc.text = block
1308                return doc
1309        return doc

文書全体が同じブロックの2回または3回の繰り返しになっている場合に, 1回分だけを残します.

テキスト抽出器がページを2回続けて出力した文書などを想定しています. 内容は1回分なら問題ないため, このフィルタは文書を破棄せず, テキストを書き換えます.

コピーの間には max_separator_len 字までの区切りを許し, 区切りの中身は比べません. ブロックの前後の空白を除いて比べるため, 空白や改行だけの区切りは, これより長くても 畳まれる場合があります. 短い定型句の繰り返しを畳まないように, 残すブロックは min_block_len 字以上とします. 文書の先頭と末尾 (3回の場合は中央も) を比べるだけなので, 計算量は文書長に比例します. 畳んだ場合は, 文書の前後の空白も取り除かれます.

>>> block = "HojiChar はテキストの前処理パイプラインを組み立てるための Python ライブラリです。"
>>> CollapseRepeatedDocument()(block + "\n" + block) == block
True
>>> CollapseRepeatedDocument()(block + "\n\n" + block + "\n\n" + block + "\n") == block
True
>>> CollapseRepeatedDocument()("短い文です。" * 2)
'短い文です。短い文です。'
CollapseRepeatedDocument( min_block_len: int = 40, max_separator_len: int = 3, *args: Any, **kwargs: Any)
1264    def __init__(
1265        self,
1266        min_block_len: int = 40,
1267        max_separator_len: int = 3,
1268        *args: Any,
1269        **kwargs: Any,
1270    ) -> None:
1271        """
1272        Args:
1273            min_block_len: Minimum length of the block to keep. Shorter repeats are left as
1274                they are.
1275            max_separator_len: Maximum length of the text between copies. Its content is not
1276                compared.
1277            *args:
1278            **kwargs:
1279        """
1280        super().__init__(*args, **kwargs)
1281        self.min_block_len = min_block_len
1282        self.max_separator_len = max_separator_len

Args: min_block_len: Minimum length of the block to keep. Shorter repeats are left as they are. max_separator_len: Maximum length of the text between copies. Its content is not compared. args: *kwargs:

def apply( self, doc: hojichar.core.models.Document) -> hojichar.core.models.Document:
1284    def apply(self, doc: Document) -> Document:
1285        text = doc.text.strip()
1286        n = len(text)
1287        if n < 2 * self.min_block_len:
1288            return doc
1289        for num_copies in (2, 3):
1290            for total_separator_len in range(self.max_separator_len * (num_copies - 1) + 1):
1291                block_len, remainder = divmod(n - total_separator_len, num_copies)
1292                if remainder or block_len < self.min_block_len:
1293                    continue
1294                block = text[:block_len].rstrip()
1295                if len(block) < self.min_block_len or block != text[n - block_len :].lstrip():
1296                    continue
1297                if num_copies == 3:
1298                    # Each separator has its own length; the middle copy need not be centered.
1299                    min_first_len = max(0, total_separator_len - self.max_separator_len)
1300                    max_first_len = min(self.max_separator_len, total_separator_len)
1301                    for first_len in range(min_first_len, max_first_len + 1):
1302                        middle_start = block_len + first_len
1303                        if block == text[middle_start : middle_start + block_len].strip():
1304                            break
1305                    else:
1306                        continue
1307                doc.text = block
1308                return doc
1309        return doc

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document