hojichar.core.filter_interface

  1import logging
  2from abc import ABC, abstractmethod
  3from collections.abc import Callable, Iterable, Sequence
  4from typing import Any, TypeVar
  5
  6import numpy as np
  7
  8from hojichar.core.models import Document, Statistics, Token, get_doc_info
  9from hojichar.utils.warn_deprecation import deprecated_since
 10
 11T = TypeVar("T")
 12
 13
 14def _is_jsonable(data: Any) -> bool:
 15    if data is None:
 16        return True
 17    elif isinstance(data, (bool, int, float, str)):
 18        return True
 19    return False
 20
 21
 22class Filter(ABC):
 23    """
 24    Base class for all filters.
 25    Document-level filters must inherit from this class.
 26
 27    The definition of text processing is in `apply` method.
 28    If you define a new filter, override the method.
 29
 30    When this class is called, apply the filter from string to string.
 31
 32    With context manager, you can use the filter as follows:
 33    ```python
 34    with YourFilter(p=0.5) as filt:
 35        text = filt("This is a sample text.")
 36    ```
 37
 38    """
 39
 40    def __init__(
 41        self,
 42        p: float = 1.0,
 43        skip_rejected: bool = True,
 44        *args: Any,
 45        random_state: int | np.random.Generator | None = None,
 46        use_batch: bool = False,
 47        batch_size: int = 128,
 48        **kwargs: Any,
 49    ) -> None:
 50        """
 51        Initialize the filter.
 52        Parameters
 53        ----------
 54        p : float
 55            The probability of applying the filter.
 56            If `p` is 1, the filter will always be applied.
 57        skip_rejected : bool
 58            If `True`, the filter will skip documents that are already rejected.
 59            If you want to apply the filter to all documents (e.g., postprocess), set this to `False`.
 60        random_state : int | np.random.Generator | None
 61            Seed for the random number generator.
 62            If `None`, a new random number generator will be created.
 63            If `None`, and use in the `Compose` class, the random state is shared with the `Compose` object.
 64        use_batch : bool
 65            If `True`, the filter will process documents in batches in the `apply_stream` method.
 66        batch_size : int
 67            The size of the batch to process documents in the `apply_stream` method.
 68        kwargs : Any
 69            Additional keyword arguments to pass to the filter.
 70        """
 71        self.name = self.__class__.__name__
 72        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
 73        assert 0 <= p <= 1
 74        self.p = p
 75        self.__init_rng(random_state)
 76        self.skip_rejected = skip_rejected
 77        self.use_batch = use_batch
 78        self.batch_size = batch_size
 79
 80        self._statistics: Statistics = Statistics()
 81
 82    @abstractmethod
 83    def apply(self, document: Document) -> Document:
 84        """
 85        Definition of filter behavior.
 86
 87        The document must have a protocol `TextContent`,
 88        and mostly used hojichar.Document class.
 89
 90        In this method, the filter will modify `document.text` or
 91        `document.extras` and set `document.is_rejected = True` to discard the document.
 92
 93        Parameters
 94        ----------
 95        document : Document
 96            Input document
 97
 98        Returns
 99        -------
100        Document
101            Processed Document
102        """
103
104    @deprecated_since(version="1.0.0", alternative="apply")
105    def apply_filter(self, document: Document) -> Document:
106        document = self.apply(document)
107        return document
108
109    def _check_skip(self, document: Document) -> bool:
110        """
111        Check if the document should be skipped by this filter.
112        If `skip_rejected` is set to `True`, this method will return `True`
113        if the document is already rejected.
114        If `p` is less than 1, this method will return `True` with a probability of `1 - p`.
115        """
116        skip = self.skip_rejected and document.is_rejected
117        if skip:
118            return True
119        if self.p < 1:
120            if self._rng.random() > self.p:
121                return True
122        return False
123
124    def _apply(self, document: Document) -> Document:
125        """
126        Apply the filter to a single document.
127        This method
128          - checks if the document should be skipped
129          - counts and logging the statistics
130          - logging the reason for rejection if the document is rejected
131
132        This method may be used in `apply` method of `Compose` class.
133        """
134
135        stats = get_doc_info(document)
136
137        if not self._check_skip(document):
138            document = self.apply(document)
139
140        new_stats = get_doc_info(document)
141        self._statistics.update_by_diff(stats, new_stats)
142
143        if not stats["is_rejected"] and new_stats["is_rejected"]:
144            document.reject_reason = self.get_jsonable_vars()
145
146        return document
147
148    def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
149        """
150        Apply the filter to a batch of documents.
151        You can override this method if you want to
152        apply the filter to a batch of documents at once.
153
154        This method may be used in `apply_batch` method of `Compose` class.
155
156        Parameters
157        ----------
158        documents : Sequence[Document]
159            List-like object of input documents
160
161        Returns
162        -------
163        list[Document]
164            List of processed documents
165        """
166        return [self.apply(document) for document in batch]
167
168    def _apply_batch(self, batch: Sequence[Document]) -> list[Document]:
169        """
170        Apply the filter to a batch of documents.
171        This method
172        - checks if the documents should be skipped
173        - counts and logs the statistics
174        - logs the reason for rejection if any document is rejected
175        """
176        skip = False
177        if self.p < 1:
178            skip = self._rng.random() > self.p
179
180        stats = [get_doc_info(document=doc) for doc in batch]
181        if not skip:
182            batch = self.apply_batch(batch)
183        batch = self._finalize_batch(batch, stats)
184        return batch
185
186    def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]:
187        """
188        Apply the filter to a stream of documents.
189        This method is used when you want to process documents one by one.
190        If `use_batch` is set to `True` in the constructor,
191        this method will process documents in batches using the `apply_batch` method.
192
193        Even if an exception occurs during processing, the process will continue, and the following actions will be taken:
194        - Set the `is_rejected` flag of the document to `True`
195        - Set the error details in `reject_reason`
196        - Increment the `errors` count in the statistics retrievable via `get_statistics`
197
198        Parameters
199        ----------
200        stream : Iterable[Document]
201            Stream of input documents
202
203        Returns
204        -------
205        Iterable[Document]
206            Stream of processed documents
207        """
208
209        if not self.use_batch:
210            for document in stream:
211                yield self._try_process(document, self._apply)
212        else:
213            batch: list[Document] = []
214            for document in stream:
215                if self._check_skip(document):
216                    yield document
217                    continue
218
219                batch.append(document)
220                if len(batch) >= self.batch_size:
221                    stats = [get_doc_info(doc) for doc in batch]
222                    batch = self._try_process(batch, self.apply_batch)
223                    batch = self._finalize_batch(batch, stats)
224                    yield from batch
225                    batch.clear()
226            if batch:
227                stats = [get_doc_info(doc) for doc in batch]
228                batch = self._try_process(batch, self.apply_batch)
229                batch = self._finalize_batch(batch, stats)
230                yield from batch
231
232    def _try_process(self, target: T, func: Callable[[T], T]) -> T:
233        try:
234            return func(target)
235        except Exception as e:
236            if isinstance(target, Document):
237                msg = f"{e!r} occurs while processing {self.name} with {target!r}"
238                target.is_rejected = True
239                target.reject_reason = {"error": msg}
240                self._statistics.errors += 1
241                self.logger.error(msg, exc_info=True)
242                return target
243            if isinstance(target, list):
244                msg = f"{e!r} occurs while batch processing {self.name}"
245                self.logger.error(msg, exc_info=True)
246                for doc in target:
247                    doc.is_rejected = True
248                    doc.reject_reason = {"error": msg}
249                self._statistics.errors += len(target)
250                return target
251            else:
252                raise e
253
254    def __call__(self, text: str, **kwargs: Any) -> str:
255        document = Document(text, **kwargs)
256        document = self._apply(document)
257        return document.text
258
259    def get_statistics(self) -> Statistics:
260        """
261        Get the statistics of this filter.
262        This method returns the statistics of the filter,
263        which includes the number of processed documents, discarded documents, and other statistics.
264        """
265        return self._statistics
266
267    def get_statistics_map(self) -> dict[str, Any]:
268        """
269        Get the statistics of this filter as a dictionary.
270        """
271        return self._statistics.to_dict()
272
273    def shutdown(self) -> None:
274        """
275        This method is called when the filter is no longer needed.
276        You can override this method to release resources or perform cleanup tasks.
277        """
278        pass
279
280    def __enter__(self) -> "Filter":
281        return self
282
283    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
284        """
285        This method is called when the filter is used in a context manager.
286        It calls the `shutdown` method to release resources or perform cleanup tasks.
287        """
288        self.shutdown()
289
290    def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict[str, Any]:
291        """
292        Get the member variable of this filter.
293        Eligible variables are primitive types; [bool, int, float, str, None],
294        and the name of the variable not starts with the underscore; `_`.
295        """
296        if exclude_keys is None:
297            exclude_keys = set()
298        return {
299            k: v
300            for k, v in vars(self).items()
301            if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_")))
302        }
303
304    def _finalize_batch(
305        self: "Filter",
306        batch: Sequence[Document],
307        old_stats: list[dict[str, Any]] = [],
308    ) -> list[Document]:
309        new_stats = [get_doc_info(doc) for doc in batch]
310        for old, new, doc in zip(old_stats, new_stats, batch):
311            self._statistics.update_by_diff(old, new)
312            if not old["is_rejected"] and new["is_rejected"]:
313                doc.reject_reason = self.get_jsonable_vars()
314        return list(batch)
315
316    def __init_rng(self, random_state: int | np.random.Generator | None) -> None:
317        self._owns_rng = True
318        if random_state is None:
319            self._rng = np.random.default_rng()
320            self._owns_rng = False
321        elif isinstance(random_state, int):
322            self._rng = np.random.default_rng(random_state)
323        elif isinstance(random_state, np.random.Generator):
324            self._rng = random_state
325
326    def _set_rng_if_not_initialized(self, rng: np.random.Generator) -> None:
327        """
328        Set the random number generator for this filter if it is not already initialized.
329        This method is called by Compose class.
330        """
331        if not self._owns_rng:
332            self._rng = rng
333
334
335@deprecated_since(version="1.0.0", alternative="Filter")
336class TokenFilter(Filter, ABC):
337    """
338    Base class for token-level filters.
339
340    Token filters, which shuld be implemented in hojichar/filters/token_filters.py,
341    must inherit from this class.
342    """
343
344    def __init__(
345        self, p: float = 1, skip_rejected: bool = True, *args: Any, **kwargs: Any
346    ) -> None:
347        self.name = self.__class__.__name__
348        self.logger = logging.getLogger("hojichar.token_filters." + self.name)
349        assert 0 <= p <= 1
350        self.p = p
351        self.skip_rejected = skip_rejected
352
353    def apply(self, token: Token) -> Token:  # type: ignore
354        raise NotImplementedError(f"{self.__class__.__name__}.apply method is not defined")
355        return token
356
357    def apply_filter(self, document: Document) -> Document:
358        document.tokens = [self.apply(token) for token in document.tokens if not token.is_rejected]
359        return document
360
361    def __call__(self, text: str) -> str:  # type: ignore
362        token = Token(text)
363        token = self.apply(token)
364        return token.text
365
366    def _apply(self, document: Document) -> Document:
367        """
368        Apply the token filter to a single document.
369        This method checks if the document should be skipped.
370        """
371        if self.skip_rejected and document.is_rejected:
372            return document
373        return self.apply_filter(document)
374
375    def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict:
376        """
377        Get the member variable of this filter.
378        Eligible variables are primitive types; [bool, int, float, str, None],
379        and the name of the variable not starts with the underscore; `_`.
380        """
381        if exclude_keys is None:
382            exclude_keys = set()
383        return {
384            k: v
385            for k, v in vars(self).items()
386            if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_")))
387        }
class Filter(abc.ABC):
 23class Filter(ABC):
 24    """
 25    Base class for all filters.
 26    Document-level filters must inherit from this class.
 27
 28    The definition of text processing is in `apply` method.
 29    If you define a new filter, override the method.
 30
 31    When this class is called, apply the filter from string to string.
 32
 33    With context manager, you can use the filter as follows:
 34    ```python
 35    with YourFilter(p=0.5) as filt:
 36        text = filt("This is a sample text.")
 37    ```
 38
 39    """
 40
 41    def __init__(
 42        self,
 43        p: float = 1.0,
 44        skip_rejected: bool = True,
 45        *args: Any,
 46        random_state: int | np.random.Generator | None = None,
 47        use_batch: bool = False,
 48        batch_size: int = 128,
 49        **kwargs: Any,
 50    ) -> None:
 51        """
 52        Initialize the filter.
 53        Parameters
 54        ----------
 55        p : float
 56            The probability of applying the filter.
 57            If `p` is 1, the filter will always be applied.
 58        skip_rejected : bool
 59            If `True`, the filter will skip documents that are already rejected.
 60            If you want to apply the filter to all documents (e.g., postprocess), set this to `False`.
 61        random_state : int | np.random.Generator | None
 62            Seed for the random number generator.
 63            If `None`, a new random number generator will be created.
 64            If `None`, and use in the `Compose` class, the random state is shared with the `Compose` object.
 65        use_batch : bool
 66            If `True`, the filter will process documents in batches in the `apply_stream` method.
 67        batch_size : int
 68            The size of the batch to process documents in the `apply_stream` method.
 69        kwargs : Any
 70            Additional keyword arguments to pass to the filter.
 71        """
 72        self.name = self.__class__.__name__
 73        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
 74        assert 0 <= p <= 1
 75        self.p = p
 76        self.__init_rng(random_state)
 77        self.skip_rejected = skip_rejected
 78        self.use_batch = use_batch
 79        self.batch_size = batch_size
 80
 81        self._statistics: Statistics = Statistics()
 82
 83    @abstractmethod
 84    def apply(self, document: Document) -> Document:
 85        """
 86        Definition of filter behavior.
 87
 88        The document must have a protocol `TextContent`,
 89        and mostly used hojichar.Document class.
 90
 91        In this method, the filter will modify `document.text` or
 92        `document.extras` and set `document.is_rejected = True` to discard the document.
 93
 94        Parameters
 95        ----------
 96        document : Document
 97            Input document
 98
 99        Returns
100        -------
101        Document
102            Processed Document
103        """
104
105    @deprecated_since(version="1.0.0", alternative="apply")
106    def apply_filter(self, document: Document) -> Document:
107        document = self.apply(document)
108        return document
109
110    def _check_skip(self, document: Document) -> bool:
111        """
112        Check if the document should be skipped by this filter.
113        If `skip_rejected` is set to `True`, this method will return `True`
114        if the document is already rejected.
115        If `p` is less than 1, this method will return `True` with a probability of `1 - p`.
116        """
117        skip = self.skip_rejected and document.is_rejected
118        if skip:
119            return True
120        if self.p < 1:
121            if self._rng.random() > self.p:
122                return True
123        return False
124
125    def _apply(self, document: Document) -> Document:
126        """
127        Apply the filter to a single document.
128        This method
129          - checks if the document should be skipped
130          - counts and logging the statistics
131          - logging the reason for rejection if the document is rejected
132
133        This method may be used in `apply` method of `Compose` class.
134        """
135
136        stats = get_doc_info(document)
137
138        if not self._check_skip(document):
139            document = self.apply(document)
140
141        new_stats = get_doc_info(document)
142        self._statistics.update_by_diff(stats, new_stats)
143
144        if not stats["is_rejected"] and new_stats["is_rejected"]:
145            document.reject_reason = self.get_jsonable_vars()
146
147        return document
148
149    def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
150        """
151        Apply the filter to a batch of documents.
152        You can override this method if you want to
153        apply the filter to a batch of documents at once.
154
155        This method may be used in `apply_batch` method of `Compose` class.
156
157        Parameters
158        ----------
159        documents : Sequence[Document]
160            List-like object of input documents
161
162        Returns
163        -------
164        list[Document]
165            List of processed documents
166        """
167        return [self.apply(document) for document in batch]
168
169    def _apply_batch(self, batch: Sequence[Document]) -> list[Document]:
170        """
171        Apply the filter to a batch of documents.
172        This method
173        - checks if the documents should be skipped
174        - counts and logs the statistics
175        - logs the reason for rejection if any document is rejected
176        """
177        skip = False
178        if self.p < 1:
179            skip = self._rng.random() > self.p
180
181        stats = [get_doc_info(document=doc) for doc in batch]
182        if not skip:
183            batch = self.apply_batch(batch)
184        batch = self._finalize_batch(batch, stats)
185        return batch
186
187    def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]:
188        """
189        Apply the filter to a stream of documents.
190        This method is used when you want to process documents one by one.
191        If `use_batch` is set to `True` in the constructor,
192        this method will process documents in batches using the `apply_batch` method.
193
194        Even if an exception occurs during processing, the process will continue, and the following actions will be taken:
195        - Set the `is_rejected` flag of the document to `True`
196        - Set the error details in `reject_reason`
197        - Increment the `errors` count in the statistics retrievable via `get_statistics`
198
199        Parameters
200        ----------
201        stream : Iterable[Document]
202            Stream of input documents
203
204        Returns
205        -------
206        Iterable[Document]
207            Stream of processed documents
208        """
209
210        if not self.use_batch:
211            for document in stream:
212                yield self._try_process(document, self._apply)
213        else:
214            batch: list[Document] = []
215            for document in stream:
216                if self._check_skip(document):
217                    yield document
218                    continue
219
220                batch.append(document)
221                if len(batch) >= self.batch_size:
222                    stats = [get_doc_info(doc) for doc in batch]
223                    batch = self._try_process(batch, self.apply_batch)
224                    batch = self._finalize_batch(batch, stats)
225                    yield from batch
226                    batch.clear()
227            if batch:
228                stats = [get_doc_info(doc) for doc in batch]
229                batch = self._try_process(batch, self.apply_batch)
230                batch = self._finalize_batch(batch, stats)
231                yield from batch
232
233    def _try_process(self, target: T, func: Callable[[T], T]) -> T:
234        try:
235            return func(target)
236        except Exception as e:
237            if isinstance(target, Document):
238                msg = f"{e!r} occurs while processing {self.name} with {target!r}"
239                target.is_rejected = True
240                target.reject_reason = {"error": msg}
241                self._statistics.errors += 1
242                self.logger.error(msg, exc_info=True)
243                return target
244            if isinstance(target, list):
245                msg = f"{e!r} occurs while batch processing {self.name}"
246                self.logger.error(msg, exc_info=True)
247                for doc in target:
248                    doc.is_rejected = True
249                    doc.reject_reason = {"error": msg}
250                self._statistics.errors += len(target)
251                return target
252            else:
253                raise e
254
255    def __call__(self, text: str, **kwargs: Any) -> str:
256        document = Document(text, **kwargs)
257        document = self._apply(document)
258        return document.text
259
260    def get_statistics(self) -> Statistics:
261        """
262        Get the statistics of this filter.
263        This method returns the statistics of the filter,
264        which includes the number of processed documents, discarded documents, and other statistics.
265        """
266        return self._statistics
267
268    def get_statistics_map(self) -> dict[str, Any]:
269        """
270        Get the statistics of this filter as a dictionary.
271        """
272        return self._statistics.to_dict()
273
274    def shutdown(self) -> None:
275        """
276        This method is called when the filter is no longer needed.
277        You can override this method to release resources or perform cleanup tasks.
278        """
279        pass
280
281    def __enter__(self) -> "Filter":
282        return self
283
284    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
285        """
286        This method is called when the filter is used in a context manager.
287        It calls the `shutdown` method to release resources or perform cleanup tasks.
288        """
289        self.shutdown()
290
291    def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict[str, Any]:
292        """
293        Get the member variable of this filter.
294        Eligible variables are primitive types; [bool, int, float, str, None],
295        and the name of the variable not starts with the underscore; `_`.
296        """
297        if exclude_keys is None:
298            exclude_keys = set()
299        return {
300            k: v
301            for k, v in vars(self).items()
302            if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_")))
303        }
304
305    def _finalize_batch(
306        self: "Filter",
307        batch: Sequence[Document],
308        old_stats: list[dict[str, Any]] = [],
309    ) -> list[Document]:
310        new_stats = [get_doc_info(doc) for doc in batch]
311        for old, new, doc in zip(old_stats, new_stats, batch):
312            self._statistics.update_by_diff(old, new)
313            if not old["is_rejected"] and new["is_rejected"]:
314                doc.reject_reason = self.get_jsonable_vars()
315        return list(batch)
316
317    def __init_rng(self, random_state: int | np.random.Generator | None) -> None:
318        self._owns_rng = True
319        if random_state is None:
320            self._rng = np.random.default_rng()
321            self._owns_rng = False
322        elif isinstance(random_state, int):
323            self._rng = np.random.default_rng(random_state)
324        elif isinstance(random_state, np.random.Generator):
325            self._rng = random_state
326
327    def _set_rng_if_not_initialized(self, rng: np.random.Generator) -> None:
328        """
329        Set the random number generator for this filter if it is not already initialized.
330        This method is called by Compose class.
331        """
332        if not self._owns_rng:
333            self._rng = rng

Base class for all filters. Document-level filters must inherit from this class.

The definition of text processing is in apply method. If you define a new filter, override the method.

When this class is called, apply the filter from string to string.

With context manager, you can use the filter as follows:

with YourFilter(p=0.5) as filt:
    text = filt("This is a sample text.")
Filter( p: float = 1.0, skip_rejected: bool = True, *args: Any, random_state: int | numpy.random._generator.Generator | None = None, use_batch: bool = False, batch_size: int = 128, **kwargs: Any)
41    def __init__(
42        self,
43        p: float = 1.0,
44        skip_rejected: bool = True,
45        *args: Any,
46        random_state: int | np.random.Generator | None = None,
47        use_batch: bool = False,
48        batch_size: int = 128,
49        **kwargs: Any,
50    ) -> None:
51        """
52        Initialize the filter.
53        Parameters
54        ----------
55        p : float
56            The probability of applying the filter.
57            If `p` is 1, the filter will always be applied.
58        skip_rejected : bool
59            If `True`, the filter will skip documents that are already rejected.
60            If you want to apply the filter to all documents (e.g., postprocess), set this to `False`.
61        random_state : int | np.random.Generator | None
62            Seed for the random number generator.
63            If `None`, a new random number generator will be created.
64            If `None`, and use in the `Compose` class, the random state is shared with the `Compose` object.
65        use_batch : bool
66            If `True`, the filter will process documents in batches in the `apply_stream` method.
67        batch_size : int
68            The size of the batch to process documents in the `apply_stream` method.
69        kwargs : Any
70            Additional keyword arguments to pass to the filter.
71        """
72        self.name = self.__class__.__name__
73        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
74        assert 0 <= p <= 1
75        self.p = p
76        self.__init_rng(random_state)
77        self.skip_rejected = skip_rejected
78        self.use_batch = use_batch
79        self.batch_size = batch_size
80
81        self._statistics: Statistics = Statistics()

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

@abstractmethod
def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
 83    @abstractmethod
 84    def apply(self, document: Document) -> Document:
 85        """
 86        Definition of filter behavior.
 87
 88        The document must have a protocol `TextContent`,
 89        and mostly used hojichar.Document class.
 90
 91        In this method, the filter will modify `document.text` or
 92        `document.extras` and set `document.is_rejected = True` to discard the document.
 93
 94        Parameters
 95        ----------
 96        document : Document
 97            Input document
 98
 99        Returns
100        -------
101        Document
102            Processed Document
103        """

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

@deprecated_since(version='1.0.0', alternative='apply')
def apply_filter( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
105    @deprecated_since(version="1.0.0", alternative="apply")
106    def apply_filter(self, document: Document) -> Document:
107        document = self.apply(document)
108        return document
def apply_batch( self, batch: collections.abc.Sequence[hojichar.core.models.Document]) -> list[hojichar.core.models.Document]:
149    def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
150        """
151        Apply the filter to a batch of documents.
152        You can override this method if you want to
153        apply the filter to a batch of documents at once.
154
155        This method may be used in `apply_batch` method of `Compose` class.
156
157        Parameters
158        ----------
159        documents : Sequence[Document]
160            List-like object of input documents
161
162        Returns
163        -------
164        list[Document]
165            List of processed documents
166        """
167        return [self.apply(document) for document in batch]

Apply the filter to a batch of documents. You can override this method if you want to apply the filter to a batch of documents at once.

This method may be used in apply_batch method of Compose class.

Parameters

documents : Sequence[Document] List-like object of input documents

Returns

list[Document] List of processed documents

def apply_stream( self, stream: collections.abc.Iterable[hojichar.core.models.Document]) -> collections.abc.Iterable[hojichar.core.models.Document]:
187    def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]:
188        """
189        Apply the filter to a stream of documents.
190        This method is used when you want to process documents one by one.
191        If `use_batch` is set to `True` in the constructor,
192        this method will process documents in batches using the `apply_batch` method.
193
194        Even if an exception occurs during processing, the process will continue, and the following actions will be taken:
195        - Set the `is_rejected` flag of the document to `True`
196        - Set the error details in `reject_reason`
197        - Increment the `errors` count in the statistics retrievable via `get_statistics`
198
199        Parameters
200        ----------
201        stream : Iterable[Document]
202            Stream of input documents
203
204        Returns
205        -------
206        Iterable[Document]
207            Stream of processed documents
208        """
209
210        if not self.use_batch:
211            for document in stream:
212                yield self._try_process(document, self._apply)
213        else:
214            batch: list[Document] = []
215            for document in stream:
216                if self._check_skip(document):
217                    yield document
218                    continue
219
220                batch.append(document)
221                if len(batch) >= self.batch_size:
222                    stats = [get_doc_info(doc) for doc in batch]
223                    batch = self._try_process(batch, self.apply_batch)
224                    batch = self._finalize_batch(batch, stats)
225                    yield from batch
226                    batch.clear()
227            if batch:
228                stats = [get_doc_info(doc) for doc in batch]
229                batch = self._try_process(batch, self.apply_batch)
230                batch = self._finalize_batch(batch, stats)
231                yield from batch

Apply the filter to a stream of documents. This method is used when you want to process documents one by one. If use_batch is set to True in the constructor, this method will process documents in batches using the apply_batch method.

Even if an exception occurs during processing, the process will continue, and the following actions will be taken:

  • Set the is_rejected flag of the document to True
  • Set the error details in reject_reason
  • Increment the errors count in the statistics retrievable via get_statistics

Parameters

stream : Iterable[Document] Stream of input documents

Returns

Iterable[Document] Stream of processed documents

def get_statistics(self) -> hojichar.core.models.Statistics:
260    def get_statistics(self) -> Statistics:
261        """
262        Get the statistics of this filter.
263        This method returns the statistics of the filter,
264        which includes the number of processed documents, discarded documents, and other statistics.
265        """
266        return self._statistics

Get the statistics of this filter. This method returns the statistics of the filter, which includes the number of processed documents, discarded documents, and other statistics.

def get_statistics_map(self) -> dict[str, typing.Any]:
268    def get_statistics_map(self) -> dict[str, Any]:
269        """
270        Get the statistics of this filter as a dictionary.
271        """
272        return self._statistics.to_dict()

Get the statistics of this filter as a dictionary.

def shutdown(self) -> None:
274    def shutdown(self) -> None:
275        """
276        This method is called when the filter is no longer needed.
277        You can override this method to release resources or perform cleanup tasks.
278        """
279        pass

This method is called when the filter is no longer needed. You can override this method to release resources or perform cleanup tasks.

def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict[str, typing.Any]:
291    def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict[str, Any]:
292        """
293        Get the member variable of this filter.
294        Eligible variables are primitive types; [bool, int, float, str, None],
295        and the name of the variable not starts with the underscore; `_`.
296        """
297        if exclude_keys is None:
298            exclude_keys = set()
299        return {
300            k: v
301            for k, v in vars(self).items()
302            if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_")))
303        }

Get the member variable of this filter. Eligible variables are primitive types; [bool, int, float, str, None], and the name of the variable not starts with the underscore; _.

@deprecated_since(version='1.0.0', alternative='Filter')
class TokenFilter(Filter, abc.ABC):
336@deprecated_since(version="1.0.0", alternative="Filter")
337class TokenFilter(Filter, ABC):
338    """
339    Base class for token-level filters.
340
341    Token filters, which shuld be implemented in hojichar/filters/token_filters.py,
342    must inherit from this class.
343    """
344
345    def __init__(
346        self, p: float = 1, skip_rejected: bool = True, *args: Any, **kwargs: Any
347    ) -> None:
348        self.name = self.__class__.__name__
349        self.logger = logging.getLogger("hojichar.token_filters." + self.name)
350        assert 0 <= p <= 1
351        self.p = p
352        self.skip_rejected = skip_rejected
353
354    def apply(self, token: Token) -> Token:  # type: ignore
355        raise NotImplementedError(f"{self.__class__.__name__}.apply method is not defined")
356        return token
357
358    def apply_filter(self, document: Document) -> Document:
359        document.tokens = [self.apply(token) for token in document.tokens if not token.is_rejected]
360        return document
361
362    def __call__(self, text: str) -> str:  # type: ignore
363        token = Token(text)
364        token = self.apply(token)
365        return token.text
366
367    def _apply(self, document: Document) -> Document:
368        """
369        Apply the token filter to a single document.
370        This method checks if the document should be skipped.
371        """
372        if self.skip_rejected and document.is_rejected:
373            return document
374        return self.apply_filter(document)
375
376    def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict:
377        """
378        Get the member variable of this filter.
379        Eligible variables are primitive types; [bool, int, float, str, None],
380        and the name of the variable not starts with the underscore; `_`.
381        """
382        if exclude_keys is None:
383            exclude_keys = set()
384        return {
385            k: v
386            for k, v in vars(self).items()
387            if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_")))
388        }

Base class for token-level filters.

Token filters, which shuld be implemented in hojichar/filters/token_filters.py, must inherit from this class.

TokenFilter(p: float = 1, skip_rejected: bool = True, *args: Any, **kwargs: Any)
345    def __init__(
346        self, p: float = 1, skip_rejected: bool = True, *args: Any, **kwargs: Any
347    ) -> None:
348        self.name = self.__class__.__name__
349        self.logger = logging.getLogger("hojichar.token_filters." + self.name)
350        assert 0 <= p <= 1
351        self.p = p
352        self.skip_rejected = skip_rejected

Initialize the filter.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None, a new random number generator will be created. If None, and use in the Compose class, the random state is shared with the Compose object. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. kwargs : Any Additional keyword arguments to pass to the filter.

def apply(self, token: hojichar.core.models.Token) -> hojichar.core.models.Token:
354    def apply(self, token: Token) -> Token:  # type: ignore
355        raise NotImplementedError(f"{self.__class__.__name__}.apply method is not defined")
356        return token

Definition of filter behavior.

The document must have a protocol TextContent, and mostly used hojichar.Document class.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

def apply_filter( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
358    def apply_filter(self, document: Document) -> Document:
359        document.tokens = [self.apply(token) for token in document.tokens if not token.is_rejected]
360        return document
def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict:
376    def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict:
377        """
378        Get the member variable of this filter.
379        Eligible variables are primitive types; [bool, int, float, str, None],
380        and the name of the variable not starts with the underscore; `_`.
381        """
382        if exclude_keys is None:
383            exclude_keys = set()
384        return {
385            k: v
386            for k, v in vars(self).items()
387            if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_")))
388        }

Get the member variable of this filter. Eligible variables are primitive types; [bool, int, float, str, None], and the name of the variable not starts with the underscore; _.