hojichar.core.filter_interface
1import logging 2from abc import ABC, abstractmethod 3from collections.abc import Callable, Iterable, Sequence 4from typing import Any, TypeVar 5 6import numpy as np 7 8from hojichar.core.models import Document, Statistics, Token, get_doc_info 9from hojichar.utils.warn_deprecation import deprecated_since 10 11T = TypeVar("T") 12 13 14def _is_jsonable(data: Any) -> bool: 15 if data is None: 16 return True 17 elif isinstance(data, (bool, int, float, str)): 18 return True 19 return False 20 21 22class Filter(ABC): 23 """ 24 Base class for all filters. 25 Document-level filters must inherit from this class. 26 27 The definition of text processing is in `apply` method. 28 If you define a new filter, override the method. 29 30 When this class is called, apply the filter from string to string. 31 32 With context manager, you can use the filter as follows: 33 ```python 34 with YourFilter(p=0.5) as filt: 35 text = filt("This is a sample text.") 36 ``` 37 38 """ 39 40 def __init__( 41 self, 42 p: float = 1.0, 43 skip_rejected: bool = True, 44 *args: Any, 45 random_state: int | np.random.Generator | None = None, 46 use_batch: bool = False, 47 batch_size: int = 128, 48 **kwargs: Any, 49 ) -> None: 50 """ 51 Initialize the filter. 52 Parameters 53 ---------- 54 p : float 55 The probability of applying the filter. 56 If `p` is 1, the filter will always be applied. 57 skip_rejected : bool 58 If `True`, the filter will skip documents that are already rejected. 59 If you want to apply the filter to all documents (e.g., postprocess), set this to `False`. 60 random_state : int | np.random.Generator | None 61 Seed for the random number generator. 62 If `None`, a new random number generator will be created. 63 If `None`, and use in the `Compose` class, the random state is shared with the `Compose` object. 64 use_batch : bool 65 If `True`, the filter will process documents in batches in the `apply_stream` method. 66 batch_size : int 67 The size of the batch to process documents in the `apply_stream` method. 68 kwargs : Any 69 Additional keyword arguments to pass to the filter. 70 """ 71 self.name = self.__class__.__name__ 72 self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}") 73 assert 0 <= p <= 1 74 self.p = p 75 self.__init_rng(random_state) 76 self.skip_rejected = skip_rejected 77 self.use_batch = use_batch 78 self.batch_size = batch_size 79 80 self._statistics: Statistics = Statistics() 81 82 @abstractmethod 83 def apply(self, document: Document) -> Document: 84 """ 85 Definition of filter behavior. 86 87 The document must have a protocol `TextContent`, 88 and mostly used hojichar.Document class. 89 90 In this method, the filter will modify `document.text` or 91 `document.extras` and set `document.is_rejected = True` to discard the document. 92 93 Parameters 94 ---------- 95 document : Document 96 Input document 97 98 Returns 99 ------- 100 Document 101 Processed Document 102 """ 103 104 @deprecated_since(version="1.0.0", alternative="apply") 105 def apply_filter(self, document: Document) -> Document: 106 document = self.apply(document) 107 return document 108 109 def _check_skip(self, document: Document) -> bool: 110 """ 111 Check if the document should be skipped by this filter. 112 If `skip_rejected` is set to `True`, this method will return `True` 113 if the document is already rejected. 114 If `p` is less than 1, this method will return `True` with a probability of `1 - p`. 115 """ 116 skip = self.skip_rejected and document.is_rejected 117 if skip: 118 return True 119 if self.p < 1: 120 if self._rng.random() > self.p: 121 return True 122 return False 123 124 def _apply(self, document: Document) -> Document: 125 """ 126 Apply the filter to a single document. 127 This method 128 - checks if the document should be skipped 129 - counts and logging the statistics 130 - logging the reason for rejection if the document is rejected 131 132 This method may be used in `apply` method of `Compose` class. 133 """ 134 135 stats = get_doc_info(document) 136 137 if not self._check_skip(document): 138 document = self.apply(document) 139 140 new_stats = get_doc_info(document) 141 self._statistics.update_by_diff(stats, new_stats) 142 143 if not stats["is_rejected"] and new_stats["is_rejected"]: 144 document.reject_reason = self.get_jsonable_vars() 145 146 return document 147 148 def apply_batch(self, batch: Sequence[Document]) -> list[Document]: 149 """ 150 Apply the filter to a batch of documents. 151 You can override this method if you want to 152 apply the filter to a batch of documents at once. 153 154 This method may be used in `apply_batch` method of `Compose` class. 155 156 Parameters 157 ---------- 158 documents : Sequence[Document] 159 List-like object of input documents 160 161 Returns 162 ------- 163 list[Document] 164 List of processed documents 165 """ 166 return [self.apply(document) for document in batch] 167 168 def _apply_batch(self, batch: Sequence[Document]) -> list[Document]: 169 """ 170 Apply the filter to a batch of documents. 171 This method 172 - checks if the documents should be skipped 173 - counts and logs the statistics 174 - logs the reason for rejection if any document is rejected 175 """ 176 skip = False 177 if self.p < 1: 178 skip = self._rng.random() > self.p 179 180 stats = [get_doc_info(document=doc) for doc in batch] 181 if not skip: 182 batch = self.apply_batch(batch) 183 batch = self._finalize_batch(batch, stats) 184 return batch 185 186 def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]: 187 """ 188 Apply the filter to a stream of documents. 189 This method is used when you want to process documents one by one. 190 If `use_batch` is set to `True` in the constructor, 191 this method will process documents in batches using the `apply_batch` method. 192 193 Even if an exception occurs during processing, the process will continue, and the following actions will be taken: 194 - Set the `is_rejected` flag of the document to `True` 195 - Set the error details in `reject_reason` 196 - Increment the `errors` count in the statistics retrievable via `get_statistics` 197 198 Parameters 199 ---------- 200 stream : Iterable[Document] 201 Stream of input documents 202 203 Returns 204 ------- 205 Iterable[Document] 206 Stream of processed documents 207 """ 208 209 if not self.use_batch: 210 for document in stream: 211 yield self._try_process(document, self._apply) 212 else: 213 batch: list[Document] = [] 214 for document in stream: 215 if self._check_skip(document): 216 yield document 217 continue 218 219 batch.append(document) 220 if len(batch) >= self.batch_size: 221 stats = [get_doc_info(doc) for doc in batch] 222 batch = self._try_process(batch, self.apply_batch) 223 batch = self._finalize_batch(batch, stats) 224 yield from batch 225 batch.clear() 226 if batch: 227 stats = [get_doc_info(doc) for doc in batch] 228 batch = self._try_process(batch, self.apply_batch) 229 batch = self._finalize_batch(batch, stats) 230 yield from batch 231 232 def _try_process(self, target: T, func: Callable[[T], T]) -> T: 233 try: 234 return func(target) 235 except Exception as e: 236 if isinstance(target, Document): 237 msg = f"{e!r} occurs while processing {self.name} with {target!r}" 238 target.is_rejected = True 239 target.reject_reason = {"error": msg} 240 self._statistics.errors += 1 241 self.logger.error(msg, exc_info=True) 242 return target 243 if isinstance(target, list): 244 msg = f"{e!r} occurs while batch processing {self.name}" 245 self.logger.error(msg, exc_info=True) 246 for doc in target: 247 doc.is_rejected = True 248 doc.reject_reason = {"error": msg} 249 self._statistics.errors += len(target) 250 return target 251 else: 252 raise e 253 254 def __call__(self, text: str, **kwargs: Any) -> str: 255 document = Document(text, **kwargs) 256 document = self._apply(document) 257 return document.text 258 259 def get_statistics(self) -> Statistics: 260 """ 261 Get the statistics of this filter. 262 This method returns the statistics of the filter, 263 which includes the number of processed documents, discarded documents, and other statistics. 264 """ 265 return self._statistics 266 267 def get_statistics_map(self) -> dict[str, Any]: 268 """ 269 Get the statistics of this filter as a dictionary. 270 """ 271 return self._statistics.to_dict() 272 273 def shutdown(self) -> None: 274 """ 275 This method is called when the filter is no longer needed. 276 You can override this method to release resources or perform cleanup tasks. 277 """ 278 pass 279 280 def __enter__(self) -> "Filter": 281 return self 282 283 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 284 """ 285 This method is called when the filter is used in a context manager. 286 It calls the `shutdown` method to release resources or perform cleanup tasks. 287 """ 288 self.shutdown() 289 290 def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict[str, Any]: 291 """ 292 Get the member variable of this filter. 293 Eligible variables are primitive types; [bool, int, float, str, None], 294 and the name of the variable not starts with the underscore; `_`. 295 """ 296 if exclude_keys is None: 297 exclude_keys = set() 298 return { 299 k: v 300 for k, v in vars(self).items() 301 if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_"))) 302 } 303 304 def _finalize_batch( 305 self: "Filter", 306 batch: Sequence[Document], 307 old_stats: list[dict[str, Any]] = [], 308 ) -> list[Document]: 309 new_stats = [get_doc_info(doc) for doc in batch] 310 for old, new, doc in zip(old_stats, new_stats, batch): 311 self._statistics.update_by_diff(old, new) 312 if not old["is_rejected"] and new["is_rejected"]: 313 doc.reject_reason = self.get_jsonable_vars() 314 return list(batch) 315 316 def __init_rng(self, random_state: int | np.random.Generator | None) -> None: 317 self._owns_rng = True 318 if random_state is None: 319 self._rng = np.random.default_rng() 320 self._owns_rng = False 321 elif isinstance(random_state, int): 322 self._rng = np.random.default_rng(random_state) 323 elif isinstance(random_state, np.random.Generator): 324 self._rng = random_state 325 326 def _set_rng_if_not_initialized(self, rng: np.random.Generator) -> None: 327 """ 328 Set the random number generator for this filter if it is not already initialized. 329 This method is called by Compose class. 330 """ 331 if not self._owns_rng: 332 self._rng = rng 333 334 335@deprecated_since(version="1.0.0", alternative="Filter") 336class TokenFilter(Filter, ABC): 337 """ 338 Base class for token-level filters. 339 340 Token filters, which shuld be implemented in hojichar/filters/token_filters.py, 341 must inherit from this class. 342 """ 343 344 def __init__( 345 self, p: float = 1, skip_rejected: bool = True, *args: Any, **kwargs: Any 346 ) -> None: 347 self.name = self.__class__.__name__ 348 self.logger = logging.getLogger("hojichar.token_filters." + self.name) 349 assert 0 <= p <= 1 350 self.p = p 351 self.skip_rejected = skip_rejected 352 353 def apply(self, token: Token) -> Token: # type: ignore 354 raise NotImplementedError(f"{self.__class__.__name__}.apply method is not defined") 355 return token 356 357 def apply_filter(self, document: Document) -> Document: 358 document.tokens = [self.apply(token) for token in document.tokens if not token.is_rejected] 359 return document 360 361 def __call__(self, text: str) -> str: # type: ignore 362 token = Token(text) 363 token = self.apply(token) 364 return token.text 365 366 def _apply(self, document: Document) -> Document: 367 """ 368 Apply the token filter to a single document. 369 This method checks if the document should be skipped. 370 """ 371 if self.skip_rejected and document.is_rejected: 372 return document 373 return self.apply_filter(document) 374 375 def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict: 376 """ 377 Get the member variable of this filter. 378 Eligible variables are primitive types; [bool, int, float, str, None], 379 and the name of the variable not starts with the underscore; `_`. 380 """ 381 if exclude_keys is None: 382 exclude_keys = set() 383 return { 384 k: v 385 for k, v in vars(self).items() 386 if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_"))) 387 }
23class Filter(ABC): 24 """ 25 Base class for all filters. 26 Document-level filters must inherit from this class. 27 28 The definition of text processing is in `apply` method. 29 If you define a new filter, override the method. 30 31 When this class is called, apply the filter from string to string. 32 33 With context manager, you can use the filter as follows: 34 ```python 35 with YourFilter(p=0.5) as filt: 36 text = filt("This is a sample text.") 37 ``` 38 39 """ 40 41 def __init__( 42 self, 43 p: float = 1.0, 44 skip_rejected: bool = True, 45 *args: Any, 46 random_state: int | np.random.Generator | None = None, 47 use_batch: bool = False, 48 batch_size: int = 128, 49 **kwargs: Any, 50 ) -> None: 51 """ 52 Initialize the filter. 53 Parameters 54 ---------- 55 p : float 56 The probability of applying the filter. 57 If `p` is 1, the filter will always be applied. 58 skip_rejected : bool 59 If `True`, the filter will skip documents that are already rejected. 60 If you want to apply the filter to all documents (e.g., postprocess), set this to `False`. 61 random_state : int | np.random.Generator | None 62 Seed for the random number generator. 63 If `None`, a new random number generator will be created. 64 If `None`, and use in the `Compose` class, the random state is shared with the `Compose` object. 65 use_batch : bool 66 If `True`, the filter will process documents in batches in the `apply_stream` method. 67 batch_size : int 68 The size of the batch to process documents in the `apply_stream` method. 69 kwargs : Any 70 Additional keyword arguments to pass to the filter. 71 """ 72 self.name = self.__class__.__name__ 73 self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}") 74 assert 0 <= p <= 1 75 self.p = p 76 self.__init_rng(random_state) 77 self.skip_rejected = skip_rejected 78 self.use_batch = use_batch 79 self.batch_size = batch_size 80 81 self._statistics: Statistics = Statistics() 82 83 @abstractmethod 84 def apply(self, document: Document) -> Document: 85 """ 86 Definition of filter behavior. 87 88 The document must have a protocol `TextContent`, 89 and mostly used hojichar.Document class. 90 91 In this method, the filter will modify `document.text` or 92 `document.extras` and set `document.is_rejected = True` to discard the document. 93 94 Parameters 95 ---------- 96 document : Document 97 Input document 98 99 Returns 100 ------- 101 Document 102 Processed Document 103 """ 104 105 @deprecated_since(version="1.0.0", alternative="apply") 106 def apply_filter(self, document: Document) -> Document: 107 document = self.apply(document) 108 return document 109 110 def _check_skip(self, document: Document) -> bool: 111 """ 112 Check if the document should be skipped by this filter. 113 If `skip_rejected` is set to `True`, this method will return `True` 114 if the document is already rejected. 115 If `p` is less than 1, this method will return `True` with a probability of `1 - p`. 116 """ 117 skip = self.skip_rejected and document.is_rejected 118 if skip: 119 return True 120 if self.p < 1: 121 if self._rng.random() > self.p: 122 return True 123 return False 124 125 def _apply(self, document: Document) -> Document: 126 """ 127 Apply the filter to a single document. 128 This method 129 - checks if the document should be skipped 130 - counts and logging the statistics 131 - logging the reason for rejection if the document is rejected 132 133 This method may be used in `apply` method of `Compose` class. 134 """ 135 136 stats = get_doc_info(document) 137 138 if not self._check_skip(document): 139 document = self.apply(document) 140 141 new_stats = get_doc_info(document) 142 self._statistics.update_by_diff(stats, new_stats) 143 144 if not stats["is_rejected"] and new_stats["is_rejected"]: 145 document.reject_reason = self.get_jsonable_vars() 146 147 return document 148 149 def apply_batch(self, batch: Sequence[Document]) -> list[Document]: 150 """ 151 Apply the filter to a batch of documents. 152 You can override this method if you want to 153 apply the filter to a batch of documents at once. 154 155 This method may be used in `apply_batch` method of `Compose` class. 156 157 Parameters 158 ---------- 159 documents : Sequence[Document] 160 List-like object of input documents 161 162 Returns 163 ------- 164 list[Document] 165 List of processed documents 166 """ 167 return [self.apply(document) for document in batch] 168 169 def _apply_batch(self, batch: Sequence[Document]) -> list[Document]: 170 """ 171 Apply the filter to a batch of documents. 172 This method 173 - checks if the documents should be skipped 174 - counts and logs the statistics 175 - logs the reason for rejection if any document is rejected 176 """ 177 skip = False 178 if self.p < 1: 179 skip = self._rng.random() > self.p 180 181 stats = [get_doc_info(document=doc) for doc in batch] 182 if not skip: 183 batch = self.apply_batch(batch) 184 batch = self._finalize_batch(batch, stats) 185 return batch 186 187 def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]: 188 """ 189 Apply the filter to a stream of documents. 190 This method is used when you want to process documents one by one. 191 If `use_batch` is set to `True` in the constructor, 192 this method will process documents in batches using the `apply_batch` method. 193 194 Even if an exception occurs during processing, the process will continue, and the following actions will be taken: 195 - Set the `is_rejected` flag of the document to `True` 196 - Set the error details in `reject_reason` 197 - Increment the `errors` count in the statistics retrievable via `get_statistics` 198 199 Parameters 200 ---------- 201 stream : Iterable[Document] 202 Stream of input documents 203 204 Returns 205 ------- 206 Iterable[Document] 207 Stream of processed documents 208 """ 209 210 if not self.use_batch: 211 for document in stream: 212 yield self._try_process(document, self._apply) 213 else: 214 batch: list[Document] = [] 215 for document in stream: 216 if self._check_skip(document): 217 yield document 218 continue 219 220 batch.append(document) 221 if len(batch) >= self.batch_size: 222 stats = [get_doc_info(doc) for doc in batch] 223 batch = self._try_process(batch, self.apply_batch) 224 batch = self._finalize_batch(batch, stats) 225 yield from batch 226 batch.clear() 227 if batch: 228 stats = [get_doc_info(doc) for doc in batch] 229 batch = self._try_process(batch, self.apply_batch) 230 batch = self._finalize_batch(batch, stats) 231 yield from batch 232 233 def _try_process(self, target: T, func: Callable[[T], T]) -> T: 234 try: 235 return func(target) 236 except Exception as e: 237 if isinstance(target, Document): 238 msg = f"{e!r} occurs while processing {self.name} with {target!r}" 239 target.is_rejected = True 240 target.reject_reason = {"error": msg} 241 self._statistics.errors += 1 242 self.logger.error(msg, exc_info=True) 243 return target 244 if isinstance(target, list): 245 msg = f"{e!r} occurs while batch processing {self.name}" 246 self.logger.error(msg, exc_info=True) 247 for doc in target: 248 doc.is_rejected = True 249 doc.reject_reason = {"error": msg} 250 self._statistics.errors += len(target) 251 return target 252 else: 253 raise e 254 255 def __call__(self, text: str, **kwargs: Any) -> str: 256 document = Document(text, **kwargs) 257 document = self._apply(document) 258 return document.text 259 260 def get_statistics(self) -> Statistics: 261 """ 262 Get the statistics of this filter. 263 This method returns the statistics of the filter, 264 which includes the number of processed documents, discarded documents, and other statistics. 265 """ 266 return self._statistics 267 268 def get_statistics_map(self) -> dict[str, Any]: 269 """ 270 Get the statistics of this filter as a dictionary. 271 """ 272 return self._statistics.to_dict() 273 274 def shutdown(self) -> None: 275 """ 276 This method is called when the filter is no longer needed. 277 You can override this method to release resources or perform cleanup tasks. 278 """ 279 pass 280 281 def __enter__(self) -> "Filter": 282 return self 283 284 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 285 """ 286 This method is called when the filter is used in a context manager. 287 It calls the `shutdown` method to release resources or perform cleanup tasks. 288 """ 289 self.shutdown() 290 291 def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict[str, Any]: 292 """ 293 Get the member variable of this filter. 294 Eligible variables are primitive types; [bool, int, float, str, None], 295 and the name of the variable not starts with the underscore; `_`. 296 """ 297 if exclude_keys is None: 298 exclude_keys = set() 299 return { 300 k: v 301 for k, v in vars(self).items() 302 if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_"))) 303 } 304 305 def _finalize_batch( 306 self: "Filter", 307 batch: Sequence[Document], 308 old_stats: list[dict[str, Any]] = [], 309 ) -> list[Document]: 310 new_stats = [get_doc_info(doc) for doc in batch] 311 for old, new, doc in zip(old_stats, new_stats, batch): 312 self._statistics.update_by_diff(old, new) 313 if not old["is_rejected"] and new["is_rejected"]: 314 doc.reject_reason = self.get_jsonable_vars() 315 return list(batch) 316 317 def __init_rng(self, random_state: int | np.random.Generator | None) -> None: 318 self._owns_rng = True 319 if random_state is None: 320 self._rng = np.random.default_rng() 321 self._owns_rng = False 322 elif isinstance(random_state, int): 323 self._rng = np.random.default_rng(random_state) 324 elif isinstance(random_state, np.random.Generator): 325 self._rng = random_state 326 327 def _set_rng_if_not_initialized(self, rng: np.random.Generator) -> None: 328 """ 329 Set the random number generator for this filter if it is not already initialized. 330 This method is called by Compose class. 331 """ 332 if not self._owns_rng: 333 self._rng = rng
Base class for all filters. Document-level filters must inherit from this class.
The definition of text processing is in apply method.
If you define a new filter, override the method.
When this class is called, apply the filter from string to string.
With context manager, you can use the filter as follows:
with YourFilter(p=0.5) as filt:
text = filt("This is a sample text.")
41 def __init__( 42 self, 43 p: float = 1.0, 44 skip_rejected: bool = True, 45 *args: Any, 46 random_state: int | np.random.Generator | None = None, 47 use_batch: bool = False, 48 batch_size: int = 128, 49 **kwargs: Any, 50 ) -> None: 51 """ 52 Initialize the filter. 53 Parameters 54 ---------- 55 p : float 56 The probability of applying the filter. 57 If `p` is 1, the filter will always be applied. 58 skip_rejected : bool 59 If `True`, the filter will skip documents that are already rejected. 60 If you want to apply the filter to all documents (e.g., postprocess), set this to `False`. 61 random_state : int | np.random.Generator | None 62 Seed for the random number generator. 63 If `None`, a new random number generator will be created. 64 If `None`, and use in the `Compose` class, the random state is shared with the `Compose` object. 65 use_batch : bool 66 If `True`, the filter will process documents in batches in the `apply_stream` method. 67 batch_size : int 68 The size of the batch to process documents in the `apply_stream` method. 69 kwargs : Any 70 Additional keyword arguments to pass to the filter. 71 """ 72 self.name = self.__class__.__name__ 73 self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}") 74 assert 0 <= p <= 1 75 self.p = p 76 self.__init_rng(random_state) 77 self.skip_rejected = skip_rejected 78 self.use_batch = use_batch 79 self.batch_size = batch_size 80 81 self._statistics: Statistics = Statistics()
Initialize the filter.
Parameters
p : float
The probability of applying the filter.
If p is 1, the filter will always be applied.
skip_rejected : bool
If True, the filter will skip documents that are already rejected.
If you want to apply the filter to all documents (e.g., postprocess), set this to False.
random_state : int | np.random.Generator | None
Seed for the random number generator.
If None, a new random number generator will be created.
If None, and use in the Compose class, the random state is shared with the Compose object.
use_batch : bool
If True, the filter will process documents in batches in the apply_stream method.
batch_size : int
The size of the batch to process documents in the apply_stream method.
kwargs : Any
Additional keyword arguments to pass to the filter.
83 @abstractmethod 84 def apply(self, document: Document) -> Document: 85 """ 86 Definition of filter behavior. 87 88 The document must have a protocol `TextContent`, 89 and mostly used hojichar.Document class. 90 91 In this method, the filter will modify `document.text` or 92 `document.extras` and set `document.is_rejected = True` to discard the document. 93 94 Parameters 95 ---------- 96 document : Document 97 Input document 98 99 Returns 100 ------- 101 Document 102 Processed Document 103 """
Definition of filter behavior.
The document must have a protocol TextContent,
and mostly used hojichar.Document class.
In this method, the filter will modify document.text or
document.extras and set document.is_rejected = True to discard the document.
Parameters
document : Document Input document
Returns
Document Processed Document
149 def apply_batch(self, batch: Sequence[Document]) -> list[Document]: 150 """ 151 Apply the filter to a batch of documents. 152 You can override this method if you want to 153 apply the filter to a batch of documents at once. 154 155 This method may be used in `apply_batch` method of `Compose` class. 156 157 Parameters 158 ---------- 159 documents : Sequence[Document] 160 List-like object of input documents 161 162 Returns 163 ------- 164 list[Document] 165 List of processed documents 166 """ 167 return [self.apply(document) for document in batch]
Apply the filter to a batch of documents. You can override this method if you want to apply the filter to a batch of documents at once.
This method may be used in apply_batch method of Compose class.
Parameters
documents : Sequence[Document] List-like object of input documents
Returns
list[Document] List of processed documents
187 def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]: 188 """ 189 Apply the filter to a stream of documents. 190 This method is used when you want to process documents one by one. 191 If `use_batch` is set to `True` in the constructor, 192 this method will process documents in batches using the `apply_batch` method. 193 194 Even if an exception occurs during processing, the process will continue, and the following actions will be taken: 195 - Set the `is_rejected` flag of the document to `True` 196 - Set the error details in `reject_reason` 197 - Increment the `errors` count in the statistics retrievable via `get_statistics` 198 199 Parameters 200 ---------- 201 stream : Iterable[Document] 202 Stream of input documents 203 204 Returns 205 ------- 206 Iterable[Document] 207 Stream of processed documents 208 """ 209 210 if not self.use_batch: 211 for document in stream: 212 yield self._try_process(document, self._apply) 213 else: 214 batch: list[Document] = [] 215 for document in stream: 216 if self._check_skip(document): 217 yield document 218 continue 219 220 batch.append(document) 221 if len(batch) >= self.batch_size: 222 stats = [get_doc_info(doc) for doc in batch] 223 batch = self._try_process(batch, self.apply_batch) 224 batch = self._finalize_batch(batch, stats) 225 yield from batch 226 batch.clear() 227 if batch: 228 stats = [get_doc_info(doc) for doc in batch] 229 batch = self._try_process(batch, self.apply_batch) 230 batch = self._finalize_batch(batch, stats) 231 yield from batch
Apply the filter to a stream of documents.
This method is used when you want to process documents one by one.
If use_batch is set to True in the constructor,
this method will process documents in batches using the apply_batch method.
Even if an exception occurs during processing, the process will continue, and the following actions will be taken:
- Set the
is_rejectedflag of the document toTrue - Set the error details in
reject_reason - Increment the
errorscount in the statistics retrievable viaget_statistics
Parameters
stream : Iterable[Document] Stream of input documents
Returns
Iterable[Document] Stream of processed documents
260 def get_statistics(self) -> Statistics: 261 """ 262 Get the statistics of this filter. 263 This method returns the statistics of the filter, 264 which includes the number of processed documents, discarded documents, and other statistics. 265 """ 266 return self._statistics
Get the statistics of this filter. This method returns the statistics of the filter, which includes the number of processed documents, discarded documents, and other statistics.
268 def get_statistics_map(self) -> dict[str, Any]: 269 """ 270 Get the statistics of this filter as a dictionary. 271 """ 272 return self._statistics.to_dict()
Get the statistics of this filter as a dictionary.
274 def shutdown(self) -> None: 275 """ 276 This method is called when the filter is no longer needed. 277 You can override this method to release resources or perform cleanup tasks. 278 """ 279 pass
This method is called when the filter is no longer needed. You can override this method to release resources or perform cleanup tasks.
291 def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict[str, Any]: 292 """ 293 Get the member variable of this filter. 294 Eligible variables are primitive types; [bool, int, float, str, None], 295 and the name of the variable not starts with the underscore; `_`. 296 """ 297 if exclude_keys is None: 298 exclude_keys = set() 299 return { 300 k: v 301 for k, v in vars(self).items() 302 if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_"))) 303 }
Get the member variable of this filter.
Eligible variables are primitive types; [bool, int, float, str, None],
and the name of the variable not starts with the underscore; _.
336@deprecated_since(version="1.0.0", alternative="Filter") 337class TokenFilter(Filter, ABC): 338 """ 339 Base class for token-level filters. 340 341 Token filters, which shuld be implemented in hojichar/filters/token_filters.py, 342 must inherit from this class. 343 """ 344 345 def __init__( 346 self, p: float = 1, skip_rejected: bool = True, *args: Any, **kwargs: Any 347 ) -> None: 348 self.name = self.__class__.__name__ 349 self.logger = logging.getLogger("hojichar.token_filters." + self.name) 350 assert 0 <= p <= 1 351 self.p = p 352 self.skip_rejected = skip_rejected 353 354 def apply(self, token: Token) -> Token: # type: ignore 355 raise NotImplementedError(f"{self.__class__.__name__}.apply method is not defined") 356 return token 357 358 def apply_filter(self, document: Document) -> Document: 359 document.tokens = [self.apply(token) for token in document.tokens if not token.is_rejected] 360 return document 361 362 def __call__(self, text: str) -> str: # type: ignore 363 token = Token(text) 364 token = self.apply(token) 365 return token.text 366 367 def _apply(self, document: Document) -> Document: 368 """ 369 Apply the token filter to a single document. 370 This method checks if the document should be skipped. 371 """ 372 if self.skip_rejected and document.is_rejected: 373 return document 374 return self.apply_filter(document) 375 376 def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict: 377 """ 378 Get the member variable of this filter. 379 Eligible variables are primitive types; [bool, int, float, str, None], 380 and the name of the variable not starts with the underscore; `_`. 381 """ 382 if exclude_keys is None: 383 exclude_keys = set() 384 return { 385 k: v 386 for k, v in vars(self).items() 387 if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_"))) 388 }
Base class for token-level filters.
Token filters, which shuld be implemented in hojichar/filters/token_filters.py, must inherit from this class.
345 def __init__( 346 self, p: float = 1, skip_rejected: bool = True, *args: Any, **kwargs: Any 347 ) -> None: 348 self.name = self.__class__.__name__ 349 self.logger = logging.getLogger("hojichar.token_filters." + self.name) 350 assert 0 <= p <= 1 351 self.p = p 352 self.skip_rejected = skip_rejected
Initialize the filter.
Parameters
p : float
The probability of applying the filter.
If p is 1, the filter will always be applied.
skip_rejected : bool
If True, the filter will skip documents that are already rejected.
If you want to apply the filter to all documents (e.g., postprocess), set this to False.
random_state : int | np.random.Generator | None
Seed for the random number generator.
If None, a new random number generator will be created.
If None, and use in the Compose class, the random state is shared with the Compose object.
use_batch : bool
If True, the filter will process documents in batches in the apply_stream method.
batch_size : int
The size of the batch to process documents in the apply_stream method.
kwargs : Any
Additional keyword arguments to pass to the filter.
354 def apply(self, token: Token) -> Token: # type: ignore 355 raise NotImplementedError(f"{self.__class__.__name__}.apply method is not defined") 356 return token
Definition of filter behavior.
The document must have a protocol TextContent,
and mostly used hojichar.Document class.
In this method, the filter will modify document.text or
document.extras and set document.is_rejected = True to discard the document.
Parameters
document : Document Input document
Returns
Document Processed Document
376 def get_jsonable_vars(self, exclude_keys: set[str] | None = None) -> dict: 377 """ 378 Get the member variable of this filter. 379 Eligible variables are primitive types; [bool, int, float, str, None], 380 and the name of the variable not starts with the underscore; `_`. 381 """ 382 if exclude_keys is None: 383 exclude_keys = set() 384 return { 385 k: v 386 for k, v in vars(self).items() 387 if (_is_jsonable(v) and (k not in exclude_keys) and (not k.startswith("_"))) 388 }
Get the member variable of this filter.
Eligible variables are primitive types; [bool, int, float, str, None],
and the name of the variable not starts with the underscore; _.