hojichar.core.composition
1import json 2import logging 3import pprint 4from collections.abc import Iterable, Sequence 5from typing import Any 6 7import numpy as np 8 9from hojichar.core import inspection 10from hojichar.core.filter_interface import Filter, TokenFilter 11from hojichar.core.models import Document, Statistics, get_doc_info 12from hojichar.utils.warn_deprecation import deprecated_since 13 14 15class Compose(Filter): 16 def __init__( 17 self, 18 filters: list[Filter | TokenFilter], 19 random_state: int | np.random.Generator | None = None, 20 *args: Any, 21 **kwargs: Any, 22 ) -> None: 23 """ 24 Compose a filter from pre-defined filter-objects. 25 Filter which has `skip_rejected` flag ignores a document which has `is_rejected` flag. 26 By doing so, Compose avoid applying filters that do not affect the output. 27 28 Parameters 29 ---------- 30 filters : list[Filter | TokenFilter] 31 Filter instances which apply to the corpus. 32 33 random_state : int | np.random.Generator | None, optional 34 Default = None 35 Seed for applying filters randomly. 36 `random_state` must be int or np.random.Generator instance. 37 """ 38 super().__init__(random_state=random_state, *args, **kwargs) 39 self.set_filters(filters) 40 self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}") 41 42 self._statistics.name = "Total" 43 44 def set_filters(self, filters: list[Filter | TokenFilter]) -> None: 45 """ 46 Set the filter to a Compose object. The filter is expanded if the 47 list of filters in the argument contains a filter bound by Compose. 48 49 Args: 50 filters (list[Filter | TokenFilter]): Target filters 51 """ 52 self.filters: list[Filter | TokenFilter] = [] 53 54 filter_idx = 0 55 for f in filters: 56 if isinstance(f, Compose): 57 for sub in f.filters: 58 sub._set_rng_if_not_initialized(self._rng) 59 name = f"{filter_idx}-{sub.__class__.__name__}" 60 sub.name = name 61 sub._statistics.name = name 62 self.filters.append(sub) 63 filter_idx += 1 64 else: 65 f._set_rng_if_not_initialized(self._rng) 66 name = f"{filter_idx}-{f.__class__.__name__}" 67 f.name = name 68 f._statistics.name = name 69 self.filters.append(f) 70 filter_idx += 1 71 72 def __call__(self, text: str, **kwargs: Any) -> str: 73 """ 74 Apply the composed filter to a text and return the processed text. 75 If the document is rejected, return an empty string. 76 """ 77 document = Document(text, **kwargs) 78 document = self.apply(document) 79 if document.is_rejected: 80 return "" 81 else: 82 return document.text 83 84 def apply(self, document: Document) -> Document: 85 """ 86 Apply the composed filter to a document and return the processed document. 87 """ 88 stat = get_doc_info(document) 89 for i, filt in enumerate(self.filters): 90 document = filt._apply(document) 91 new_stat = get_doc_info(document) 92 self._statistics.update_by_diff(stat, new_stat) 93 return document 94 95 def apply_batch(self, batch: Sequence[Document]) -> list[Document]: 96 """ 97 Apply the composed filter to a batch of documents and return the processed documents. 98 The `apply_batch` method implemented in sub-filters is called in order. 99 """ 100 101 stats = [get_doc_info(doc) for doc in batch] 102 for i, filt in enumerate(self.filters): 103 batch = filt._apply_batch(batch) 104 batch = self._finalize_batch(batch, stats) 105 return list(batch) 106 107 def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]: 108 """ 109 Apply the composed filter to a stream of documents and return the processed documents. 110 The `apply_stream` method implemented in sub-filters is called in order. 111 112 113 In a sub-filter, if `apply_batch` is overridden and implemented, you need to set `use_batch` 114 to True at that filter to utilize that implementation. Otherwise, the 115 method implemented in `apply` will be applied to the stream. 116 """ 117 stream = self._count_input_stats(stream) 118 for i, filt in enumerate(self.filters): 119 stream = filt.apply_stream(stream) 120 121 for doc in stream: 122 in_stat = doc._get_initial_stats() 123 if in_stat is None: 124 in_stat = get_doc_info(doc) 125 self.logger.debug( 126 "Initial stats missing for document during stream aggregation; " 127 "using current stats as fallback" 128 ) 129 out_stat = get_doc_info(doc) 130 131 self._statistics.update_by_diff(in_stat, out_stat) 132 doc._clear_initial_stats() 133 yield doc 134 135 def _count_input_stats(self, stream: Iterable[Document]) -> Iterable[Document]: 136 for doc in stream: 137 doc._set_initial_stats(get_doc_info(doc)) 138 yield doc 139 140 def get_total_statistics(self) -> list[Statistics]: 141 """ 142 Get the statistics of the Compose object and sub filters. 143 144 The statistics of the Compose class are stored in an object with the name "Total", 145 and sub-filters's are stored with names in the format {filter_index}-{filter class name}. 146 """ 147 stats = [] 148 stats.append(self.get_statistics()) 149 for i, filt in enumerate(self.filters): 150 stats.append(filt.get_statistics()) 151 return stats 152 153 def get_total_statistics_map(self) -> list[dict[str, Any]]: 154 """ 155 Get the statistics of the Compose object and sub filters as a list of dictionaries. 156 """ 157 stats = self.get_total_statistics() 158 return [stat.to_dict() for stat in stats] 159 160 def shutdown(self) -> None: 161 for f in self.filters: 162 f.shutdown() 163 164 super().shutdown() 165 166 @property 167 def statistics(self) -> dict: 168 """ 169 Deprecated 170 171 Get the statistics of the Compose object and sub filters. 172 173 This property is retained for compatibility with previous versions. 174 Please use `get_total_statistics` or `get_total_statistics_map` instead. 175 """ 176 return inspection.statistics_obj_adapter( # type: ignore 177 self.get_total_statistics() 178 ).get_human_readable_values() 179 180 @property 181 def statistics_obj(self) -> inspection.StatsContainer: 182 """ 183 Deprecated 184 185 Get the statistics of the Compose object and sub filters. 186 This method returns a StatsContainer object which contains the statistics 187 of the Compose object and sub filters. 188 189 This property is retained for compatibility with previous versions. 190 Please use `get_total_statistics` or `get_total_statistics_map` instead. 191 """ 192 return inspection.statistics_obj_adapter(self.get_total_statistics()) # type: ignore 193 194 @deprecated_since("1.0.0", "get_total_statistics") 195 def summary(self, format: str = "print") -> None: 196 info = [ 197 { 198 "layer": i, 199 "name": filt.name, 200 "doc": filt.__doc__, 201 } 202 for i, filt in enumerate(self.filters) 203 ] 204 205 def to_json(filter_info: dict) -> dict: 206 filter_info["doc"] = "".join(d.strip() for d in filter_info["doc"].split("\n")) 207 return filter_info 208 209 if format == "json": 210 print(json.dumps(list(map(to_json, info)), ensure_ascii=False, indent="\t")) 211 if format == "print": 212 for layer in info: 213 print(f"[{layer['layer']}] {layer['name']}") 214 pprint.pprint(layer["doc"])
16class Compose(Filter): 17 def __init__( 18 self, 19 filters: list[Filter | TokenFilter], 20 random_state: int | np.random.Generator | None = None, 21 *args: Any, 22 **kwargs: Any, 23 ) -> None: 24 """ 25 Compose a filter from pre-defined filter-objects. 26 Filter which has `skip_rejected` flag ignores a document which has `is_rejected` flag. 27 By doing so, Compose avoid applying filters that do not affect the output. 28 29 Parameters 30 ---------- 31 filters : list[Filter | TokenFilter] 32 Filter instances which apply to the corpus. 33 34 random_state : int | np.random.Generator | None, optional 35 Default = None 36 Seed for applying filters randomly. 37 `random_state` must be int or np.random.Generator instance. 38 """ 39 super().__init__(random_state=random_state, *args, **kwargs) 40 self.set_filters(filters) 41 self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}") 42 43 self._statistics.name = "Total" 44 45 def set_filters(self, filters: list[Filter | TokenFilter]) -> None: 46 """ 47 Set the filter to a Compose object. The filter is expanded if the 48 list of filters in the argument contains a filter bound by Compose. 49 50 Args: 51 filters (list[Filter | TokenFilter]): Target filters 52 """ 53 self.filters: list[Filter | TokenFilter] = [] 54 55 filter_idx = 0 56 for f in filters: 57 if isinstance(f, Compose): 58 for sub in f.filters: 59 sub._set_rng_if_not_initialized(self._rng) 60 name = f"{filter_idx}-{sub.__class__.__name__}" 61 sub.name = name 62 sub._statistics.name = name 63 self.filters.append(sub) 64 filter_idx += 1 65 else: 66 f._set_rng_if_not_initialized(self._rng) 67 name = f"{filter_idx}-{f.__class__.__name__}" 68 f.name = name 69 f._statistics.name = name 70 self.filters.append(f) 71 filter_idx += 1 72 73 def __call__(self, text: str, **kwargs: Any) -> str: 74 """ 75 Apply the composed filter to a text and return the processed text. 76 If the document is rejected, return an empty string. 77 """ 78 document = Document(text, **kwargs) 79 document = self.apply(document) 80 if document.is_rejected: 81 return "" 82 else: 83 return document.text 84 85 def apply(self, document: Document) -> Document: 86 """ 87 Apply the composed filter to a document and return the processed document. 88 """ 89 stat = get_doc_info(document) 90 for i, filt in enumerate(self.filters): 91 document = filt._apply(document) 92 new_stat = get_doc_info(document) 93 self._statistics.update_by_diff(stat, new_stat) 94 return document 95 96 def apply_batch(self, batch: Sequence[Document]) -> list[Document]: 97 """ 98 Apply the composed filter to a batch of documents and return the processed documents. 99 The `apply_batch` method implemented in sub-filters is called in order. 100 """ 101 102 stats = [get_doc_info(doc) for doc in batch] 103 for i, filt in enumerate(self.filters): 104 batch = filt._apply_batch(batch) 105 batch = self._finalize_batch(batch, stats) 106 return list(batch) 107 108 def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]: 109 """ 110 Apply the composed filter to a stream of documents and return the processed documents. 111 The `apply_stream` method implemented in sub-filters is called in order. 112 113 114 In a sub-filter, if `apply_batch` is overridden and implemented, you need to set `use_batch` 115 to True at that filter to utilize that implementation. Otherwise, the 116 method implemented in `apply` will be applied to the stream. 117 """ 118 stream = self._count_input_stats(stream) 119 for i, filt in enumerate(self.filters): 120 stream = filt.apply_stream(stream) 121 122 for doc in stream: 123 in_stat = doc._get_initial_stats() 124 if in_stat is None: 125 in_stat = get_doc_info(doc) 126 self.logger.debug( 127 "Initial stats missing for document during stream aggregation; " 128 "using current stats as fallback" 129 ) 130 out_stat = get_doc_info(doc) 131 132 self._statistics.update_by_diff(in_stat, out_stat) 133 doc._clear_initial_stats() 134 yield doc 135 136 def _count_input_stats(self, stream: Iterable[Document]) -> Iterable[Document]: 137 for doc in stream: 138 doc._set_initial_stats(get_doc_info(doc)) 139 yield doc 140 141 def get_total_statistics(self) -> list[Statistics]: 142 """ 143 Get the statistics of the Compose object and sub filters. 144 145 The statistics of the Compose class are stored in an object with the name "Total", 146 and sub-filters's are stored with names in the format {filter_index}-{filter class name}. 147 """ 148 stats = [] 149 stats.append(self.get_statistics()) 150 for i, filt in enumerate(self.filters): 151 stats.append(filt.get_statistics()) 152 return stats 153 154 def get_total_statistics_map(self) -> list[dict[str, Any]]: 155 """ 156 Get the statistics of the Compose object and sub filters as a list of dictionaries. 157 """ 158 stats = self.get_total_statistics() 159 return [stat.to_dict() for stat in stats] 160 161 def shutdown(self) -> None: 162 for f in self.filters: 163 f.shutdown() 164 165 super().shutdown() 166 167 @property 168 def statistics(self) -> dict: 169 """ 170 Deprecated 171 172 Get the statistics of the Compose object and sub filters. 173 174 This property is retained for compatibility with previous versions. 175 Please use `get_total_statistics` or `get_total_statistics_map` instead. 176 """ 177 return inspection.statistics_obj_adapter( # type: ignore 178 self.get_total_statistics() 179 ).get_human_readable_values() 180 181 @property 182 def statistics_obj(self) -> inspection.StatsContainer: 183 """ 184 Deprecated 185 186 Get the statistics of the Compose object and sub filters. 187 This method returns a StatsContainer object which contains the statistics 188 of the Compose object and sub filters. 189 190 This property is retained for compatibility with previous versions. 191 Please use `get_total_statistics` or `get_total_statistics_map` instead. 192 """ 193 return inspection.statistics_obj_adapter(self.get_total_statistics()) # type: ignore 194 195 @deprecated_since("1.0.0", "get_total_statistics") 196 def summary(self, format: str = "print") -> None: 197 info = [ 198 { 199 "layer": i, 200 "name": filt.name, 201 "doc": filt.__doc__, 202 } 203 for i, filt in enumerate(self.filters) 204 ] 205 206 def to_json(filter_info: dict) -> dict: 207 filter_info["doc"] = "".join(d.strip() for d in filter_info["doc"].split("\n")) 208 return filter_info 209 210 if format == "json": 211 print(json.dumps(list(map(to_json, info)), ensure_ascii=False, indent="\t")) 212 if format == "print": 213 for layer in info: 214 print(f"[{layer['layer']}] {layer['name']}") 215 pprint.pprint(layer["doc"])
Base class for all filters. Document-level filters must inherit from this class.
The definition of text processing is in apply method.
If you define a new filter, override the method.
When this class is called, apply the filter from string to string.
With context manager, you can use the filter as follows:
with YourFilter(p=0.5) as filt:
text = filt("This is a sample text.")
17 def __init__( 18 self, 19 filters: list[Filter | TokenFilter], 20 random_state: int | np.random.Generator | None = None, 21 *args: Any, 22 **kwargs: Any, 23 ) -> None: 24 """ 25 Compose a filter from pre-defined filter-objects. 26 Filter which has `skip_rejected` flag ignores a document which has `is_rejected` flag. 27 By doing so, Compose avoid applying filters that do not affect the output. 28 29 Parameters 30 ---------- 31 filters : list[Filter | TokenFilter] 32 Filter instances which apply to the corpus. 33 34 random_state : int | np.random.Generator | None, optional 35 Default = None 36 Seed for applying filters randomly. 37 `random_state` must be int or np.random.Generator instance. 38 """ 39 super().__init__(random_state=random_state, *args, **kwargs) 40 self.set_filters(filters) 41 self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}") 42 43 self._statistics.name = "Total"
Compose a filter from pre-defined filter-objects.
Filter which has skip_rejected flag ignores a document which has is_rejected flag.
By doing so, Compose avoid applying filters that do not affect the output.
Parameters
filters : list[Filter | TokenFilter] Filter instances which apply to the corpus.
random_state : int | np.random.Generator | None, optional
Default = None
Seed for applying filters randomly.
random_state must be int or np.random.Generator instance.
45 def set_filters(self, filters: list[Filter | TokenFilter]) -> None: 46 """ 47 Set the filter to a Compose object. The filter is expanded if the 48 list of filters in the argument contains a filter bound by Compose. 49 50 Args: 51 filters (list[Filter | TokenFilter]): Target filters 52 """ 53 self.filters: list[Filter | TokenFilter] = [] 54 55 filter_idx = 0 56 for f in filters: 57 if isinstance(f, Compose): 58 for sub in f.filters: 59 sub._set_rng_if_not_initialized(self._rng) 60 name = f"{filter_idx}-{sub.__class__.__name__}" 61 sub.name = name 62 sub._statistics.name = name 63 self.filters.append(sub) 64 filter_idx += 1 65 else: 66 f._set_rng_if_not_initialized(self._rng) 67 name = f"{filter_idx}-{f.__class__.__name__}" 68 f.name = name 69 f._statistics.name = name 70 self.filters.append(f) 71 filter_idx += 1
Set the filter to a Compose object. The filter is expanded if the list of filters in the argument contains a filter bound by Compose.
Args: filters (list[Filter | TokenFilter]): Target filters
85 def apply(self, document: Document) -> Document: 86 """ 87 Apply the composed filter to a document and return the processed document. 88 """ 89 stat = get_doc_info(document) 90 for i, filt in enumerate(self.filters): 91 document = filt._apply(document) 92 new_stat = get_doc_info(document) 93 self._statistics.update_by_diff(stat, new_stat) 94 return document
Apply the composed filter to a document and return the processed document.
96 def apply_batch(self, batch: Sequence[Document]) -> list[Document]: 97 """ 98 Apply the composed filter to a batch of documents and return the processed documents. 99 The `apply_batch` method implemented in sub-filters is called in order. 100 """ 101 102 stats = [get_doc_info(doc) for doc in batch] 103 for i, filt in enumerate(self.filters): 104 batch = filt._apply_batch(batch) 105 batch = self._finalize_batch(batch, stats) 106 return list(batch)
Apply the composed filter to a batch of documents and return the processed documents.
The apply_batch method implemented in sub-filters is called in order.
108 def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]: 109 """ 110 Apply the composed filter to a stream of documents and return the processed documents. 111 The `apply_stream` method implemented in sub-filters is called in order. 112 113 114 In a sub-filter, if `apply_batch` is overridden and implemented, you need to set `use_batch` 115 to True at that filter to utilize that implementation. Otherwise, the 116 method implemented in `apply` will be applied to the stream. 117 """ 118 stream = self._count_input_stats(stream) 119 for i, filt in enumerate(self.filters): 120 stream = filt.apply_stream(stream) 121 122 for doc in stream: 123 in_stat = doc._get_initial_stats() 124 if in_stat is None: 125 in_stat = get_doc_info(doc) 126 self.logger.debug( 127 "Initial stats missing for document during stream aggregation; " 128 "using current stats as fallback" 129 ) 130 out_stat = get_doc_info(doc) 131 132 self._statistics.update_by_diff(in_stat, out_stat) 133 doc._clear_initial_stats() 134 yield doc
Apply the composed filter to a stream of documents and return the processed documents.
The apply_stream method implemented in sub-filters is called in order.
In a sub-filter, if apply_batch is overridden and implemented, you need to set use_batch
to True at that filter to utilize that implementation. Otherwise, the
method implemented in apply will be applied to the stream.
141 def get_total_statistics(self) -> list[Statistics]: 142 """ 143 Get the statistics of the Compose object and sub filters. 144 145 The statistics of the Compose class are stored in an object with the name "Total", 146 and sub-filters's are stored with names in the format {filter_index}-{filter class name}. 147 """ 148 stats = [] 149 stats.append(self.get_statistics()) 150 for i, filt in enumerate(self.filters): 151 stats.append(filt.get_statistics()) 152 return stats
Get the statistics of the Compose object and sub filters.
The statistics of the Compose class are stored in an object with the name "Total", and sub-filters's are stored with names in the format {filter_index}-{filter class name}.
154 def get_total_statistics_map(self) -> list[dict[str, Any]]: 155 """ 156 Get the statistics of the Compose object and sub filters as a list of dictionaries. 157 """ 158 stats = self.get_total_statistics() 159 return [stat.to_dict() for stat in stats]
Get the statistics of the Compose object and sub filters as a list of dictionaries.
161 def shutdown(self) -> None: 162 for f in self.filters: 163 f.shutdown() 164 165 super().shutdown()
This method is called when the filter is no longer needed. You can override this method to release resources or perform cleanup tasks.
Deprecated
Get the statistics of the Compose object and sub filters.
This property is retained for compatibility with previous versions.
Please use get_total_statistics or get_total_statistics_map instead.
Deprecated
Get the statistics of the Compose object and sub filters. This method returns a StatsContainer object which contains the statistics of the Compose object and sub filters.
This property is retained for compatibility with previous versions.
Please use get_total_statistics or get_total_statistics_map instead.
195 @deprecated_since("1.0.0", "get_total_statistics") 196 def summary(self, format: str = "print") -> None: 197 info = [ 198 { 199 "layer": i, 200 "name": filt.name, 201 "doc": filt.__doc__, 202 } 203 for i, filt in enumerate(self.filters) 204 ] 205 206 def to_json(filter_info: dict) -> dict: 207 filter_info["doc"] = "".join(d.strip() for d in filter_info["doc"].split("\n")) 208 return filter_info 209 210 if format == "json": 211 print(json.dumps(list(map(to_json, info)), ensure_ascii=False, indent="\t")) 212 if format == "print": 213 for layer in info: 214 print(f"[{layer['layer']}] {layer['name']}") 215 pprint.pprint(layer["doc"])