hojichar.core.composition

  1import json
  2import logging
  3import pprint
  4from collections.abc import Iterable, Sequence
  5from typing import Any
  6
  7import numpy as np
  8
  9from hojichar.core import inspection
 10from hojichar.core.filter_interface import Filter, TokenFilter
 11from hojichar.core.models import Document, Statistics, get_doc_info
 12from hojichar.utils.warn_deprecation import deprecated_since
 13
 14
 15class Compose(Filter):
 16    def __init__(
 17        self,
 18        filters: list[Filter | TokenFilter],
 19        random_state: int | np.random.Generator | None = None,
 20        *args: Any,
 21        **kwargs: Any,
 22    ) -> None:
 23        """
 24        Compose a filter from pre-defined filter-objects.
 25        Filter which has `skip_rejected` flag ignores a document which has `is_rejected` flag.
 26        By doing so, Compose avoid applying filters that do not affect the output.
 27
 28        Parameters
 29        ----------
 30        filters : list[Filter | TokenFilter]
 31            Filter instances which apply to the corpus.
 32
 33        random_state : int | np.random.Generator | None, optional
 34            Default = None
 35            Seed for applying filters randomly.
 36            `random_state` must be int or np.random.Generator instance.
 37        """
 38        super().__init__(random_state=random_state, *args, **kwargs)
 39        self.set_filters(filters)
 40        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
 41
 42        self._statistics.name = "Total"
 43
 44    def set_filters(self, filters: list[Filter | TokenFilter]) -> None:
 45        """
 46        Set the filter to a Compose object. The filter is expanded if the
 47        list of filters in the argument contains a filter bound by Compose.
 48
 49        Args:
 50            filters (list[Filter | TokenFilter]): Target filters
 51        """
 52        self.filters: list[Filter | TokenFilter] = []
 53
 54        filter_idx = 0
 55        for f in filters:
 56            if isinstance(f, Compose):
 57                for sub in f.filters:
 58                    sub._set_rng_if_not_initialized(self._rng)
 59                    name = f"{filter_idx}-{sub.__class__.__name__}"
 60                    sub.name = name
 61                    sub._statistics.name = name
 62                    self.filters.append(sub)
 63                    filter_idx += 1
 64            else:
 65                f._set_rng_if_not_initialized(self._rng)
 66                name = f"{filter_idx}-{f.__class__.__name__}"
 67                f.name = name
 68                f._statistics.name = name
 69                self.filters.append(f)
 70                filter_idx += 1
 71
 72    def __call__(self, text: str, **kwargs: Any) -> str:
 73        """
 74        Apply the composed filter to a text and return the processed text.
 75        If the document is rejected, return an empty string.
 76        """
 77        document = Document(text, **kwargs)
 78        document = self.apply(document)
 79        if document.is_rejected:
 80            return ""
 81        else:
 82            return document.text
 83
 84    def apply(self, document: Document) -> Document:
 85        """
 86        Apply the composed filter to a document and return the processed document.
 87        """
 88        stat = get_doc_info(document)
 89        for i, filt in enumerate(self.filters):
 90            document = filt._apply(document)
 91        new_stat = get_doc_info(document)
 92        self._statistics.update_by_diff(stat, new_stat)
 93        return document
 94
 95    def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
 96        """
 97        Apply the composed filter to a batch of documents and return the processed documents.
 98        The `apply_batch` method implemented in sub-filters is called in order.
 99        """
100
101        stats = [get_doc_info(doc) for doc in batch]
102        for i, filt in enumerate(self.filters):
103            batch = filt._apply_batch(batch)
104        batch = self._finalize_batch(batch, stats)
105        return list(batch)
106
107    def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]:
108        """
109        Apply the composed filter to a stream of documents and return the processed documents.
110        The `apply_stream` method implemented in sub-filters is called in order.
111
112
113        In a sub-filter, if `apply_batch` is overridden and implemented, you need to set `use_batch`
114        to True at that filter to utilize that implementation. Otherwise, the
115        method implemented in `apply` will be applied to the stream.
116        """
117        stream = self._count_input_stats(stream)
118        for i, filt in enumerate(self.filters):
119            stream = filt.apply_stream(stream)
120
121        for doc in stream:
122            in_stat = doc._get_initial_stats()
123            if in_stat is None:
124                in_stat = get_doc_info(doc)
125                self.logger.debug(
126                    "Initial stats missing for document during stream aggregation; "
127                    "using current stats as fallback"
128                )
129            out_stat = get_doc_info(doc)
130
131            self._statistics.update_by_diff(in_stat, out_stat)
132            doc._clear_initial_stats()
133            yield doc
134
135    def _count_input_stats(self, stream: Iterable[Document]) -> Iterable[Document]:
136        for doc in stream:
137            doc._set_initial_stats(get_doc_info(doc))
138            yield doc
139
140    def get_total_statistics(self) -> list[Statistics]:
141        """
142        Get the statistics of the Compose object and sub filters.
143
144        The statistics of the Compose class are stored in an object with the name "Total",
145        and sub-filters's are stored with names in the format {filter_index}-{filter class name}.
146        """
147        stats = []
148        stats.append(self.get_statistics())
149        for i, filt in enumerate(self.filters):
150            stats.append(filt.get_statistics())
151        return stats
152
153    def get_total_statistics_map(self) -> list[dict[str, Any]]:
154        """
155        Get the statistics of the Compose object and sub filters as a list of dictionaries.
156        """
157        stats = self.get_total_statistics()
158        return [stat.to_dict() for stat in stats]
159
160    def shutdown(self) -> None:
161        for f in self.filters:
162            f.shutdown()
163
164        super().shutdown()
165
166    @property
167    def statistics(self) -> dict:
168        """
169        Deprecated
170
171        Get the statistics of the Compose object and sub filters.
172
173        This property is retained for compatibility with previous versions.
174        Please use `get_total_statistics` or `get_total_statistics_map` instead.
175        """
176        return inspection.statistics_obj_adapter(  # type: ignore
177            self.get_total_statistics()
178        ).get_human_readable_values()
179
180    @property
181    def statistics_obj(self) -> inspection.StatsContainer:
182        """
183        Deprecated
184
185        Get the statistics of the Compose object and sub filters.
186        This method returns a StatsContainer object which contains the statistics
187        of the Compose object and sub filters.
188
189        This property is retained for compatibility with previous versions.
190        Please use `get_total_statistics` or `get_total_statistics_map` instead.
191        """
192        return inspection.statistics_obj_adapter(self.get_total_statistics())  # type: ignore
193
194    @deprecated_since("1.0.0", "get_total_statistics")
195    def summary(self, format: str = "print") -> None:
196        info = [
197            {
198                "layer": i,
199                "name": filt.name,
200                "doc": filt.__doc__,
201            }
202            for i, filt in enumerate(self.filters)
203        ]
204
205        def to_json(filter_info: dict) -> dict:
206            filter_info["doc"] = "".join(d.strip() for d in filter_info["doc"].split("\n"))
207            return filter_info
208
209        if format == "json":
210            print(json.dumps(list(map(to_json, info)), ensure_ascii=False, indent="\t"))
211        if format == "print":
212            for layer in info:
213                print(f"[{layer['layer']}] {layer['name']}")
214                pprint.pprint(layer["doc"])
class Compose(hojichar.core.filter_interface.Filter):
 16class Compose(Filter):
 17    def __init__(
 18        self,
 19        filters: list[Filter | TokenFilter],
 20        random_state: int | np.random.Generator | None = None,
 21        *args: Any,
 22        **kwargs: Any,
 23    ) -> None:
 24        """
 25        Compose a filter from pre-defined filter-objects.
 26        Filter which has `skip_rejected` flag ignores a document which has `is_rejected` flag.
 27        By doing so, Compose avoid applying filters that do not affect the output.
 28
 29        Parameters
 30        ----------
 31        filters : list[Filter | TokenFilter]
 32            Filter instances which apply to the corpus.
 33
 34        random_state : int | np.random.Generator | None, optional
 35            Default = None
 36            Seed for applying filters randomly.
 37            `random_state` must be int or np.random.Generator instance.
 38        """
 39        super().__init__(random_state=random_state, *args, **kwargs)
 40        self.set_filters(filters)
 41        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
 42
 43        self._statistics.name = "Total"
 44
 45    def set_filters(self, filters: list[Filter | TokenFilter]) -> None:
 46        """
 47        Set the filter to a Compose object. The filter is expanded if the
 48        list of filters in the argument contains a filter bound by Compose.
 49
 50        Args:
 51            filters (list[Filter | TokenFilter]): Target filters
 52        """
 53        self.filters: list[Filter | TokenFilter] = []
 54
 55        filter_idx = 0
 56        for f in filters:
 57            if isinstance(f, Compose):
 58                for sub in f.filters:
 59                    sub._set_rng_if_not_initialized(self._rng)
 60                    name = f"{filter_idx}-{sub.__class__.__name__}"
 61                    sub.name = name
 62                    sub._statistics.name = name
 63                    self.filters.append(sub)
 64                    filter_idx += 1
 65            else:
 66                f._set_rng_if_not_initialized(self._rng)
 67                name = f"{filter_idx}-{f.__class__.__name__}"
 68                f.name = name
 69                f._statistics.name = name
 70                self.filters.append(f)
 71                filter_idx += 1
 72
 73    def __call__(self, text: str, **kwargs: Any) -> str:
 74        """
 75        Apply the composed filter to a text and return the processed text.
 76        If the document is rejected, return an empty string.
 77        """
 78        document = Document(text, **kwargs)
 79        document = self.apply(document)
 80        if document.is_rejected:
 81            return ""
 82        else:
 83            return document.text
 84
 85    def apply(self, document: Document) -> Document:
 86        """
 87        Apply the composed filter to a document and return the processed document.
 88        """
 89        stat = get_doc_info(document)
 90        for i, filt in enumerate(self.filters):
 91            document = filt._apply(document)
 92        new_stat = get_doc_info(document)
 93        self._statistics.update_by_diff(stat, new_stat)
 94        return document
 95
 96    def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
 97        """
 98        Apply the composed filter to a batch of documents and return the processed documents.
 99        The `apply_batch` method implemented in sub-filters is called in order.
100        """
101
102        stats = [get_doc_info(doc) for doc in batch]
103        for i, filt in enumerate(self.filters):
104            batch = filt._apply_batch(batch)
105        batch = self._finalize_batch(batch, stats)
106        return list(batch)
107
108    def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]:
109        """
110        Apply the composed filter to a stream of documents and return the processed documents.
111        The `apply_stream` method implemented in sub-filters is called in order.
112
113
114        In a sub-filter, if `apply_batch` is overridden and implemented, you need to set `use_batch`
115        to True at that filter to utilize that implementation. Otherwise, the
116        method implemented in `apply` will be applied to the stream.
117        """
118        stream = self._count_input_stats(stream)
119        for i, filt in enumerate(self.filters):
120            stream = filt.apply_stream(stream)
121
122        for doc in stream:
123            in_stat = doc._get_initial_stats()
124            if in_stat is None:
125                in_stat = get_doc_info(doc)
126                self.logger.debug(
127                    "Initial stats missing for document during stream aggregation; "
128                    "using current stats as fallback"
129                )
130            out_stat = get_doc_info(doc)
131
132            self._statistics.update_by_diff(in_stat, out_stat)
133            doc._clear_initial_stats()
134            yield doc
135
136    def _count_input_stats(self, stream: Iterable[Document]) -> Iterable[Document]:
137        for doc in stream:
138            doc._set_initial_stats(get_doc_info(doc))
139            yield doc
140
141    def get_total_statistics(self) -> list[Statistics]:
142        """
143        Get the statistics of the Compose object and sub filters.
144
145        The statistics of the Compose class are stored in an object with the name "Total",
146        and sub-filters's are stored with names in the format {filter_index}-{filter class name}.
147        """
148        stats = []
149        stats.append(self.get_statistics())
150        for i, filt in enumerate(self.filters):
151            stats.append(filt.get_statistics())
152        return stats
153
154    def get_total_statistics_map(self) -> list[dict[str, Any]]:
155        """
156        Get the statistics of the Compose object and sub filters as a list of dictionaries.
157        """
158        stats = self.get_total_statistics()
159        return [stat.to_dict() for stat in stats]
160
161    def shutdown(self) -> None:
162        for f in self.filters:
163            f.shutdown()
164
165        super().shutdown()
166
167    @property
168    def statistics(self) -> dict:
169        """
170        Deprecated
171
172        Get the statistics of the Compose object and sub filters.
173
174        This property is retained for compatibility with previous versions.
175        Please use `get_total_statistics` or `get_total_statistics_map` instead.
176        """
177        return inspection.statistics_obj_adapter(  # type: ignore
178            self.get_total_statistics()
179        ).get_human_readable_values()
180
181    @property
182    def statistics_obj(self) -> inspection.StatsContainer:
183        """
184        Deprecated
185
186        Get the statistics of the Compose object and sub filters.
187        This method returns a StatsContainer object which contains the statistics
188        of the Compose object and sub filters.
189
190        This property is retained for compatibility with previous versions.
191        Please use `get_total_statistics` or `get_total_statistics_map` instead.
192        """
193        return inspection.statistics_obj_adapter(self.get_total_statistics())  # type: ignore
194
195    @deprecated_since("1.0.0", "get_total_statistics")
196    def summary(self, format: str = "print") -> None:
197        info = [
198            {
199                "layer": i,
200                "name": filt.name,
201                "doc": filt.__doc__,
202            }
203            for i, filt in enumerate(self.filters)
204        ]
205
206        def to_json(filter_info: dict) -> dict:
207            filter_info["doc"] = "".join(d.strip() for d in filter_info["doc"].split("\n"))
208            return filter_info
209
210        if format == "json":
211            print(json.dumps(list(map(to_json, info)), ensure_ascii=False, indent="\t"))
212        if format == "print":
213            for layer in info:
214                print(f"[{layer['layer']}] {layer['name']}")
215                pprint.pprint(layer["doc"])

Base class for all filters. Document-level filters must inherit from this class.

The definition of text processing is in apply method. If you define a new filter, override the method.

When this class is called, apply the filter from string to string.

With context manager, you can use the filter as follows:

with YourFilter(p=0.5) as filt:
    text = filt("This is a sample text.")
Compose( filters: list[hojichar.core.filter_interface.Filter | hojichar.core.filter_interface.TokenFilter], random_state: int | numpy.random._generator.Generator | None = None, *args: Any, **kwargs: Any)
17    def __init__(
18        self,
19        filters: list[Filter | TokenFilter],
20        random_state: int | np.random.Generator | None = None,
21        *args: Any,
22        **kwargs: Any,
23    ) -> None:
24        """
25        Compose a filter from pre-defined filter-objects.
26        Filter which has `skip_rejected` flag ignores a document which has `is_rejected` flag.
27        By doing so, Compose avoid applying filters that do not affect the output.
28
29        Parameters
30        ----------
31        filters : list[Filter | TokenFilter]
32            Filter instances which apply to the corpus.
33
34        random_state : int | np.random.Generator | None, optional
35            Default = None
36            Seed for applying filters randomly.
37            `random_state` must be int or np.random.Generator instance.
38        """
39        super().__init__(random_state=random_state, *args, **kwargs)
40        self.set_filters(filters)
41        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
42
43        self._statistics.name = "Total"

Compose a filter from pre-defined filter-objects. Filter which has skip_rejected flag ignores a document which has is_rejected flag. By doing so, Compose avoid applying filters that do not affect the output.

Parameters

filters : list[Filter | TokenFilter] Filter instances which apply to the corpus.

random_state : int | np.random.Generator | None, optional Default = None Seed for applying filters randomly. random_state must be int or np.random.Generator instance.

def set_filters( self, filters: list[hojichar.core.filter_interface.Filter | hojichar.core.filter_interface.TokenFilter]) -> None:
45    def set_filters(self, filters: list[Filter | TokenFilter]) -> None:
46        """
47        Set the filter to a Compose object. The filter is expanded if the
48        list of filters in the argument contains a filter bound by Compose.
49
50        Args:
51            filters (list[Filter | TokenFilter]): Target filters
52        """
53        self.filters: list[Filter | TokenFilter] = []
54
55        filter_idx = 0
56        for f in filters:
57            if isinstance(f, Compose):
58                for sub in f.filters:
59                    sub._set_rng_if_not_initialized(self._rng)
60                    name = f"{filter_idx}-{sub.__class__.__name__}"
61                    sub.name = name
62                    sub._statistics.name = name
63                    self.filters.append(sub)
64                    filter_idx += 1
65            else:
66                f._set_rng_if_not_initialized(self._rng)
67                name = f"{filter_idx}-{f.__class__.__name__}"
68                f.name = name
69                f._statistics.name = name
70                self.filters.append(f)
71                filter_idx += 1

Set the filter to a Compose object. The filter is expanded if the list of filters in the argument contains a filter bound by Compose.

Args: filters (list[Filter | TokenFilter]): Target filters

def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
85    def apply(self, document: Document) -> Document:
86        """
87        Apply the composed filter to a document and return the processed document.
88        """
89        stat = get_doc_info(document)
90        for i, filt in enumerate(self.filters):
91            document = filt._apply(document)
92        new_stat = get_doc_info(document)
93        self._statistics.update_by_diff(stat, new_stat)
94        return document

Apply the composed filter to a document and return the processed document.

def apply_batch( self, batch: collections.abc.Sequence[hojichar.core.models.Document]) -> list[hojichar.core.models.Document]:
 96    def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
 97        """
 98        Apply the composed filter to a batch of documents and return the processed documents.
 99        The `apply_batch` method implemented in sub-filters is called in order.
100        """
101
102        stats = [get_doc_info(doc) for doc in batch]
103        for i, filt in enumerate(self.filters):
104            batch = filt._apply_batch(batch)
105        batch = self._finalize_batch(batch, stats)
106        return list(batch)

Apply the composed filter to a batch of documents and return the processed documents. The apply_batch method implemented in sub-filters is called in order.

def apply_stream( self, stream: collections.abc.Iterable[hojichar.core.models.Document]) -> collections.abc.Iterable[hojichar.core.models.Document]:
108    def apply_stream(self, stream: Iterable[Document]) -> Iterable[Document]:
109        """
110        Apply the composed filter to a stream of documents and return the processed documents.
111        The `apply_stream` method implemented in sub-filters is called in order.
112
113
114        In a sub-filter, if `apply_batch` is overridden and implemented, you need to set `use_batch`
115        to True at that filter to utilize that implementation. Otherwise, the
116        method implemented in `apply` will be applied to the stream.
117        """
118        stream = self._count_input_stats(stream)
119        for i, filt in enumerate(self.filters):
120            stream = filt.apply_stream(stream)
121
122        for doc in stream:
123            in_stat = doc._get_initial_stats()
124            if in_stat is None:
125                in_stat = get_doc_info(doc)
126                self.logger.debug(
127                    "Initial stats missing for document during stream aggregation; "
128                    "using current stats as fallback"
129                )
130            out_stat = get_doc_info(doc)
131
132            self._statistics.update_by_diff(in_stat, out_stat)
133            doc._clear_initial_stats()
134            yield doc

Apply the composed filter to a stream of documents and return the processed documents. The apply_stream method implemented in sub-filters is called in order.

In a sub-filter, if apply_batch is overridden and implemented, you need to set use_batch to True at that filter to utilize that implementation. Otherwise, the method implemented in apply will be applied to the stream.

def get_total_statistics(self) -> list[hojichar.core.models.Statistics]:
141    def get_total_statistics(self) -> list[Statistics]:
142        """
143        Get the statistics of the Compose object and sub filters.
144
145        The statistics of the Compose class are stored in an object with the name "Total",
146        and sub-filters's are stored with names in the format {filter_index}-{filter class name}.
147        """
148        stats = []
149        stats.append(self.get_statistics())
150        for i, filt in enumerate(self.filters):
151            stats.append(filt.get_statistics())
152        return stats

Get the statistics of the Compose object and sub filters.

The statistics of the Compose class are stored in an object with the name "Total", and sub-filters's are stored with names in the format {filter_index}-{filter class name}.

def get_total_statistics_map(self) -> list[dict[str, typing.Any]]:
154    def get_total_statistics_map(self) -> list[dict[str, Any]]:
155        """
156        Get the statistics of the Compose object and sub filters as a list of dictionaries.
157        """
158        stats = self.get_total_statistics()
159        return [stat.to_dict() for stat in stats]

Get the statistics of the Compose object and sub filters as a list of dictionaries.

def shutdown(self) -> None:
161    def shutdown(self) -> None:
162        for f in self.filters:
163            f.shutdown()
164
165        super().shutdown()

This method is called when the filter is no longer needed. You can override this method to release resources or perform cleanup tasks.

statistics: dict

Deprecated

Get the statistics of the Compose object and sub filters.

This property is retained for compatibility with previous versions. Please use get_total_statistics or get_total_statistics_map instead.

Deprecated

Get the statistics of the Compose object and sub filters. This method returns a StatsContainer object which contains the statistics of the Compose object and sub filters.

This property is retained for compatibility with previous versions. Please use get_total_statistics or get_total_statistics_map instead.

@deprecated_since('1.0.0', 'get_total_statistics')
def summary(self, format: str = 'print') -> None:
195    @deprecated_since("1.0.0", "get_total_statistics")
196    def summary(self, format: str = "print") -> None:
197        info = [
198            {
199                "layer": i,
200                "name": filt.name,
201                "doc": filt.__doc__,
202            }
203            for i, filt in enumerate(self.filters)
204        ]
205
206        def to_json(filter_info: dict) -> dict:
207            filter_info["doc"] = "".join(d.strip() for d in filter_info["doc"].split("\n"))
208            return filter_info
209
210        if format == "json":
211            print(json.dumps(list(map(to_json, info)), ensure_ascii=False, indent="\t"))
212        if format == "print":
213            for layer in info:
214                print(f"[{layer['layer']}] {layer['name']}")
215                pprint.pprint(layer["doc"])