hojichar.core.async_composition

  1from __future__ import annotations
  2
  3import asyncio
  4import logging
  5from collections.abc import AsyncGenerator, AsyncIterable, Iterable, Sequence
  6from concurrent.futures import Executor, ThreadPoolExecutor
  7from typing import Any
  8
  9import numpy as np
 10
 11from hojichar.core import inspection
 12from hojichar.core.async_filter_interface import AsyncFilter
 13from hojichar.core.composition import Compose
 14from hojichar.core.filter_interface import Filter
 15from hojichar.core.models import Document, Statistics, get_doc_info
 16from hojichar.utils.async_handlers import (
 17    AsyncToSyncIterator,
 18    handle_async_stream_as_sync,
 19    handle_stream_as_async,
 20)
 21
 22
 23class AsyncFilterAdapter(AsyncFilter):
 24    """
 25    Adapter class for executing hojichar.Filter asynchronously.
 26    """
 27
 28    def __init__(
 29        self,
 30        sync_filter: Filter,
 31        *args: Any,
 32        executor: Executor | None = None,
 33        use_batch: bool = True,
 34        **kwargs: Any,
 35    ):
 36        """
 37        Adapter class for executing hojichar.Filter asynchronously.
 38        Used to incorporate Filter into AsyncCompose.
 39
 40        To reduce the overhead of asynchronous context switching,
 41        use_batch is set to True by default to process in batches
 42        in apply_stream, regardless of the sync_filter's use_batch setting.
 43
 44        If performing CPU-bound and heavy processing, you can specify an executor
 45        to offload the processing to the executor. However, due to Python's GIL
 46        constraints, using ThreadPoolExecutor will not parallelize CPU-bound
 47        processing, and the entire process will be locked.
 48
 49        By using ProcessPoolExecutor as the executor, it may be possible to
 50        parallelize CPU-bound processing. However, for parallelizing CPU-bound
 51        processing, it is recommended to use the hojichar.Parallel class to
 52        parallelize synchronous Compose pipeline.
 53        """
 54        super().__init__(*args, use_batch=use_batch, **kwargs)
 55        self.sync_filter = sync_filter
 56        self._has_external_executor = executor is not None
 57        self._executor = executor or ThreadPoolExecutor()
 58        self.batch_size = sync_filter.batch_size
 59
 60    async def apply(self, document: Document) -> Document:
 61        loop = asyncio.get_running_loop()
 62        return await loop.run_in_executor(self._executor, self.sync_filter.apply, document)
 63
 64    async def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
 65        loop = asyncio.get_running_loop()
 66        return await loop.run_in_executor(
 67            self._executor,
 68            lambda: self.sync_filter.apply_batch(batch),
 69        )
 70
 71    async def shutdown(self) -> None:
 72        self.sync_filter.shutdown()
 73        if not self._has_external_executor:
 74            self._executor.shutdown()
 75
 76
 77class AsyncCompose(AsyncFilter):
 78    def __init__(
 79        self,
 80        filters: list[AsyncFilter | Filter],
 81        random_state: int | np.random.Generator | None = None,
 82        executor: ThreadPoolExecutor | None = None,
 83        *args: Any,
 84        **kwargs: Any,
 85    ):
 86        super().__init__(random_state=random_state, *args, **kwargs)
 87        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
 88        self._statistics.name = "Total"
 89        self._has_external_executor = executor is not None
 90        self._executor = executor or ThreadPoolExecutor()
 91        self.set_filters(filters)
 92
 93    def set_filters(self, filters: list[AsyncFilter | Filter]) -> None:
 94        self.filters: list[AsyncFilter] = []
 95        filter_idx = 0
 96        for f in filters:
 97            if isinstance(f, (AsyncCompose, Compose)):
 98                for sub in f.filters:
 99                    name = f"{filter_idx}-{sub.__class__.__name__}"
100                    if isinstance(sub, Filter):
101                        name = f"{filter_idx}-{sub.__class__.__name__}"
102                        sub = AsyncFilterAdapter(sub, executor=self._executor)
103
104                    sub._set_rng_if_not_initialized(self._rng)
105                    sub.name = name
106                    sub._statistics.name = name
107                    self.filters.append(sub)
108                    filter_idx += 1
109            else:
110                name = f"{filter_idx}-{f.__class__.__name__}"
111                if isinstance(f, Filter):
112                    name = f"{filter_idx}-{f.__class__.__name__}"
113                    f = AsyncFilterAdapter(f, executor=self._executor)
114                f._set_rng_if_not_initialized(self._rng)
115                f.name = name
116                f._statistics.name = name
117                self.filters.append(f)
118                filter_idx += 1
119
120    async def apply(self, document: Document) -> Document:
121        stat = get_doc_info(document)
122        for filter_idx, filt in enumerate(self.filters):
123            document = await filt._apply(document)
124        new_stat = get_doc_info(document)
125        async with self._stats_lock:
126            self._statistics.update_by_diff(stat, new_stat)
127        return document
128
129    async def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
130        stats = [get_doc_info(doc) for doc in batch]
131        for i, filt in enumerate(self.filters):
132            batch = await filt._apply_batch(batch)
133        batch = await self._finalize_batch(batch, stats)
134        return list(batch)
135
136    async def apply_stream(
137        self,
138        stream: AsyncIterable[Document] | Iterable[Document],
139    ) -> AsyncGenerator[Document, None]:
140        async_stream = handle_stream_as_async(stream, chunk_size=1000, executor=self._executor)
141        async_stream = self._count_input_stats(async_stream)
142
143        for i, filt in enumerate(self.filters):
144            async_stream = filt.apply_stream(async_stream)
145
146        async for doc in async_stream:
147            in_stat = doc._get_initial_stats()
148            if in_stat is None:
149                in_stat = get_doc_info(doc)
150                self.logger.debug(
151                    "Initial stats missing for document during async stream aggregation; "
152                    "using current stats as fallback"
153                )
154            out_stat = get_doc_info(doc)
155            async with self._stats_lock:
156                self._statistics.update_by_diff(in_stat, out_stat)
157            doc._clear_initial_stats()
158            yield doc
159
160    def imap_apply(
161        self,
162        stream: AsyncIterable[Document] | Iterable[Document],
163        *,
164        buffer_size: int = 128,
165        shutdown: bool = True,
166    ) -> AsyncToSyncIterator[Document]:
167        """Synchronously consume this asynchronous pipeline.
168
169        The returned iterator runs the pipeline on a dedicated background event loop. It is
170        intended for one-shot use and shuts down the pipeline after consumption by default.
171        Use the iterator as a context manager if iteration may stop early.
172        """
173
174        finalizer = self.shutdown if shutdown else None
175        return handle_async_stream_as_sync(
176            self.apply_stream(stream),
177            buffer_size=buffer_size,
178            finalizer=finalizer,
179        )
180
181    async def _count_input_stats(
182        self, async_stream: AsyncIterable[Document]
183    ) -> AsyncGenerator[Document, None]:
184        async for doc in async_stream:
185            doc._set_initial_stats(get_doc_info(doc))
186            yield doc
187
188    def get_total_statistics(self) -> list[Statistics]:
189        """
190        Get the statistics of the Compose object and sub filters.
191
192        The statistics of the Compose class are stored in an object with the name "Total",
193        and sub-filters's are stored with names in the format {filter_index}-{filter class name}.
194        """
195        stats = []
196        stats.append(self.get_statistics())
197        for i, filt in enumerate(self.filters):
198            stats.append(filt.get_statistics())
199        return stats
200
201    def get_total_statistics_map(self) -> list[dict[str, Any]]:
202        """
203        Get the statistics of the Compose object and sub filters as a list of dictionaries.
204        """
205        stats = self.get_total_statistics()
206        return [stat.to_dict() for stat in stats]
207
208    @property
209    def statistics(self) -> dict:
210        """
211        Deprecated
212
213        Get the statistics of the Compose object and sub filters.
214
215        This property is retained for compatibility with previous versions.
216        Please use `get_total_statistics` or `get_total_statistics_map` instead.
217        """
218        return inspection.statistics_obj_adapter(  # type: ignore
219            self.get_total_statistics()
220        ).get_human_readable_values()
221
222    @property
223    def statistics_obj(self) -> inspection.StatsContainer:
224        """
225        Deprecated
226
227        Get the statistics of the AsyncCompose object and sub filters.
228        This method returns a StatsContainer object which contains the statistics
229        of the AsyncCompose object and sub filters.
230
231        This property is retained for compatibility with previous versions.
232        Please use `get_total_statistics` or `get_total_statistics_map` instead.
233        """
234        return inspection.statistics_obj_adapter(self.get_total_statistics())  # type: ignore
235
236    async def shutdown(self) -> None:
237        for filt in self.filters:
238            await filt.shutdown()
239        if not self._has_external_executor:
240            self._executor.shutdown()
241
242    async def __aenter__(self) -> "AsyncCompose":
243        return self
244
245    async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
246        await self.shutdown()
247        if exc_type is not None:
248            raise exc_value
class AsyncFilterAdapter(hojichar.core.async_filter_interface.AsyncFilter):
24class AsyncFilterAdapter(AsyncFilter):
25    """
26    Adapter class for executing hojichar.Filter asynchronously.
27    """
28
29    def __init__(
30        self,
31        sync_filter: Filter,
32        *args: Any,
33        executor: Executor | None = None,
34        use_batch: bool = True,
35        **kwargs: Any,
36    ):
37        """
38        Adapter class for executing hojichar.Filter asynchronously.
39        Used to incorporate Filter into AsyncCompose.
40
41        To reduce the overhead of asynchronous context switching,
42        use_batch is set to True by default to process in batches
43        in apply_stream, regardless of the sync_filter's use_batch setting.
44
45        If performing CPU-bound and heavy processing, you can specify an executor
46        to offload the processing to the executor. However, due to Python's GIL
47        constraints, using ThreadPoolExecutor will not parallelize CPU-bound
48        processing, and the entire process will be locked.
49
50        By using ProcessPoolExecutor as the executor, it may be possible to
51        parallelize CPU-bound processing. However, for parallelizing CPU-bound
52        processing, it is recommended to use the hojichar.Parallel class to
53        parallelize synchronous Compose pipeline.
54        """
55        super().__init__(*args, use_batch=use_batch, **kwargs)
56        self.sync_filter = sync_filter
57        self._has_external_executor = executor is not None
58        self._executor = executor or ThreadPoolExecutor()
59        self.batch_size = sync_filter.batch_size
60
61    async def apply(self, document: Document) -> Document:
62        loop = asyncio.get_running_loop()
63        return await loop.run_in_executor(self._executor, self.sync_filter.apply, document)
64
65    async def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
66        loop = asyncio.get_running_loop()
67        return await loop.run_in_executor(
68            self._executor,
69            lambda: self.sync_filter.apply_batch(batch),
70        )
71
72    async def shutdown(self) -> None:
73        self.sync_filter.shutdown()
74        if not self._has_external_executor:
75            self._executor.shutdown()

Adapter class for executing hojichar.Filter asynchronously.

AsyncFilterAdapter( sync_filter: hojichar.core.filter_interface.Filter, *args: Any, executor: concurrent.futures._base.Executor | None = None, use_batch: bool = True, **kwargs: Any)
29    def __init__(
30        self,
31        sync_filter: Filter,
32        *args: Any,
33        executor: Executor | None = None,
34        use_batch: bool = True,
35        **kwargs: Any,
36    ):
37        """
38        Adapter class for executing hojichar.Filter asynchronously.
39        Used to incorporate Filter into AsyncCompose.
40
41        To reduce the overhead of asynchronous context switching,
42        use_batch is set to True by default to process in batches
43        in apply_stream, regardless of the sync_filter's use_batch setting.
44
45        If performing CPU-bound and heavy processing, you can specify an executor
46        to offload the processing to the executor. However, due to Python's GIL
47        constraints, using ThreadPoolExecutor will not parallelize CPU-bound
48        processing, and the entire process will be locked.
49
50        By using ProcessPoolExecutor as the executor, it may be possible to
51        parallelize CPU-bound processing. However, for parallelizing CPU-bound
52        processing, it is recommended to use the hojichar.Parallel class to
53        parallelize synchronous Compose pipeline.
54        """
55        super().__init__(*args, use_batch=use_batch, **kwargs)
56        self.sync_filter = sync_filter
57        self._has_external_executor = executor is not None
58        self._executor = executor or ThreadPoolExecutor()
59        self.batch_size = sync_filter.batch_size

Adapter class for executing hojichar.Filter asynchronously. Used to incorporate Filter into AsyncCompose.

To reduce the overhead of asynchronous context switching, use_batch is set to True by default to process in batches in apply_stream, regardless of the sync_filter's use_batch setting.

If performing CPU-bound and heavy processing, you can specify an executor to offload the processing to the executor. However, due to Python's GIL constraints, using ThreadPoolExecutor will not parallelize CPU-bound processing, and the entire process will be locked.

By using ProcessPoolExecutor as the executor, it may be possible to parallelize CPU-bound processing. However, for parallelizing CPU-bound processing, it is recommended to use the hojichar.Parallel class to parallelize synchronous Compose pipeline.

async def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
61    async def apply(self, document: Document) -> Document:
62        loop = asyncio.get_running_loop()
63        return await loop.run_in_executor(self._executor, self.sync_filter.apply, document)

Definition of async filter behavior.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

async def apply_batch( self, batch: collections.abc.Sequence[hojichar.core.models.Document]) -> list[hojichar.core.models.Document]:
65    async def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
66        loop = asyncio.get_running_loop()
67        return await loop.run_in_executor(
68            self._executor,
69            lambda: self.sync_filter.apply_batch(batch),
70        )

Apply the filter to a Sequence of documents. By default, the processing implemented in apply is executed asynchronously and concurrently. If the filter processing can be optimized for batch processing, override this method.

async def shutdown(self) -> None:
72    async def shutdown(self) -> None:
73        self.sync_filter.shutdown()
74        if not self._has_external_executor:
75            self._executor.shutdown()

You can override this method to release resources or perform cleanup tasks.

class AsyncCompose(hojichar.core.async_filter_interface.AsyncFilter):
 78class AsyncCompose(AsyncFilter):
 79    def __init__(
 80        self,
 81        filters: list[AsyncFilter | Filter],
 82        random_state: int | np.random.Generator | None = None,
 83        executor: ThreadPoolExecutor | None = None,
 84        *args: Any,
 85        **kwargs: Any,
 86    ):
 87        super().__init__(random_state=random_state, *args, **kwargs)
 88        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
 89        self._statistics.name = "Total"
 90        self._has_external_executor = executor is not None
 91        self._executor = executor or ThreadPoolExecutor()
 92        self.set_filters(filters)
 93
 94    def set_filters(self, filters: list[AsyncFilter | Filter]) -> None:
 95        self.filters: list[AsyncFilter] = []
 96        filter_idx = 0
 97        for f in filters:
 98            if isinstance(f, (AsyncCompose, Compose)):
 99                for sub in f.filters:
100                    name = f"{filter_idx}-{sub.__class__.__name__}"
101                    if isinstance(sub, Filter):
102                        name = f"{filter_idx}-{sub.__class__.__name__}"
103                        sub = AsyncFilterAdapter(sub, executor=self._executor)
104
105                    sub._set_rng_if_not_initialized(self._rng)
106                    sub.name = name
107                    sub._statistics.name = name
108                    self.filters.append(sub)
109                    filter_idx += 1
110            else:
111                name = f"{filter_idx}-{f.__class__.__name__}"
112                if isinstance(f, Filter):
113                    name = f"{filter_idx}-{f.__class__.__name__}"
114                    f = AsyncFilterAdapter(f, executor=self._executor)
115                f._set_rng_if_not_initialized(self._rng)
116                f.name = name
117                f._statistics.name = name
118                self.filters.append(f)
119                filter_idx += 1
120
121    async def apply(self, document: Document) -> Document:
122        stat = get_doc_info(document)
123        for filter_idx, filt in enumerate(self.filters):
124            document = await filt._apply(document)
125        new_stat = get_doc_info(document)
126        async with self._stats_lock:
127            self._statistics.update_by_diff(stat, new_stat)
128        return document
129
130    async def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
131        stats = [get_doc_info(doc) for doc in batch]
132        for i, filt in enumerate(self.filters):
133            batch = await filt._apply_batch(batch)
134        batch = await self._finalize_batch(batch, stats)
135        return list(batch)
136
137    async def apply_stream(
138        self,
139        stream: AsyncIterable[Document] | Iterable[Document],
140    ) -> AsyncGenerator[Document, None]:
141        async_stream = handle_stream_as_async(stream, chunk_size=1000, executor=self._executor)
142        async_stream = self._count_input_stats(async_stream)
143
144        for i, filt in enumerate(self.filters):
145            async_stream = filt.apply_stream(async_stream)
146
147        async for doc in async_stream:
148            in_stat = doc._get_initial_stats()
149            if in_stat is None:
150                in_stat = get_doc_info(doc)
151                self.logger.debug(
152                    "Initial stats missing for document during async stream aggregation; "
153                    "using current stats as fallback"
154                )
155            out_stat = get_doc_info(doc)
156            async with self._stats_lock:
157                self._statistics.update_by_diff(in_stat, out_stat)
158            doc._clear_initial_stats()
159            yield doc
160
161    def imap_apply(
162        self,
163        stream: AsyncIterable[Document] | Iterable[Document],
164        *,
165        buffer_size: int = 128,
166        shutdown: bool = True,
167    ) -> AsyncToSyncIterator[Document]:
168        """Synchronously consume this asynchronous pipeline.
169
170        The returned iterator runs the pipeline on a dedicated background event loop. It is
171        intended for one-shot use and shuts down the pipeline after consumption by default.
172        Use the iterator as a context manager if iteration may stop early.
173        """
174
175        finalizer = self.shutdown if shutdown else None
176        return handle_async_stream_as_sync(
177            self.apply_stream(stream),
178            buffer_size=buffer_size,
179            finalizer=finalizer,
180        )
181
182    async def _count_input_stats(
183        self, async_stream: AsyncIterable[Document]
184    ) -> AsyncGenerator[Document, None]:
185        async for doc in async_stream:
186            doc._set_initial_stats(get_doc_info(doc))
187            yield doc
188
189    def get_total_statistics(self) -> list[Statistics]:
190        """
191        Get the statistics of the Compose object and sub filters.
192
193        The statistics of the Compose class are stored in an object with the name "Total",
194        and sub-filters's are stored with names in the format {filter_index}-{filter class name}.
195        """
196        stats = []
197        stats.append(self.get_statistics())
198        for i, filt in enumerate(self.filters):
199            stats.append(filt.get_statistics())
200        return stats
201
202    def get_total_statistics_map(self) -> list[dict[str, Any]]:
203        """
204        Get the statistics of the Compose object and sub filters as a list of dictionaries.
205        """
206        stats = self.get_total_statistics()
207        return [stat.to_dict() for stat in stats]
208
209    @property
210    def statistics(self) -> dict:
211        """
212        Deprecated
213
214        Get the statistics of the Compose object and sub filters.
215
216        This property is retained for compatibility with previous versions.
217        Please use `get_total_statistics` or `get_total_statistics_map` instead.
218        """
219        return inspection.statistics_obj_adapter(  # type: ignore
220            self.get_total_statistics()
221        ).get_human_readable_values()
222
223    @property
224    def statistics_obj(self) -> inspection.StatsContainer:
225        """
226        Deprecated
227
228        Get the statistics of the AsyncCompose object and sub filters.
229        This method returns a StatsContainer object which contains the statistics
230        of the AsyncCompose object and sub filters.
231
232        This property is retained for compatibility with previous versions.
233        Please use `get_total_statistics` or `get_total_statistics_map` instead.
234        """
235        return inspection.statistics_obj_adapter(self.get_total_statistics())  # type: ignore
236
237    async def shutdown(self) -> None:
238        for filt in self.filters:
239            await filt.shutdown()
240        if not self._has_external_executor:
241            self._executor.shutdown()
242
243    async def __aenter__(self) -> "AsyncCompose":
244        return self
245
246    async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
247        await self.shutdown()
248        if exc_type is not None:
249            raise exc_value

Helper class that provides a standard way to create an ABC using inheritance.

AsyncCompose( filters: list[hojichar.core.async_filter_interface.AsyncFilter | hojichar.core.filter_interface.Filter], random_state: int | numpy.random._generator.Generator | None = None, executor: concurrent.futures.thread.ThreadPoolExecutor | None = None, *args: Any, **kwargs: Any)
79    def __init__(
80        self,
81        filters: list[AsyncFilter | Filter],
82        random_state: int | np.random.Generator | None = None,
83        executor: ThreadPoolExecutor | None = None,
84        *args: Any,
85        **kwargs: Any,
86    ):
87        super().__init__(random_state=random_state, *args, **kwargs)
88        self.logger = logging.getLogger(f"{self.__module__}.{self.__class__.__name__}")
89        self._statistics.name = "Total"
90        self._has_external_executor = executor is not None
91        self._executor = executor or ThreadPoolExecutor()
92        self.set_filters(filters)

Base class for asynchronous filters.

Parameters

p : float The probability of applying the filter. If p is 1, the filter will always be applied. skip_rejected : bool If True, the filter will skip documents that are already rejected. If you want to apply the filter to all documents (e.g., postprocess), set this to False. random_state : int | np.random.Generator | None Seed for the random number generator. If None is specified, the random number generator managed by the Compose class will be used. use_batch : bool If True, the filter will process documents in batches in the apply_stream method. batch_size : int The size of the batch to process documents in the apply_stream method. When apply_batch is not overridden, this is also the maximum number of in-flight document tasks used by the sliding-window scheduler. Ordered processing applies backpressure after at most two windows of started but not yet yielded documents. ordered : bool If True, apply_stream yields documents in input order. If False, filters using the default apply_batch implementation yield documents as soon as their processing completes.

def set_filters( self, filters: list[hojichar.core.async_filter_interface.AsyncFilter | hojichar.core.filter_interface.Filter]) -> None:
 94    def set_filters(self, filters: list[AsyncFilter | Filter]) -> None:
 95        self.filters: list[AsyncFilter] = []
 96        filter_idx = 0
 97        for f in filters:
 98            if isinstance(f, (AsyncCompose, Compose)):
 99                for sub in f.filters:
100                    name = f"{filter_idx}-{sub.__class__.__name__}"
101                    if isinstance(sub, Filter):
102                        name = f"{filter_idx}-{sub.__class__.__name__}"
103                        sub = AsyncFilterAdapter(sub, executor=self._executor)
104
105                    sub._set_rng_if_not_initialized(self._rng)
106                    sub.name = name
107                    sub._statistics.name = name
108                    self.filters.append(sub)
109                    filter_idx += 1
110            else:
111                name = f"{filter_idx}-{f.__class__.__name__}"
112                if isinstance(f, Filter):
113                    name = f"{filter_idx}-{f.__class__.__name__}"
114                    f = AsyncFilterAdapter(f, executor=self._executor)
115                f._set_rng_if_not_initialized(self._rng)
116                f.name = name
117                f._statistics.name = name
118                self.filters.append(f)
119                filter_idx += 1
async def apply( self, document: hojichar.core.models.Document) -> hojichar.core.models.Document:
121    async def apply(self, document: Document) -> Document:
122        stat = get_doc_info(document)
123        for filter_idx, filt in enumerate(self.filters):
124            document = await filt._apply(document)
125        new_stat = get_doc_info(document)
126        async with self._stats_lock:
127            self._statistics.update_by_diff(stat, new_stat)
128        return document

Definition of async filter behavior.

In this method, the filter will modify document.text or document.extras and set document.is_rejected = True to discard the document.

Parameters

document : Document Input document

Returns

Document Processed Document

async def apply_batch( self, batch: collections.abc.Sequence[hojichar.core.models.Document]) -> list[hojichar.core.models.Document]:
130    async def apply_batch(self, batch: Sequence[Document]) -> list[Document]:
131        stats = [get_doc_info(doc) for doc in batch]
132        for i, filt in enumerate(self.filters):
133            batch = await filt._apply_batch(batch)
134        batch = await self._finalize_batch(batch, stats)
135        return list(batch)

Apply the filter to a Sequence of documents. By default, the processing implemented in apply is executed asynchronously and concurrently. If the filter processing can be optimized for batch processing, override this method.

async def apply_stream( self, stream: collections.abc.AsyncIterable[hojichar.core.models.Document] | collections.abc.Iterable[hojichar.core.models.Document]) -> collections.abc.AsyncGenerator[hojichar.core.models.Document, None]:
137    async def apply_stream(
138        self,
139        stream: AsyncIterable[Document] | Iterable[Document],
140    ) -> AsyncGenerator[Document, None]:
141        async_stream = handle_stream_as_async(stream, chunk_size=1000, executor=self._executor)
142        async_stream = self._count_input_stats(async_stream)
143
144        for i, filt in enumerate(self.filters):
145            async_stream = filt.apply_stream(async_stream)
146
147        async for doc in async_stream:
148            in_stat = doc._get_initial_stats()
149            if in_stat is None:
150                in_stat = get_doc_info(doc)
151                self.logger.debug(
152                    "Initial stats missing for document during async stream aggregation; "
153                    "using current stats as fallback"
154                )
155            out_stat = get_doc_info(doc)
156            async with self._stats_lock:
157                self._statistics.update_by_diff(in_stat, out_stat)
158            doc._clear_initial_stats()
159            yield doc

Apply the filter to a stream of documents (Iterable or AsyncIterable). If use_batch is set to True at initialization, the filter will process documents in batches. If the stream is not asynchronous, use handle_stream_as_async to convert it to an asynchronous stream.

Even if an exception occurs during processing, the process will continue, and the following actions will be taken:

  • Set the is_rejected flag of the document to True
  • Set the error details in reject_reason
  • Increment the errors count in the statistics retrievable via get_statistics
def imap_apply( self, stream: collections.abc.AsyncIterable[hojichar.core.models.Document] | collections.abc.Iterable[hojichar.core.models.Document], *, buffer_size: int = 128, shutdown: bool = True) -> hojichar.utils.async_handlers.AsyncToSyncIterator[hojichar.core.models.Document]:
161    def imap_apply(
162        self,
163        stream: AsyncIterable[Document] | Iterable[Document],
164        *,
165        buffer_size: int = 128,
166        shutdown: bool = True,
167    ) -> AsyncToSyncIterator[Document]:
168        """Synchronously consume this asynchronous pipeline.
169
170        The returned iterator runs the pipeline on a dedicated background event loop. It is
171        intended for one-shot use and shuts down the pipeline after consumption by default.
172        Use the iterator as a context manager if iteration may stop early.
173        """
174
175        finalizer = self.shutdown if shutdown else None
176        return handle_async_stream_as_sync(
177            self.apply_stream(stream),
178            buffer_size=buffer_size,
179            finalizer=finalizer,
180        )

Synchronously consume this asynchronous pipeline.

The returned iterator runs the pipeline on a dedicated background event loop. It is intended for one-shot use and shuts down the pipeline after consumption by default. Use the iterator as a context manager if iteration may stop early.

def get_total_statistics(self) -> list[hojichar.core.models.Statistics]:
189    def get_total_statistics(self) -> list[Statistics]:
190        """
191        Get the statistics of the Compose object and sub filters.
192
193        The statistics of the Compose class are stored in an object with the name "Total",
194        and sub-filters's are stored with names in the format {filter_index}-{filter class name}.
195        """
196        stats = []
197        stats.append(self.get_statistics())
198        for i, filt in enumerate(self.filters):
199            stats.append(filt.get_statistics())
200        return stats

Get the statistics of the Compose object and sub filters.

The statistics of the Compose class are stored in an object with the name "Total", and sub-filters's are stored with names in the format {filter_index}-{filter class name}.

def get_total_statistics_map(self) -> list[dict[str, typing.Any]]:
202    def get_total_statistics_map(self) -> list[dict[str, Any]]:
203        """
204        Get the statistics of the Compose object and sub filters as a list of dictionaries.
205        """
206        stats = self.get_total_statistics()
207        return [stat.to_dict() for stat in stats]

Get the statistics of the Compose object and sub filters as a list of dictionaries.

statistics: dict

Deprecated

Get the statistics of the Compose object and sub filters.

This property is retained for compatibility with previous versions. Please use get_total_statistics or get_total_statistics_map instead.

Deprecated

Get the statistics of the AsyncCompose object and sub filters. This method returns a StatsContainer object which contains the statistics of the AsyncCompose object and sub filters.

This property is retained for compatibility with previous versions. Please use get_total_statistics or get_total_statistics_map instead.

async def shutdown(self) -> None:
237    async def shutdown(self) -> None:
238        for filt in self.filters:
239            await filt.shutdown()
240        if not self._has_external_executor:
241            self._executor.shutdown()

You can override this method to release resources or perform cleanup tasks.