diff --git a/docs/02_concepts/01_actor_lifecycle.mdx b/docs/02_concepts/01_actor_lifecycle.mdx index 38281046..be96e7d4 100644 --- a/docs/02_concepts/01_actor_lifecycle.mdx +++ b/docs/02_concepts/01_actor_lifecycle.mdx @@ -4,52 +4,98 @@ title: Actor lifecycle --- import CodeBlock from '@theme/CodeBlock'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +import ClassContextExample from '!!raw-loader!./code/01_class_context.py'; +import ClassManualExample from '!!raw-loader!./code/01_class_manual.py'; +import InstanceContextExample from '!!raw-loader!./code/01_instance_context.py'; +import InstanceManualExample from '!!raw-loader!./code/01_instance_manual.py'; + +import ErrorHandlingContextExample from '!!raw-loader!./code/01_error_handling_context.py'; +import ErrorHandlingManualExample from '!!raw-loader!./code/01_error_handling_manual.py'; -import InitExitExample from '!!raw-loader!./code/01_init_exit.py'; -import ContextManagerExample from '!!raw-loader!./code/01_context_manager.py'; import RebootExample from '!!raw-loader!./code/01_reboot.py'; + import StatusMessageExample from '!!raw-loader!./code/01_status_message.py'; -In this guide, we will show you how to manage the lifecycle of an Apify Actor. +This guide explains how an **Apify Actor** starts, runs, and shuts down, describing the complete Actor lifecycle. For information about the core concepts such as Actors, the Apify Console, storages, and events, check out the [Apify platform documentation](https://docs.apify.com/platform). + +## Actor initialization + +During initialization, the SDK prepares all the components required to integrate with the Apify platform. It loads configuration from environment variables, initializes access to platform storages such as the [key-value store, dataset, and request queue](https://docs.apify.com/platform/storage), sets up event handling for [platform events](https://docs.apify.com/platform/integrations/webhooks/events), and configures logging. + +The recommended approach in Python is to use the global [`Actor`](https://docs.apify.com/sdk/python/reference/class/Actor) class as an asynchronous context manager. This approach automatically manages setup and teardown and keeps your code concise. When entering the context, the SDK loads configuration and initializes clients lazily—for example, a dataset is opened only when it is first accessed. If the Actor runs on the Apify platform, it also begins listening for platform events. + +When the Actor exits, either normally or due to an exception, the SDK performs a graceful shutdown. It persists the final Actor state, stops event handling, and sets the terminal exit code together with the [status message](https://docs.apify.com/platform/actors/development/programming-interface/status-messages). + + + + + {ClassContextExample} + + + + + {ClassManualExample} + + + + +You can also create an [`Actor`](https://docs.apify.com/sdk/python/reference/class/Actor) instance directly. This does not change its capabilities but allows you to specify optional parameters during initialization, such as disabling automatic `sys.exit()` calls or customizing timeouts. The choice between using a context manager or manual initialization depends on how much control you require over the Actor's startup and shutdown sequence. + + + + + {InstanceContextExample} + + + + + {InstanceManualExample} + + + + +## Error handling + +Good error handling lets your Actor fail fast on critical errors, retry transient issues safely, and keep data consistent. Normally you rely on the `async with Actor:` block—if it finishes, the run succeeds (exit code 0); if an unhandled exception occurs, the run fails (exit code 1). + +The SDK provides helper methods for explicit control: + +- [`Actor.exit`](https://docs.apify.com/sdk/python/reference/class/Actor#exit) - terminates the run successfully (default exit code 0). +- [`Actor.fail`](https://docs.apify.com/sdk/python/reference/class/Actor#fail) - marks the run as failed (default exit code 1). -## Initialization and cleanup +Any non-zero exit code is treated as a `FAILED` run. You rarely need to call these methods directly unless you want to perform a controlled shutdown or customize the exit behavior. -At the start of its runtime, the Actor needs to initialize itself, its event manager and its storages, and at the end of the runtime it needs to close these cleanly. The Apify SDK provides several options on how to manage this. +Catch exceptions only when necessary - for example, to retry network timeouts or map specific errors to exit codes. Keep retry loops bounded with backoff and re-raise once exhausted. Make your processing idempotent so that restarts don't corrupt results. Both [`Actor.exit`](https://docs.apify.com/sdk/python/reference/class/Actor#exit) and [`Actor.fail`](https://docs.apify.com/sdk/python/reference/class/Actor#fail) perform the same cleanup, so complete any long-running persistence before calling them. -The [`Actor.init`](../../reference/class/Actor#init) method initializes the Actor, the event manager which processes the Actor events from the platform event websocket, and the storage client used in the execution environment. It should be called before performing any other Actor operations. +Below is a minimal context-manager example where an unhandled exception automatically fails the run, followed by a manual pattern giving you more control. -The [`Actor.exit`](../../reference/class/Actor#exit) method then exits the Actor cleanly, tearing down the event manager and the storage client. There is also the [`Actor.fail`](../../reference/class/Actor#fail) method, which exits the Actor while marking it as failed. +{ErrorHandlingContextExample} - - {InitExitExample} - +If you need explicit control over exit codes or status messages, you can manage the Actor manually using [`Actor.init`](https://docs.apify.com/sdk/python/reference/class/Actor#init), [`Actor.exit`](https://docs.apify.com/sdk/python/reference/class/Actor#exit), and [`Actor.fail`](https://docs.apify.com/sdk/python/reference/class/Actor#fail). -### Context manager +{ErrorHandlingManualExample} -So that you don't have to call the lifecycle methods manually, the [`Actor`](../../reference/class/Actor) class provides a context manager, which calls the [`Actor.init`](../../reference/class/Actor#init) method on enter, the [`Actor.exit`](../../reference/class/Actor#exit) method on a clean exit, and the [`Actor.fail`](../../reference/class/Actor#fail) method when there is an exception during the run of the Actor. +## Reboot -This is the recommended way to work with the `Actor` class. +Rebooting (available on the Apify platform only) instructs the platform worker to restart your Actor from the beginning of its execution. Use this mechanism only for transient conditions that are likely to resolve after a fresh start — for example, rotating a blocked proxy pool or recovering from a stuck browser environment. - - {ContextManagerExample} - +Before triggering a reboot, persist any essential state externally (e.g., to the key-value store or dataset), as all in-memory data is lost after reboot. The example below tracks a reboot counter in the default key-value store and allows at most three restarts before exiting normally. -## Rebooting an Actor +{RebootExample} -Sometimes, you want to restart your Actor to make it run from the beginning again. To do that, you can use the [`Actor.reboot`](../../reference/class/Actor#reboot) method. When you call it, the Apify platform stops the container of the run, and starts a new container of the same Actor with the same run ID and storages. +## Status message -Don't do it unconditionally, or you might get the Actor in a reboot loop. +[Status messages](https://docs.apify.com/platform/actors/development/programming-interface/status-messages) are lightweight, human-readable progress indicators displayed with the Actor run on the Apify platform (separate from logs). Use them to communicate high-level phases or milestones, such as "Fetching list", "Processed 120/500 pages", or "Uploading results". - - {RebootExample} - +Update the status only when the user's understanding of progress changes - avoid frequent updates for every processed item. Detailed information should go to logs or storages (dataset, key-value store) instead. -## Actor status message +The SDK optimizes updates by sending an API request only when the message text changes, so repeating the same message incurs no additional cost. -To inform you or the users running your Actors about the progress of their runs, you can set the status message for the run, which will then be visible in the run detail in Apify Console, or accessible through the Apify API. +{StatusMessageExample} -To set the status message for the Actor run, you can use the [`Actor.set_status_message`](../../reference/class/Actor#set_status_message) method. +## Conclusion - - {StatusMessageExample} - +This page has presented the full Actor lifecycle: initialization, execution, error handling, rebooting, shutdown and status messages. You've seen how the SDK supports both context-based and manual control patterns. For deeper dives, explore the [reference docs](https://docs.apify.com/sdk/python/reference), [guides](https://docs.apify.com/sdk/python/docs/guides/beautifulsoup-httpx), and [platform documentation](https://docs.apify.com/platform). diff --git a/docs/02_concepts/code/01_class_context.py b/docs/02_concepts/code/01_class_context.py new file mode 100644 index 00000000..86b0875a --- /dev/null +++ b/docs/02_concepts/code/01_class_context.py @@ -0,0 +1,21 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Get input + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor!', 'input': actor_input} + await Actor.push_data(data) + + # Set status message + await Actor.set_status_message('Actor completed successfully') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_class_manual.py b/docs/02_concepts/code/01_class_manual.py new file mode 100644 index 00000000..fbc1c434 --- /dev/null +++ b/docs/02_concepts/code/01_class_manual.py @@ -0,0 +1,26 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + await Actor.init() + + try: + # Get input + actor_input = await Actor.get_input() + Actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor!', 'input': actor_input} + await Actor.push_data(data) + + # Set status message + await Actor.set_status_message('Actor completed successfully') + + finally: + await Actor.exit() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_context_manager.py b/docs/02_concepts/code/01_context_manager.py index 8a3e8654..a98c6ebe 100644 --- a/docs/02_concepts/code/01_context_manager.py +++ b/docs/02_concepts/code/01_context_manager.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -7,3 +9,7 @@ async def main() -> None: Actor.log.info('Actor input: %s', actor_input) await Actor.set_value('OUTPUT', 'Hello, world!') raise RuntimeError('Ouch!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_error_handling_context.py b/docs/02_concepts/code/01_error_handling_context.py new file mode 100644 index 00000000..435563a3 --- /dev/null +++ b/docs/02_concepts/code/01_error_handling_context.py @@ -0,0 +1,13 @@ +import asyncio + +from apify import Actor + + +async def main() -> None: + async with Actor: + # Any unhandled exception triggers Actor.fail() automatically + raise RuntimeError('Boom') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_error_handling_manual.py b/docs/02_concepts/code/01_error_handling_manual.py new file mode 100644 index 00000000..367695a8 --- /dev/null +++ b/docs/02_concepts/code/01_error_handling_manual.py @@ -0,0 +1,33 @@ +import asyncio +import random + +from apify import Actor + + +async def do_work() -> None: + # Simulate random outcomes: success or one of two exception types. + outcome = random.random() + + if outcome < 0.33: + raise ValueError('Invalid input data encountered') + if outcome < 0.66: + raise RuntimeError('Unexpected runtime failure') + + # Simulate successful work + Actor.log.info('Work completed successfully') + + +async def main() -> None: + await Actor.init() + try: + await do_work() + except ValueError as exc: # Specific error mapping example + await Actor.fail(exit_code=10, exception=exc) + except Exception as exc: # Catch-all for unexpected errors + await Actor.fail(exit_code=91, exception=exc) + else: + await Actor.exit(status_message='Actor completed successfully') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_init_exit.py b/docs/02_concepts/code/01_init_exit.py index 674c9285..38b53465 100644 --- a/docs/02_concepts/code/01_init_exit.py +++ b/docs/02_concepts/code/01_init_exit.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -14,3 +16,7 @@ async def main() -> None: await Actor.fail(exit_code=91, exception=exc) await Actor.exit() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_instance_context.py b/docs/02_concepts/code/01_instance_context.py new file mode 100644 index 00000000..5803d543 --- /dev/null +++ b/docs/02_concepts/code/01_instance_context.py @@ -0,0 +1,27 @@ +import asyncio +from datetime import timedelta + +from apify import Actor + + +async def main() -> None: + actor = Actor( + event_listeners_timeout=timedelta(seconds=30), + cleanup_timeout=timedelta(seconds=30), + ) + + async with actor: + # Get input + actor_input = await actor.get_input() + actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor instance!', 'input': actor_input} + await actor.push_data(data) + + # Set status message + await actor.set_status_message('Actor completed successfully') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_instance_manual.py b/docs/02_concepts/code/01_instance_manual.py new file mode 100644 index 00000000..b5a2dcaf --- /dev/null +++ b/docs/02_concepts/code/01_instance_manual.py @@ -0,0 +1,32 @@ +import asyncio +from datetime import timedelta + +from apify import Actor + + +async def main() -> None: + actor = Actor( + event_listeners_timeout=timedelta(seconds=30), + cleanup_timeout=timedelta(seconds=30), + ) + + await actor.init() + + try: + # Get input + actor_input = await actor.get_input() + actor.log.info('Actor input: %s', actor_input) + + # Your Actor logic here + data = {'message': 'Hello from Actor!', 'input': actor_input} + await actor.push_data(data) + + # Set status message + await actor.set_status_message('Actor completed successfully') + + finally: + await actor.exit() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_reboot.py b/docs/02_concepts/code/01_reboot.py index e398c5f4..b12066c5 100644 --- a/docs/02_concepts/code/01_reboot.py +++ b/docs/02_concepts/code/01_reboot.py @@ -1,7 +1,23 @@ +import asyncio + from apify import Actor async def main() -> None: async with Actor: - # ... your code here ... - await Actor.reboot() + # Use the KVS to persist a simple reboot counter across restarts. + kvs = await Actor.open_key_value_store() + reboot_counter = await kvs.get_value('reboot_counter', 0) + + # Limit the number of reboots to avoid infinite loops. + if reboot_counter < 3: + await kvs.set_value('reboot_counter', reboot_counter + 1) + Actor.log.info(f'Reboot attempt {reboot_counter + 1}/3') + # Trigger a platform reboot; after restart the code runs from the beginning. + await Actor.reboot() + + Actor.log.info('Reboot limit reached, finishing run') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/01_status_message.py b/docs/02_concepts/code/01_status_message.py index 13bf2b34..9f239874 100644 --- a/docs/02_concepts/code/01_status_message.py +++ b/docs/02_concepts/code/01_status_message.py @@ -1,3 +1,5 @@ +import asyncio + from apify import Actor @@ -5,10 +7,18 @@ async def main() -> None: async with Actor: await Actor.set_status_message('Here we go!') # Do some work... + await asyncio.sleep(3) await Actor.set_status_message('So far so good...') + await asyncio.sleep(3) # Do some more work... await Actor.set_status_message('Steady as she goes...') + await asyncio.sleep(3) # Do even more work... await Actor.set_status_message('Almost there...') + await asyncio.sleep(3) # Finish the job await Actor.set_status_message('Phew! That was not that hard!') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index b30377f7..6be01f28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,7 @@ ignore = [ "PLR0911", # Too many return statements "PLR0913", # Too many arguments in function definition "PLR0915", # Too many statements + "PLR2004", # Magic value used in comparison, consider replacing `0.2` with a constant variable "PTH", # flake8-use-pathlib "PYI034", # `__aenter__` methods in classes like `{name}` usually return `self` at runtime "PYI036", # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None` diff --git a/src/apify/_actor.py b/src/apify/_actor.py index 965725e2..fa8b5aec 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -32,7 +32,7 @@ from apify._crypto import decrypt_input_secrets, load_private_key from apify._models import ActorRun from apify._proxy_configuration import ProxyConfiguration -from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython, maybe_extract_enum_member_value +from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython from apify.events import ApifyEventManager, EventManager, LocalEventManager from apify.log import _configure_logging, logger from apify.storage_clients import ApifyStorageClient @@ -48,10 +48,10 @@ from typing_extensions import Self from crawlee.proxy_configuration import _NewUrlFunction + from crawlee.storage_clients import StorageClient from apify._models import Webhook - MainReturnType = TypeVar('MainReturnType') @@ -98,7 +98,10 @@ async def main() -> None: """ _is_rebooting = False + """Whether the Actor is currently rebooting.""" + _is_any_instance_initialized = False + """Whether any Actor instance was initialized.""" def __init__( self, @@ -106,63 +109,142 @@ def __init__( *, configure_logging: bool = True, exit_process: bool | None = None, + exit_code: int = 0, + status_message: str | None = None, + event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT, + cleanup_timeout: timedelta = timedelta(seconds=30), ) -> None: - """Create an Actor instance. - - Note that you don't have to do this, all the functionality is accessible using the default instance - (e.g. `Actor.open_dataset()`). + """Initialize a new instance. Args: - configuration: The Actor configuration to be used. If not passed, a new Configuration instance will - be created. - configure_logging: Should the default logging configuration be configured? - exit_process: Whether the Actor should call `sys.exit` when the context manager exits. The default is - True except for the IPython, Pytest and Scrapy environments. + configuration: The Actor configuration to use. If not provided, a default configuration is created. + configure_logging: Whether to set up the default logging configuration. + exit_process: Whether the Actor should call `sys.exit` when the context manager exits. + Defaults to True, except in IPython, Pytest, and Scrapy environments. + exit_code: The exit code the Actor should use when exiting. + status_message: Final status message to display upon Actor termination. + event_listeners_timeout: Maximum time to wait for Actor event listeners to complete before exiting. + cleanup_timeout: Maximum time to wait for cleanup tasks to finish. """ + self._configuration = configuration + self._configure_logging = configure_logging self._exit_process = self._get_default_exit_process() if exit_process is None else exit_process - self._is_exiting = False + self._exit_code = exit_code + self._status_message = status_message + self._event_listeners_timeout = event_listeners_timeout + self._cleanup_timeout = cleanup_timeout # Actor state when this method is being executed is unpredictable. # Actor can be initialized by lazy object proxy or by user directly, or by both. # Until `init` method is run, this state of uncertainty remains. This is the reason why any setting done here in # `__init__` method should not be considered final. - self._configuration = configuration - self._configure_logging = configure_logging self._apify_client: ApifyClientAsync | None = None + self._local_storage_client: StorageClient | None = None + self._is_exiting = False self._is_initialized = False async def __aenter__(self) -> Self: - """Initialize the Actor. + """Enter the Actor context. + + Initializes the Actor when used in an `async with` block. This method: - Automatically initializes the Actor instance when you use it in an `async with ...` statement. + - Sets up local or cloud storage clients depending on whether the Actor runs locally or on the Apify platform. + - Configures the event manager and starts periodic state persistence. + - Initializes the charging manager for handling charging events. + - Configures logging after all core services are registered. - When you exit the `async with` block, the `Actor.exit()` method is called, and if any exception happens while - executing the block code, the `Actor.fail` method is called. + This method must be called exactly once per Actor instance. Re-initializing an Actor or having multiple + active Actor instances is not standard usage and may lead to warnings or unexpected behavior. """ - await self.init() + self.log.info('Initializing Actor...') + self.log.info('System info', extra=get_system_info()) + + if self._is_initialized: + raise RuntimeError('The Actor was already initialized!') + + if _ActorType._is_any_instance_initialized: + self.log.warning('Repeated Actor initialization detected - this is non-standard usage, proceed with care.') + + if self._configuration: + # Set explicitly the configuration in the service locator. + service_locator.set_configuration(self.configuration) + else: + # Ensure that the configuration (cached property) is set. + _ = self.configuration + + # Make sure that the currently initialized instance is also available through the global `Actor` proxy. + cast('Proxy', Actor).__wrapped__ = self + + self._is_exiting = False + self._was_final_persist_state_emitted = False + + service_locator.set_event_manager(self.event_manager) + + # Initialize storage client to ensure it's available in service locator. + _ = self._storage_client + + # The logging configuration has to be called after all service_locator set methods. + if self._configure_logging: + _configure_logging() + + await self.event_manager.__aenter__() + await self._charging_manager_implementation.__aenter__() + + self._is_initialized = True + _ActorType._is_any_instance_initialized = True return self async def __aexit__( self, - _exc_type: type[BaseException] | None, + exc_type: type[BaseException] | None, exc_value: BaseException | None, - _exc_traceback: TracebackType | None, + exc_traceback: TracebackType | None, ) -> None: - """Exit the Actor, handling any exceptions properly. + """Exit the Actor context. - When you exit the `async with` block, the `Actor.exit()` method is called, and if any exception happens while - executing the block code, the `Actor.fail` method is called. + If the block exits with an exception, the Actor fails with a non-zero exit code. + Otherwise, it exits cleanly. In both cases the Actor: + + - Cancels periodic `PERSIST_STATE` events. + - Sends a final `PERSIST_STATE` event. + - Waits for all event listeners to finish. + - Stops the event manager and the charging manager. + - Optionally terminates the process with the selected exit code. """ - if not self._is_exiting: - if exc_value: - await self.fail( - exit_code=ActorExitCodes.ERROR_USER_FUNCTION_THREW.value, - exception=exc_value, - ) - else: - await self.exit() + if self._is_exiting: + return + + self._raise_if_not_initialized() + + if exc_value and not is_running_in_ipython(): + # In IPython, we don't run `sys.exit()` during Actor exits, + # so the exception traceback will be printed on its own + self.log.exception('Actor failed with an exception', exc_info=exc_value) + self.exit_code = ActorExitCodes.ERROR_USER_FUNCTION_THREW.value + + self._is_exiting = True + self.log.info('Exiting Actor', extra={'exit_code': self.exit_code}) + + async def finalize() -> None: + if self.status_message is not None: + await self.set_status_message(self.status_message, is_terminal=True) + + # Sleep for a bit so that the listeners have a chance to trigger + await asyncio.sleep(0.1) + + if self._event_listeners_timeout: + await self.event_manager.wait_for_all_listeners_to_complete(timeout=self._event_listeners_timeout) + + await self.event_manager.__aexit__(None, None, None) + await self._charging_manager_implementation.__aexit__(None, None, None) + + await asyncio.wait_for(finalize(), self._cleanup_timeout.total_seconds()) + self._is_initialized = False + + if self._exit_process: + sys.exit(self.exit_code) def __repr__(self) -> str: if self is cast('Proxy', Actor).__wrapped__: @@ -176,24 +258,58 @@ def __call__( *, configure_logging: bool = True, exit_process: bool | None = None, + exit_code: int = 0, + event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT, + status_message: str | None = None, + cleanup_timeout: timedelta = timedelta(seconds=30), ) -> Self: - """Make a new Actor instance with a non-default configuration.""" + """Make a new Actor instance with a non-default configuration. + + This is necessary due to the lazy object proxying of the global `Actor` instance. + """ return self.__class__( configuration=configuration, configure_logging=configure_logging, exit_process=exit_process, + exit_code=exit_code, + event_listeners_timeout=event_listeners_timeout, + status_message=status_message, + cleanup_timeout=cleanup_timeout, ) + @property + def log(self) -> logging.Logger: + """Logger configured for this Actor.""" + return logger + + @property + def exit_code(self) -> int: + """The exit code the Actor will use when exiting.""" + return self._exit_code + + @exit_code.setter + def exit_code(self, value: int) -> None: + self._exit_code = value + + @property + def status_message(self) -> str | None: + """The final status message that the Actor will display upon termination.""" + return self._status_message + + @status_message.setter + def status_message(self, value: str | None) -> None: + self._status_message = value + @property def apify_client(self) -> ApifyClientAsync: - """The ApifyClientAsync instance the Actor instance uses.""" + """Asynchronous Apify client for interacting with the Apify API.""" if not self._apify_client: self._apify_client = self.new_client() return self._apify_client @cached_property def configuration(self) -> Configuration: - """The Configuration instance the Actor instance uses.""" + """Actor configuration, uses the default instance if not explicitly set.""" if self._configuration: return self._configuration @@ -214,7 +330,10 @@ def configuration(self) -> Configuration: @cached_property def event_manager(self) -> EventManager: - """The EventManager instance the Actor instance uses.""" + """Manages Apify platform events. + + It uses `ApifyEventManager` on the Apify platform and `LocalEventManager` otherwise. + """ return ( ApifyEventManager( configuration=self.configuration, @@ -227,18 +346,13 @@ def event_manager(self) -> EventManager: ) ) - @property - def log(self) -> logging.Logger: - """The logging.Logger instance the Actor uses.""" - return logger - - def _raise_if_not_initialized(self) -> None: - if not self._is_initialized: - raise RuntimeError('The Actor was not initialized!') + @cached_property + def _charging_manager_implementation(self) -> ChargingManagerImplementation: + return ChargingManagerImplementation(self.configuration, self.apify_client) @cached_property def _storage_client(self) -> SmartApifyStorageClient: - """Storage client used by the actor. + """Storage client used by the Actor. Depending on the initialization of the service locator the client can be created in different ways. """ @@ -250,7 +364,7 @@ def _storage_client(self) -> SmartApifyStorageClient: service_locator.set_storage_client(implicit_storage_client) except ServiceConflictError: self.log.debug( - 'Storage client in service locator was set explicitly before Actor.init was called.' + 'Storage client in service locator was set explicitly before Actor.init was called. ' 'Using the existing storage client as implicit storage client for the Actor.' ) else: @@ -270,100 +384,35 @@ def _storage_client(self) -> SmartApifyStorageClient: ) async def init(self) -> None: - """Initialize the Actor instance. - - This initializes the Actor instance. It configures the right storage client based on whether the Actor is - running locally or on the Apify platform, it initializes the event manager for processing Actor events, - and starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist - its state in response to these events. + """Initialize the Actor without using context-manager syntax. - This method should be called immediately before performing any additional Actor actions, and it should be - called only once. + Equivalent to `await Actor.__aenter__()`. """ - self.log.info('Initializing Actor...') - if self._configuration: - # Set explicitly the configuration in the service locator - service_locator.set_configuration(self.configuration) - else: - # Ensure that the configuration (cached property) is set - _ = self.configuration - - if self._is_initialized: - raise RuntimeError('The Actor was already initialized!') - - if _ActorType._is_any_instance_initialized: - self.log.warning('Repeated Actor initialization detected - this is non-standard usage, proceed with care') - - # Make sure that the currently initialized instance is also available through the global `Actor` proxy - cast('Proxy', Actor).__wrapped__ = self - - self._is_exiting = False - self._was_final_persist_state_emitted = False - - self.log.debug(f'Storage client set to {self._storage_client}') - - service_locator.set_event_manager(self.event_manager) - - # The logging configuration has to be called after all service_locator set methods. - if self._configure_logging: - _configure_logging() - - self.log.info('System info', extra=get_system_info()) - - await self.event_manager.__aenter__() - self.log.debug('Event manager initialized') - - await self._charging_manager_implementation.__aenter__() - self.log.debug('Charging manager initialized') - - self._is_initialized = True - _ActorType._is_any_instance_initialized = True + await self.__aenter__() async def exit( self, *, exit_code: int = 0, - event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT, status_message: str | None = None, + event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT, cleanup_timeout: timedelta = timedelta(seconds=30), ) -> None: - """Exit the Actor instance. + """Exit the Actor without using context-manager syntax. - This stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events, - sends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager. + Equivalent to `await Actor.__aexit__()`. Args: - exit_code: The exit code with which the Actor should fail (defaults to `0`). - event_listeners_timeout: How long should the Actor wait for Actor event listeners to finish before exiting. - status_message: The final status message that the Actor should display. - cleanup_timeout: How long we should wait for event listeners. + exit_code: The exit code the Actor should use when exiting. + status_message: Final status message to display upon Actor termination. + event_listeners_timeout: Maximum time to wait for Actor event listeners to complete before exiting. + cleanup_timeout: Maximum time to wait for cleanup tasks to finish. """ - self._raise_if_not_initialized() - - self._is_exiting = True - - exit_code = maybe_extract_enum_member_value(exit_code) - - self.log.info('Exiting Actor', extra={'exit_code': exit_code}) - - async def finalize() -> None: - if status_message is not None: - await self.set_status_message(status_message, is_terminal=True) - - # Sleep for a bit so that the listeners have a chance to trigger - await asyncio.sleep(0.1) - - if event_listeners_timeout: - await self.event_manager.wait_for_all_listeners_to_complete(timeout=event_listeners_timeout) - - await self.event_manager.__aexit__(None, None, None) - await self._charging_manager_implementation.__aexit__(None, None, None) - - await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds()) - self._is_initialized = False - - if self._exit_process: - sys.exit(exit_code) + self.exit_code = exit_code + self.status_message = status_message + self._event_listeners_timeout = event_listeners_timeout + self._cleanup_timeout = cleanup_timeout + await self.__aexit__(None, None, None) async def fail( self, @@ -372,23 +421,24 @@ async def fail( exception: BaseException | None = None, status_message: str | None = None, ) -> None: - """Fail the Actor instance. + """Fail the Actor instance without using context-manager syntax. - This performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default). + Equivalent to setting the `self.exit_code` and `self.status_message` properties and using + `await Actor.__aexit__()`. Args: exit_code: The exit code with which the Actor should fail (defaults to `1`). exception: The exception with which the Actor failed. status_message: The final status message that the Actor should display. """ - self._raise_if_not_initialized() - - # In IPython, we don't run `sys.exit()` during Actor exits, - # so the exception traceback will be printed on its own - if exception and not is_running_in_ipython(): - self.log.exception('Actor failed with an exception', exc_info=exception) + self.exit_code = exit_code + self.status_message = status_message - await self.exit(exit_code=exit_code, status_message=status_message) + await self.__aexit__( + exc_type=type(exception) if exception else None, + exc_value=exception, + exc_traceback=exception.__traceback__ if exception else None, + ) def new_client( self, @@ -626,10 +676,6 @@ def get_charging_manager(self) -> ChargingManager: self._raise_if_not_initialized() return self._charging_manager_implementation - @cached_property - def _charging_manager_implementation(self) -> ChargingManagerImplementation: - return ChargingManagerImplementation(self.configuration, self.apify_client) - async def charge(self, event_name: str, count: int = 1) -> ChargeResult: """Charge for a specified number of events - sub-operations of the Actor. @@ -822,18 +868,6 @@ async def start( return ActorRun.model_validate(api_result) - def _get_remaining_time(self) -> timedelta | None: - """Get time remaining from the Actor timeout. Returns `None` if not on an Apify platform.""" - if self.is_at_home() and self.configuration.timeout_at: - return self.configuration.timeout_at - datetime.now(tz=timezone.utc) - - self.log.warning( - 'Returning `None` instead of remaining time. Using `RemainingTime` argument is only possible when the Actor' - ' is running on the Apify platform and when the timeout for the Actor run is set. ' - f'{self.is_at_home()=}, {self.configuration.timeout_at=}' - ) - return None - async def abort( self, run_id: str, @@ -1242,6 +1276,10 @@ async def create_proxy_configuration( return proxy_configuration + def _raise_if_not_initialized(self) -> None: + if not self._is_initialized: + raise RuntimeError('The Actor was not initialized!') + def _get_default_exit_process(self) -> bool: """Return False for IPython, Pytest, and Scrapy environments, True otherwise.""" if is_running_in_ipython(): @@ -1262,6 +1300,18 @@ def _get_default_exit_process(self) -> bool: return True + def _get_remaining_time(self) -> timedelta | None: + """Get time remaining from the Actor timeout. Returns `None` if not on an Apify platform.""" + if self.is_at_home() and self.configuration.timeout_at: + return self.configuration.timeout_at - datetime.now(tz=timezone.utc) + + self.log.warning( + 'Returning `None` instead of remaining time. Using `RemainingTime` argument is only possible when the Actor' + ' is running on the Apify platform and when the timeout for the Actor run is set. ' + f'{self.is_at_home()=}, {self.configuration.timeout_at=}' + ) + return None + Actor = cast('_ActorType', Proxy(_ActorType)) """The entry point of the SDK, through which all the Actor operations should be done.""" diff --git a/tests/unit/actor/test_actor_lifecycle.py b/tests/unit/actor/test_actor_lifecycle.py index 7d04557b..da575914 100644 --- a/tests/unit/actor/test_actor_lifecycle.py +++ b/tests/unit/actor/test_actor_lifecycle.py @@ -5,65 +5,185 @@ import json import sys from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from unittest import mock from unittest.mock import AsyncMock, Mock import pytest import websockets.asyncio.server -from apify_shared.consts import ActorEnvVars, ApifyEnvVars +from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars from crawlee.events._types import Event, EventPersistStateData -import apify._actor from apify import Actor -from apify._actor import _ActorType if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import AsyncGenerator, Callable + + from apify._actor import _ActorType + + +@pytest.fixture( + params=[ + pytest.param(('instance', 'manual'), id='instance-manual'), + pytest.param(('instance', 'async_with'), id='instance-async-with'), + pytest.param(('class', 'manual'), id='class-manual'), + pytest.param(('class', 'async_with'), id='class-async-with'), + ] +) +async def actor( + request: pytest.FixtureRequest, +) -> AsyncGenerator[_ActorType, None]: + """Yield Actor instance or class in different initialization modes. + + - instance-manual: Actor() with manual init()/exit() + - instance-async-with: Actor() used as async context manager + - class-manual: Actor class with manual init()/exit() + - class-async-with: Actor class used as async context manager + + Each Actor is properly initialized before yielding and cleaned up after. + """ + scope, mode = request.param + + if scope == 'instance': + if mode == 'manual': + instance = Actor() + await instance.init() + yield instance + await instance.exit() + else: + async with Actor() as instance: + yield instance + + elif scope == 'class': + if mode == 'manual': + await Actor.init() + yield Actor + await Actor.exit() + else: + async with Actor: + yield Actor - from lazy_object_proxy import Proxy + else: + raise ValueError(f'Unknown scope: {scope}') -async def test_actor_properly_init_with_async() -> None: - async with Actor: - assert cast('Proxy', apify._actor.Actor).__wrapped__ is not None - assert cast('Proxy', apify._actor.Actor).__wrapped__._is_initialized - assert not cast('Proxy', apify._actor.Actor).__wrapped__._is_initialized +async def test_actor_init_instance_manual() -> None: + """Test that Actor instance can be properly initialized and cleaned up manually.""" + actor = Actor() + await actor.init() + assert actor._is_initialized is True + await actor.exit() + assert actor._is_initialized is False + +async def test_actor_init_instance_async_with() -> None: + """Test that Actor instance can be properly initialized and cleaned up using async context manager.""" + actor = Actor() + async with actor: + assert actor._is_initialized is True -async def test_actor_init() -> None: - my_actor = _ActorType() + assert actor._is_initialized is False - await my_actor.init() - assert my_actor._is_initialized is True - await my_actor.exit() - assert my_actor._is_initialized is False +async def test_actor_init_class_manual() -> None: + """Test that Actor class can be properly initialized and cleaned up manually.""" + await Actor.init() + assert Actor._is_initialized is True + await Actor.exit() + assert not Actor._is_initialized -async def test_double_init_raises_error(prepare_test_env: Callable) -> None: +async def test_actor_init_class_async_with() -> None: + """Test that Actor class can be properly initialized and cleaned up using async context manager.""" async with Actor: - assert Actor._is_initialized - with pytest.raises(RuntimeError): - await Actor.init() + assert Actor._is_initialized is True - prepare_test_env() + assert not Actor._is_initialized - async with Actor() as actor: - assert actor._is_initialized - with pytest.raises(RuntimeError): - await actor.init() - prepare_test_env() +async def test_fail_properly_deinitializes_actor(actor: _ActorType) -> None: + """Test that fail() method properly deinitializes the Actor.""" + assert actor._is_initialized + await actor.fail() + assert actor._is_initialized is False - async with _ActorType() as actor: - assert actor._is_initialized - with pytest.raises(RuntimeError): - await actor.init() +async def test_actor_handles_exceptions_and_cleans_up_properly() -> None: + """Test that Actor properly cleans up when an exception occurs in the async context manager.""" + actor = None + + with contextlib.suppress(Exception): + async with Actor() as actor: + assert actor._is_initialized + raise Exception('Failed') # noqa: TRY002 + + assert actor is not None + assert actor._is_initialized is False + + +async def test_double_init_raises_runtime_error(actor: _ActorType) -> None: + """Test that attempting to initialize an already initialized Actor raises RuntimeError.""" + assert actor._is_initialized + with pytest.raises(RuntimeError): + await actor.init() + + +async def test_exit_without_init_raises_runtime_error() -> None: + """Test that calling exit() on an uninitialized Actor raises RuntimeError.""" + with pytest.raises(RuntimeError): + await Actor.exit() + + with pytest.raises(RuntimeError): + await Actor().exit() + + +async def test_fail_without_init_raises_runtime_error() -> None: + """Test that calling fail() on an uninitialized Actor raises RuntimeError.""" + with pytest.raises(RuntimeError): + await Actor.fail() + + with pytest.raises(RuntimeError): + await Actor().fail() + + +async def test_reboot_in_local_environment_logs_error_message( + actor: _ActorType, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that reboot() logs an error when not running on the Apify platform.""" + await actor.reboot() + + # Check that the error message was logged + assert 'Actor.reboot() is only supported when running on the Apify platform.' in caplog.text -async def test_actor_exits_cleanly_with_events(monkeypatch: pytest.MonkeyPatch) -> None: + +async def test_exit_sets_custom_exit_code_and_status_message(actor: _ActorType) -> None: + """Test that exit() properly sets custom exit code and status message.""" + await actor.exit(exit_code=42, status_message='Exiting with code 42') + assert actor.exit_code == 42 + assert actor.status_message == 'Exiting with code 42' + + +async def test_fail_sets_custom_exit_code_and_status_message(actor: _ActorType) -> None: + """Test that fail() properly sets custom exit code and status message.""" + await actor.fail(exit_code=99, status_message='Failing with code 99') + assert actor.exit_code == 99 + assert actor.status_message == 'Failing with code 99' + + +async def test_unhandled_exception_sets_error_exit_code() -> None: + """Test that unhandled exceptions in context manager set the error exit code.""" + actor = Actor(exit_process=False) + with pytest.raises(RuntimeError): + async with actor: + raise RuntimeError('Test error') + + assert actor.exit_code == ActorExitCodes.ERROR_USER_FUNCTION_THREW.value + + +async def test_actor_stops_periodic_events_after_exit(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that periodic events (PERSIST_STATE and SYSTEM_INFO) stop emitting after Actor exits.""" monkeypatch.setenv(ApifyEnvVars.SYSTEM_INFO_INTERVAL_MILLIS, '100') monkeypatch.setenv(ApifyEnvVars.PERSIST_STATE_INTERVAL_MILLIS, '100') on_persist = [] @@ -78,11 +198,11 @@ def on_event(event_type: Event) -> Callable: return lambda data: on_system_info.append(data) return lambda data: print(data) - my_actor = _ActorType() - async with my_actor: - assert my_actor._is_initialized - my_actor.on(Event.PERSIST_STATE, on_event(Event.PERSIST_STATE)) - my_actor.on(Event.SYSTEM_INFO, on_event(Event.SYSTEM_INFO)) + actor = Actor() + async with actor: + assert actor._is_initialized + actor.on(Event.PERSIST_STATE, on_event(Event.PERSIST_STATE)) + actor.on(Event.SYSTEM_INFO, on_event(Event.SYSTEM_INFO)) await asyncio.sleep(1) on_persist_count = len(on_persist) @@ -96,42 +216,9 @@ def on_event(event_type: Event) -> Callable: assert on_system_info_count == len(on_system_info) -async def test_exit_without_init_raises_error() -> None: - with pytest.raises(RuntimeError): - await Actor.exit() - - -async def test_actor_fails_cleanly() -> None: - async with _ActorType() as my_actor: - assert my_actor._is_initialized - await my_actor.fail() - assert my_actor._is_initialized is False - - -async def test_actor_handles_failure_gracefully() -> None: - my_actor = None - - with contextlib.suppress(Exception): - async with _ActorType() as my_actor: - assert my_actor._is_initialized - raise Exception('Failed') # noqa: TRY002 - - assert my_actor is not None - assert my_actor._is_initialized is False - - -async def test_fail_without_init_raises_error() -> None: - with pytest.raises(RuntimeError): - await Actor.fail() - - -async def test_actor_reboot_fails_locally() -> None: - with pytest.raises(RuntimeError): - await Actor.reboot() - - @pytest.mark.skipif(sys.version_info >= (3, 13), reason='Suffers flaky behavior on Python 3.13') async def test_actor_handles_migrating_event_correctly(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that Actor handles MIGRATING events correctly by emitting PERSIST_STATE.""" # This should test whether when you get a MIGRATING event, # the Actor automatically emits the PERSIST_STATE event with data `{'isMigrating': True}` monkeypatch.setenv(ApifyEnvVars.PERSIST_STATE_INTERVAL_MILLIS, '500') diff --git a/tests/unit/actor/test_actor_log.py b/tests/unit/actor/test_actor_log.py index ecb90ab6..73925605 100644 --- a/tests/unit/actor/test_actor_log.py +++ b/tests/unit/actor/test_actor_log.py @@ -37,7 +37,7 @@ async def test_actor_logs_messages_correctly(caplog: pytest.LogCaptureFixture) - raise RuntimeError('Dummy RuntimeError') # Updated expected number of log records (an extra record is now captured) - assert len(caplog.records) == 15 + assert len(caplog.records) == 12 # Record 0: Extra Pytest context log assert caplog.records[0].levelno == logging.DEBUG @@ -51,58 +51,46 @@ async def test_actor_logs_messages_correctly(caplog: pytest.LogCaptureFixture) - assert caplog.records[2].levelno == logging.INFO assert caplog.records[2].message == 'Initializing Actor...' - # Record 2: Initializing Actor... - assert caplog.records[3].levelno == logging.DEBUG - assert caplog.records[3].message.startswith('Storage client set to') - # Record 3: System info - assert caplog.records[4].levelno == logging.INFO - assert caplog.records[4].message == 'System info' + assert caplog.records[3].levelno == logging.INFO + assert caplog.records[3].message == 'System info' # Record 4: Event manager initialized - assert caplog.records[5].levelno == logging.DEBUG - assert caplog.records[5].message == 'Event manager initialized' - - # Record 5: Charging manager initialized - assert caplog.records[6].levelno == logging.DEBUG - assert caplog.records[6].message == 'Charging manager initialized' - - # Record 6: Debug message - assert caplog.records[7].levelno == logging.DEBUG - assert caplog.records[7].message == 'Debug message' + assert caplog.records[4].levelno == logging.DEBUG + assert caplog.records[4].message == 'Debug message' # Record 7: Info message - assert caplog.records[8].levelno == logging.INFO - assert caplog.records[8].message == 'Info message' + assert caplog.records[5].levelno == logging.INFO + assert caplog.records[5].message == 'Info message' # Record 8: Warning message - assert caplog.records[9].levelno == logging.WARNING - assert caplog.records[9].message == 'Warning message' + assert caplog.records[6].levelno == logging.WARNING + assert caplog.records[6].message == 'Warning message' # Record 9: Error message - assert caplog.records[10].levelno == logging.ERROR - assert caplog.records[10].message == 'Error message' + assert caplog.records[7].levelno == logging.ERROR + assert caplog.records[7].message == 'Error message' # Record 10: Exception message with traceback (ValueError) - assert caplog.records[11].levelno == logging.ERROR - assert caplog.records[11].message == 'Exception message' - assert caplog.records[11].exc_info is not None - assert caplog.records[11].exc_info[0] is ValueError - assert isinstance(caplog.records[11].exc_info[1], ValueError) - assert str(caplog.records[11].exc_info[1]) == 'Dummy ValueError' + assert caplog.records[8].levelno == logging.ERROR + assert caplog.records[8].message == 'Exception message' + assert caplog.records[8].exc_info is not None + assert caplog.records[8].exc_info[0] is ValueError + assert isinstance(caplog.records[8].exc_info[1], ValueError) + assert str(caplog.records[8].exc_info[1]) == 'Dummy ValueError' # Record 11: Multiline log message - assert caplog.records[12].levelno == logging.INFO - assert caplog.records[12].message == 'Multi\nline\nlog\nmessage' + assert caplog.records[9].levelno == logging.INFO + assert caplog.records[9].message == 'Multi\nline\nlog\nmessage' # Record 12: Actor failed with an exception (RuntimeError) - assert caplog.records[13].levelno == logging.ERROR - assert caplog.records[13].message == 'Actor failed with an exception' - assert caplog.records[13].exc_info is not None - assert caplog.records[13].exc_info[0] is RuntimeError - assert isinstance(caplog.records[13].exc_info[1], RuntimeError) - assert str(caplog.records[13].exc_info[1]) == 'Dummy RuntimeError' + assert caplog.records[10].levelno == logging.ERROR + assert caplog.records[10].message == 'Actor failed with an exception' + assert caplog.records[10].exc_info is not None + assert caplog.records[10].exc_info[0] is RuntimeError + assert isinstance(caplog.records[10].exc_info[1], RuntimeError) + assert str(caplog.records[10].exc_info[1]) == 'Dummy RuntimeError' # Record 13: Exiting Actor - assert caplog.records[14].levelno == logging.INFO - assert caplog.records[14].message == 'Exiting Actor' + assert caplog.records[11].levelno == logging.INFO + assert caplog.records[11].message == 'Exiting Actor'