feat: Use Actor env vars (#105)

pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
     "aiofiles >= 22.1.0",
     "aioshutil >= 1.0",
     "apify-client >= 1.3.1",
-    "apify-shared >= 1.0.0",
+    "apify-shared >= 1.0.2",
     "colorama >= 0.4.6",
     "cryptography >= 39.0.0",
     "httpx >= 0.24.1",

src/apify/_utils.py

@@ -34,6 +34,7 @@
     INTEGER_ENV_VARS,
     INTEGER_ENV_VARS_TYPE,
     STRING_ENV_VARS_TYPE,
+    ActorEnvVars,
     ApifyEnvVars,
 )
 from apify_shared.utils import ignore_docs, is_content_type_json, is_content_type_text, is_content_type_xml, maybe_extract_enum_member_value
@@ -144,7 +145,7 @@ def _fetch_and_parse_env_var(env_var: STRING_ENV_VARS_TYPE) -> Optional[str]:
 
 
 @overload
-def _fetch_and_parse_env_var(env_var: ApifyEnvVars) -> Optional[Any]:
+def _fetch_and_parse_env_var(env_var: Union[ActorEnvVars, ApifyEnvVars]) -> Optional[Any]:
     ...
 
 

src/apify/actor.py

@@ -9,7 +9,7 @@
 from typing import Any, Awaitable, Callable, Dict, List, Optional, Type, TypeVar, Union, cast
 
 from apify_client import ApifyClientAsync
-from apify_shared.consts import ActorEventTypes, ActorExitCodes, ApifyEnvVars, WebhookEventType
+from apify_shared.consts import ActorEnvVars, ActorEventTypes, ActorExitCodes, ApifyEnvVars, WebhookEventType
 from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
 
 from ._crypto import _decrypt_input_secrets, _load_private_key
@@ -748,7 +748,7 @@ def _get_env_internal(self) -> Dict:
         self._raise_if_not_initialized()
 
         return {
-            env_var.name.lower(): _fetch_and_parse_env_var(env_var) for env_var in ApifyEnvVars
+            env_var.name.lower(): _fetch_and_parse_env_var(env_var) for env_var in [*ActorEnvVars, *ApifyEnvVars]
         }
 
     @classmethod

src/apify/config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from apify_shared.consts import ApifyEnvVars
+from apify_shared.consts import ActorEnvVars, ApifyEnvVars
 
 from ._utils import _fetch_and_parse_env_var
 
@@ -67,30 +67,30 @@ def __init__(
             system_info_interval_millis (str, optional): How often should the actor emit the SYSTEM_INFO event when running locally.
         """
         # TODO: Document all these members
-        self.actor_build_id = _fetch_and_parse_env_var(ApifyEnvVars.ACTOR_BUILD_ID)
-        self.actor_build_number = _fetch_and_parse_env_var(ApifyEnvVars.ACTOR_BUILD_NUMBER)
-        self.actor_events_ws_url = _fetch_and_parse_env_var(ApifyEnvVars.ACTOR_EVENTS_WS_URL)
-        self.actor_id = _fetch_and_parse_env_var(ApifyEnvVars.ACTOR_ID)
-        self.actor_run_id = _fetch_and_parse_env_var(ApifyEnvVars.ACTOR_RUN_ID)
-        self.actor_task_id = _fetch_and_parse_env_var(ApifyEnvVars.ACTOR_TASK_ID)
+        self.actor_build_id = _fetch_and_parse_env_var(ActorEnvVars.BUILD_ID)
+        self.actor_build_number = _fetch_and_parse_env_var(ActorEnvVars.BUILD_NUMBER)
+        self.actor_events_ws_url = _fetch_and_parse_env_var(ActorEnvVars.EVENTS_WEBSOCKET_URL)
+        self.actor_id = _fetch_and_parse_env_var(ActorEnvVars.ID)
+        self.actor_run_id = _fetch_and_parse_env_var(ActorEnvVars.RUN_ID)
+        self.actor_task_id = _fetch_and_parse_env_var(ActorEnvVars.TASK_ID)
         self.api_base_url = api_base_url or _fetch_and_parse_env_var(ApifyEnvVars.API_BASE_URL, 'https://api.apify.com')
         self.api_public_base_url = api_public_base_url or _fetch_and_parse_env_var(ApifyEnvVars.API_PUBLIC_BASE_URL, 'https://api.apify.com')
         self.chrome_executable_path = _fetch_and_parse_env_var(ApifyEnvVars.CHROME_EXECUTABLE_PATH)
-        self.container_port = container_port or _fetch_and_parse_env_var(ApifyEnvVars.CONTAINER_PORT, 4321)
-        self.container_url = container_url or _fetch_and_parse_env_var(ApifyEnvVars.CONTAINER_URL, 'http://localhost:4321')
+        self.container_port = container_port or _fetch_and_parse_env_var(ActorEnvVars.WEB_SERVER_PORT, 4321)
+        self.container_url = container_url or _fetch_and_parse_env_var(ActorEnvVars.WEB_SERVER_URL, 'http://localhost:4321')
         self.dedicated_cpus = _fetch_and_parse_env_var(ApifyEnvVars.DEDICATED_CPUS)
         self.default_browser_path = _fetch_and_parse_env_var(ApifyEnvVars.DEFAULT_BROWSER_PATH)
-        self.default_dataset_id = default_dataset_id or _fetch_and_parse_env_var(ApifyEnvVars.DEFAULT_DATASET_ID, 'default')
-        self.default_key_value_store_id = default_key_value_store_id or _fetch_and_parse_env_var(ApifyEnvVars.DEFAULT_KEY_VALUE_STORE_ID, 'default')
-        self.default_request_queue_id = default_request_queue_id or _fetch_and_parse_env_var(ApifyEnvVars.DEFAULT_REQUEST_QUEUE_ID, 'default')
+        self.default_dataset_id = default_dataset_id or _fetch_and_parse_env_var(ActorEnvVars.DEFAULT_DATASET_ID, 'default')
+        self.default_key_value_store_id = default_key_value_store_id or _fetch_and_parse_env_var(ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID, 'default')
+        self.default_request_queue_id = default_request_queue_id or _fetch_and_parse_env_var(ActorEnvVars.DEFAULT_REQUEST_QUEUE_ID, 'default')
         self.disable_browser_sandbox = _fetch_and_parse_env_var(ApifyEnvVars.DISABLE_BROWSER_SANDBOX, False)
         self.headless = _fetch_and_parse_env_var(ApifyEnvVars.HEADLESS, True)
-        self.input_key = input_key or _fetch_and_parse_env_var(ApifyEnvVars.INPUT_KEY, 'INPUT')
+        self.input_key = input_key or _fetch_and_parse_env_var(ActorEnvVars.INPUT_KEY, 'INPUT')
         self.input_secrets_private_key_file = _fetch_and_parse_env_var(ApifyEnvVars.INPUT_SECRETS_PRIVATE_KEY_FILE)
         self.input_secrets_private_key_passphrase = _fetch_and_parse_env_var(ApifyEnvVars.INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE)
         self.is_at_home = _fetch_and_parse_env_var(ApifyEnvVars.IS_AT_HOME, False)
         self.max_used_cpu_ratio = max_used_cpu_ratio or _fetch_and_parse_env_var(ApifyEnvVars.MAX_USED_CPU_RATIO, 0.95)
-        self.memory_mbytes = _fetch_and_parse_env_var(ApifyEnvVars.MEMORY_MBYTES)
+        self.memory_mbytes = _fetch_and_parse_env_var(ActorEnvVars.MEMORY_MBYTES)
         self.meta_origin = _fetch_and_parse_env_var(ApifyEnvVars.META_ORIGIN)
         self.metamorph_after_sleep_millis = metamorph_after_sleep_millis or _fetch_and_parse_env_var(ApifyEnvVars.METAMORPH_AFTER_SLEEP_MILLIS, 300000)  # noqa: E501
         self.persist_state_interval_millis = persist_state_interval_millis or _fetch_and_parse_env_var(ApifyEnvVars.PERSIST_STATE_INTERVAL_MILLIS, 60000)  # noqa: E501
@@ -100,8 +100,8 @@ def __init__(
         self.proxy_port = proxy_port or _fetch_and_parse_env_var(ApifyEnvVars.PROXY_PORT, 8000)
         self.proxy_status_url = proxy_status_url or _fetch_and_parse_env_var(ApifyEnvVars.PROXY_STATUS_URL, 'http://proxy.apify.com')
         self.purge_on_start = purge_on_start or _fetch_and_parse_env_var(ApifyEnvVars.PURGE_ON_START, False)
-        self.started_at = _fetch_and_parse_env_var(ApifyEnvVars.STARTED_AT)
-        self.timeout_at = _fetch_and_parse_env_var(ApifyEnvVars.TIMEOUT_AT)
+        self.started_at = _fetch_and_parse_env_var(ActorEnvVars.STARTED_AT)
+        self.timeout_at = _fetch_and_parse_env_var(ActorEnvVars.TIMEOUT_AT)
         self.token = token or _fetch_and_parse_env_var(ApifyEnvVars.TOKEN)
         self.user_id = _fetch_and_parse_env_var(ApifyEnvVars.USER_ID)
         self.xvfb = _fetch_and_parse_env_var(ApifyEnvVars.XVFB, False)

tests/integration/README.md

@@ -63,9 +63,9 @@ You can also pass extra imports directly to the main function:
 async def test_something(self, make_actor: ActorFactory) -> None:
     async def main():
         import os
-        from apify_shared.consts import ActorEventTypes, ApifyEnvVars
+        from apify_shared.consts import ActorEventTypes, ActorEnvVars
         async with Actor:
-            print('The actor is running with ' + os.getenv(ApifyEnvVars.MEMORY_MBYTES) + 'MB of memory')
+            print('The actor is running with ' + os.getenv(ActorEnvVars.MEMORY_MBYTES) + 'MB of memory')
             await Actor.on(ActorEventTypes.SYSTEM_INFO, lambda event_data: print(event_data))
 
     actor = await make_actor('something', main_func=main)

tests/integration/test_actor_api_helpers.py

@@ -54,13 +54,13 @@ async def test_actor_new_client(self, make_actor: ActorFactory) -> None:
         async def main() -> None:
             import os
 
-            from apify_shared.consts import ApifyEnvVars
+            from apify_shared.consts import ActorEnvVars
 
             async with Actor:
                 new_client = Actor.new_client()
                 assert new_client is not Actor.apify_client
 
-                default_key_value_store_id = os.getenv(ApifyEnvVars.DEFAULT_KEY_VALUE_STORE_ID)
+                default_key_value_store_id = os.getenv(ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID)
                 assert default_key_value_store_id is not None
                 kv_store_client = new_client.key_value_store(default_key_value_store_id)
                 await kv_store_client.set_record('OUTPUT', 'TESTING-OUTPUT')
@@ -272,11 +272,11 @@ async def test_actor_metamorph(self, make_actor: ActorFactory) -> None:
         async def main_inner() -> None:
             import os
 
-            from apify_shared.consts import ApifyEnvVars
+            from apify_shared.consts import ActorEnvVars
 
             async with Actor:
-                assert os.getenv(ApifyEnvVars.INPUT_KEY) is not None
-                assert os.getenv(ApifyEnvVars.INPUT_KEY) != 'INPUT'
+                assert os.getenv(ActorEnvVars.INPUT_KEY) is not None
+                assert os.getenv(ActorEnvVars.INPUT_KEY) != 'INPUT'
                 input = await Actor.get_input() or {}
 
                 test_value = input.get('test_value', '')
@@ -329,7 +329,7 @@ async def main_server() -> None:
             import os
             from http.server import BaseHTTPRequestHandler, HTTPServer
 
-            from apify_shared.consts import ApifyEnvVars
+            from apify_shared.consts import ActorEnvVars
 
             webhook_body = ''
 
@@ -351,7 +351,7 @@ def do_POST(self) -> None:  # noqa: N802
                         self.end_headers()
                         self.wfile.write(bytes('Hello, world!', encoding='utf-8'))
 
-                container_port = int(os.getenv(ApifyEnvVars.CONTAINER_PORT, ''))
+                container_port = int(os.getenv(ActorEnvVars.WEB_SERVER_PORT, ''))
                 with HTTPServer(('', container_port), WebhookHandler) as server:
                     await Actor.set_value('INITIALIZED', True)
                     while not webhook_body:

tests/integration/test_fixtures.py

@@ -12,10 +12,10 @@ async def test_main_func(self, make_actor: ActorFactory) -> None:
         async def main() -> None:
             import os
 
-            from apify_shared.consts import ApifyEnvVars
+            from apify_shared.consts import ActorEnvVars
 
             async with Actor:
-                await Actor.set_value('OUTPUT', os.getenv(ApifyEnvVars.ACTOR_ID))
+                await Actor.set_value('OUTPUT', os.getenv(ActorEnvVars.ID))
 
         actor = await make_actor('make-actor-main-func', main_func=main)
 

tests/unit/actor/test_actor_dataset.py

@@ -3,7 +3,7 @@
 
 from apify import Actor
 from apify._memory_storage import MemoryStorageClient
-from apify_shared.consts import ApifyEnvVars
+from apify_shared.consts import ActorEnvVars
 
 # NOTE: We only test the dataset methond available on Actor class/instance. Actual tests for the implementations are in storages/.
 
@@ -36,7 +36,7 @@ async def test_open_datatset_based_env_var(
         memory_storage_client: MemoryStorageClient,
     ) -> None:
         default_dataset_id = 'my-new-default-id'
-        monkeypatch.setenv(ApifyEnvVars.DEFAULT_DATASET_ID, default_dataset_id)
+        monkeypatch.setenv(ActorEnvVars.DEFAULT_DATASET_ID, default_dataset_id)
 
         async with Actor:
             ddt = await Actor.open_dataset()

tests/unit/actor/test_actor_env_helpers.py

@@ -6,7 +6,7 @@
 import pytest
 
 from apify import Actor
-from apify_shared.consts import BOOL_ENV_VARS, DATETIME_ENV_VARS, FLOAT_ENV_VARS, INTEGER_ENV_VARS, STRING_ENV_VARS, ApifyEnvVars
+from apify_shared.consts import BOOL_ENV_VARS, DATETIME_ENV_VARS, FLOAT_ENV_VARS, INTEGER_ENV_VARS, STRING_ENV_VARS, ActorEnvVars, ApifyEnvVars
 
 
 class TestIsAtHome:
@@ -52,8 +52,8 @@ async def test_get_env_use_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> No
             monkeypatch.setenv(string_env_var, expected_get_env[string_get_env_var])
 
         # We need this override so that the actor doesn't fail when connecting to the platform events websocket
-        monkeypatch.delenv(ApifyEnvVars.ACTOR_EVENTS_WS_URL)
-        expected_get_env[ApifyEnvVars.ACTOR_EVENTS_WS_URL.name.lower()] = None
+        monkeypatch.delenv(ActorEnvVars.EVENTS_WEBSOCKET_URL)
+        expected_get_env[ActorEnvVars.EVENTS_WEBSOCKET_URL.name.lower()] = None
 
         await Actor.init()
         assert expected_get_env == Actor.get_env()

tests/unit/test_config.py

@@ -3,7 +3,7 @@
 import pytest
 
 from apify.config import Configuration
-from apify_shared.consts import ApifyEnvVars
+from apify_shared.consts import ActorEnvVars, ApifyEnvVars
 
 
 class TestConfiguration:
@@ -28,13 +28,13 @@ def test_configuration_from_env_vars(self, monkeypatch: pytest.MonkeyPatch) -> N
         monkeypatch.setenv(ApifyEnvVars.PROXY_PASSWORD, 'DUMMY_PROXY_PASSWORD')
         monkeypatch.setenv(ApifyEnvVars.API_BASE_URL, 'DUMMY_API_BASE_URL')
         monkeypatch.setenv(ApifyEnvVars.PROXY_HOSTNAME, 'DUMMY_PROXY_HOSTNAME')
-        monkeypatch.setenv(ApifyEnvVars.DEFAULT_KEY_VALUE_STORE_ID, 'DUMMY_DEFAULT_KEY_VALUE_STORE_ID')
-        monkeypatch.setenv(ApifyEnvVars.DEFAULT_REQUEST_QUEUE_ID, 'DUMMY_DEFAULT_REQUEST_QUEUE_ID')
-        monkeypatch.setenv(ApifyEnvVars.DEFAULT_DATASET_ID, 'DUMMY_DEFAULT_DATASET_ID')
+        monkeypatch.setenv(ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID, 'DUMMY_DEFAULT_KEY_VALUE_STORE_ID')
+        monkeypatch.setenv(ActorEnvVars.DEFAULT_REQUEST_QUEUE_ID, 'DUMMY_DEFAULT_REQUEST_QUEUE_ID')
+        monkeypatch.setenv(ActorEnvVars.DEFAULT_DATASET_ID, 'DUMMY_DEFAULT_DATASET_ID')
         monkeypatch.setenv(ApifyEnvVars.IS_AT_HOME, '1')
         monkeypatch.setenv(ApifyEnvVars.PROXY_PORT, '1234')
-        monkeypatch.setenv(ApifyEnvVars.MEMORY_MBYTES, '1024')
-        monkeypatch.setenv(ApifyEnvVars.STARTED_AT, '2023-01-01T12:34:56.789Z')
+        monkeypatch.setenv(ActorEnvVars.MEMORY_MBYTES, '1024')
+        monkeypatch.setenv(ActorEnvVars.STARTED_AT, '2023-01-01T12:34:56.789Z')
 
         config = Configuration()
         assert config.token == 'DUMMY_TOKEN'
@@ -55,9 +55,9 @@ def test_configuration_from_constructor_arguments(self, monkeypatch: pytest.Monk
         monkeypatch.setenv(ApifyEnvVars.PROXY_PASSWORD, 'DUMMY_PROXY_PASSWORD')
         monkeypatch.setenv(ApifyEnvVars.API_BASE_URL, 'DUMMY_API_BASE_URL')
         monkeypatch.setenv(ApifyEnvVars.PROXY_HOSTNAME, 'DUMMY_PROXY_HOSTNAME')
-        monkeypatch.setenv(ApifyEnvVars.DEFAULT_DATASET_ID, 'DUMMY_DEFAULT_DATASET_ID')
-        monkeypatch.setenv(ApifyEnvVars.DEFAULT_KEY_VALUE_STORE_ID, 'DUMMY_DEFAULT_KEY_VALUE_STORE_ID')
-        monkeypatch.setenv(ApifyEnvVars.DEFAULT_REQUEST_QUEUE_ID, 'DUMMY_DEFAULT_REQUEST_QUEUE_ID')
+        monkeypatch.setenv(ActorEnvVars.DEFAULT_DATASET_ID, 'DUMMY_DEFAULT_DATASET_ID')
+        monkeypatch.setenv(ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID, 'DUMMY_DEFAULT_KEY_VALUE_STORE_ID')
+        monkeypatch.setenv(ActorEnvVars.DEFAULT_REQUEST_QUEUE_ID, 'DUMMY_DEFAULT_REQUEST_QUEUE_ID')
         monkeypatch.setenv(ApifyEnvVars.PROXY_PORT, '1234')
 
         config = Configuration(

tests/unit/test_event_manager.py

@@ -12,7 +12,7 @@
 
 from apify.config import Configuration
 from apify.event_manager import EventManager
-from apify_shared.consts import ActorEventTypes, ApifyEnvVars
+from apify_shared.consts import ActorEnvVars, ActorEventTypes
 
 
 class TestEventManagerLocal:
@@ -269,7 +269,7 @@ async def event_handler(data: Any) -> None:
 
 class TestEventManagerOnPlatform:
     async def test_lifecycle_on_platform_without_websocket(self, monkeypatch: pytest.MonkeyPatch) -> None:
-        monkeypatch.setenv(ApifyEnvVars.ACTOR_EVENTS_WS_URL, 'ws://localhost:56565')
+        monkeypatch.setenv(ActorEnvVars.EVENTS_WEBSOCKET_URL, 'ws://localhost:56565')
 
         config = Configuration()
         event_manager = EventManager(config)
@@ -293,7 +293,7 @@ async def handler(websocket: websockets.server.WebSocketServerProtocol) -> None:
             # When you don't specify a port explicitly, the websocket connection is opened on a random free port.
             # We need to find out which port is that.
             port: int = ws_server.sockets[0].getsockname()[1]  # type: ignore[index]
-            monkeypatch.setenv(ApifyEnvVars.ACTOR_EVENTS_WS_URL, f'ws://localhost:{port}')
+            monkeypatch.setenv(ActorEnvVars.EVENTS_WEBSOCKET_URL, f'ws://localhost:{port}')
 
             config = Configuration()
             event_manager = EventManager(config)
@@ -328,7 +328,7 @@ async def send_platform_event(event_name: ActorEventTypes, data: Any = None) ->
             # When you don't specify a port explicitly, the websocket connection is opened on a random free port.
             # We need to find out which port is that.
             port: int = ws_server.sockets[0].getsockname()[1]  # type: ignore[index]
-            monkeypatch.setenv(ApifyEnvVars.ACTOR_EVENTS_WS_URL, f'ws://localhost:{port}')
+            monkeypatch.setenv(ActorEnvVars.EVENTS_WEBSOCKET_URL, f'ws://localhost:{port}')
 
             config = Configuration()
             event_manager = EventManager(config)

tests/unit/test_utils.py

@@ -26,23 +26,23 @@
     _unique_key_to_request_id,
 )
 from apify.consts import _StorageTypes
-from apify_shared.consts import ApifyEnvVars
+from apify_shared.consts import ActorEnvVars, ApifyEnvVars
 
 
 def test__fetch_and_parse_env_var(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv(ApifyEnvVars.IS_AT_HOME, 'True')
-    monkeypatch.setenv(ApifyEnvVars.MEMORY_MBYTES, '1024')
+    monkeypatch.setenv(ActorEnvVars.MEMORY_MBYTES, '1024')
     monkeypatch.setenv(ApifyEnvVars.META_ORIGIN, 'API')
-    monkeypatch.setenv(ApifyEnvVars.STARTED_AT, '2022-12-02T15:19:34.907Z')
+    monkeypatch.setenv(ActorEnvVars.STARTED_AT, '2022-12-02T15:19:34.907Z')
     monkeypatch.setenv('DUMMY_BOOL', '1')
     monkeypatch.setenv('DUMMY_DATETIME', '2022-12-02T15:19:34.907Z')
     monkeypatch.setenv('DUMMY_INT', '1')
     monkeypatch.setenv('DUMMY_STRING', 'DUMMY')
 
     assert _fetch_and_parse_env_var(ApifyEnvVars.IS_AT_HOME) is True
-    assert _fetch_and_parse_env_var(ApifyEnvVars.MEMORY_MBYTES) == 1024
+    assert _fetch_and_parse_env_var(ActorEnvVars.MEMORY_MBYTES) == 1024
     assert _fetch_and_parse_env_var(ApifyEnvVars.META_ORIGIN) == 'API'
-    assert _fetch_and_parse_env_var(ApifyEnvVars.STARTED_AT) == \
+    assert _fetch_and_parse_env_var(ActorEnvVars.STARTED_AT) == \
         datetime(2022, 12, 2, 15, 19, 34, 907000, tzinfo=timezone.utc)
 
     assert _fetch_and_parse_env_var('DUMMY_BOOL') == '1'  # type: ignore