From d0615e2b97a59470616edb460edb7e364becd20a Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Feb 2024 12:31:23 +0100 Subject: [PATCH 1/6] Fix retries on GCP deployment --- examples/cloud_deployment/gcp/deploy.py | 25 +++++++++++++++---- .../deploy/agent_example.py | 9 ++++--- .../deploy/gcp/utils.py | 9 +++++++ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/examples/cloud_deployment/gcp/deploy.py b/examples/cloud_deployment/gcp/deploy.py index eca66f74..39dba896 100644 --- a/examples/cloud_deployment/gcp/deploy.py +++ b/examples/cloud_deployment/gcp/deploy.py @@ -1,13 +1,24 @@ import getpass +import typer -from prediction_market_agent_tooling.deploy.agent_example import DeployableCoinFlipAgent +from prediction_market_agent_tooling.deploy.agent_example import ( + DeployableAgent, + DeployableCoinFlipAgent, + DeployableAlwaysRaiseAgent, +) from prediction_market_agent_tooling.markets.markets import MarketType -if __name__ == "__main__": - agent = DeployableCoinFlipAgent() + +def main( + agent_name: str, cron_schedule: str = "0 */2 * * *", branch: str = "main" +) -> None: + agent: DeployableAgent = { + "coin_flip": DeployableCoinFlipAgent, + "always_raise": DeployableAlwaysRaiseAgent, + }[agent_name]() agent.deploy_gcp( # TODO: Switch to main. - repository="git+https://github.com/gnosis/prediction-market-agent-tooling.git@peter/refactor-deployment", + repository="git+https://github.com/gnosis/prediction-market-agent-tooling.git@{branch}", market_type=MarketType.MANIFOLD, labels={ "owner": getpass.getuser() @@ -18,5 +29,9 @@ "MANIFOLD_API_KEY": f"JUNG_PERSONAL_GMAIL_MANIFOLD_API_KEY:latest" }, # Must be in the format "env_var_in_container => secret_name:version", you can create secrets using `gcloud secrets create --labels owner= ` command. memory=256, - cron_schedule="0 */2 * * *", + cron_schedule=cron_schedule, ) + + +if __name__ == "__main__": + typer.run(main) diff --git a/prediction_market_agent_tooling/deploy/agent_example.py b/prediction_market_agent_tooling/deploy/agent_example.py index 8d2e822d..393c309e 100644 --- a/prediction_market_agent_tooling/deploy/agent_example.py +++ b/prediction_market_agent_tooling/deploy/agent_example.py @@ -6,9 +6,12 @@ class DeployableCoinFlipAgent(DeployableAgent): def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: - if len(markets) > 1: - return random.sample(markets, 1) - return markets + return random.sample(markets, 1) def answer_binary_market(self, market: AgentMarket) -> bool: return random.choice([True, False]) + + +class DeployableAlwaysRaiseAgent(DeployableAgent): + def answer_binary_market(self, market: AgentMarket) -> bool: + raise RuntimeError("I always raise!") diff --git a/prediction_market_agent_tooling/deploy/gcp/utils.py b/prediction_market_agent_tooling/deploy/gcp/utils.py index 9b331fa8..0f557982 100644 --- a/prediction_market_agent_tooling/deploy/gcp/utils.py +++ b/prediction_market_agent_tooling/deploy/gcp/utils.py @@ -15,6 +15,8 @@ def gcloud_deploy_cmd( env_vars: dict[str, str] | None, secrets: dict[str, str] | None, memory: int, # in MB + timeout: int = 180, + retry_on_failure: bool = False, ) -> str: cmd = ( f"gcloud functions deploy {gcp_function_name} " @@ -26,7 +28,14 @@ def gcloud_deploy_cmd( f"--entry-point {entry_point} " f"--memory {memory}MB " f"--no-allow-unauthenticated " + f"--timeout {timeout}s " + # Explicitly set no concurrency, min instances to 0 (agent is executed only once in a while) and max instances to 1 (parallel agents aren't allowed). + "--concurrency 1 " + "--min-instances 0 " + "--max-instances 1 " ) + if retry_on_failure: + cmd += "--retry " if labels: for k, v in labels.items(): cmd += f"--update-labels {k}={v} " From dc0de779f97173143ab2b934d599a2e2cbeab26b Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Feb 2024 12:32:16 +0100 Subject: [PATCH 2/6] Fix variable --- examples/cloud_deployment/gcp/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cloud_deployment/gcp/deploy.py b/examples/cloud_deployment/gcp/deploy.py index 39dba896..e9c06da0 100644 --- a/examples/cloud_deployment/gcp/deploy.py +++ b/examples/cloud_deployment/gcp/deploy.py @@ -18,7 +18,7 @@ def main( }[agent_name]() agent.deploy_gcp( # TODO: Switch to main. - repository="git+https://github.com/gnosis/prediction-market-agent-tooling.git@{branch}", + repository=f"git+https://github.com/gnosis/prediction-market-agent-tooling.git@{branch}", market_type=MarketType.MANIFOLD, labels={ "owner": getpass.getuser() From 10d8625d298ed95d8d15229e44cbe4f062211525 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Feb 2024 12:51:25 +0100 Subject: [PATCH 3/6] Fix type --- prediction_market_agent_tooling/markets/data_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prediction_market_agent_tooling/markets/data_models.py b/prediction_market_agent_tooling/markets/data_models.py index 68cc5e4f..6cb30db1 100644 --- a/prediction_market_agent_tooling/markets/data_models.py +++ b/prediction_market_agent_tooling/markets/data_models.py @@ -156,7 +156,7 @@ class ManifoldMarket(BaseModel): isResolved: bool resolution: t.Optional[str] = None resolutionTime: t.Optional[datetime] = None - lastBetTime: datetime + lastBetTime: t.Optional[datetime] = None lastCommentTime: t.Optional[datetime] = None lastUpdatedTime: datetime mechanism: str From 9169eea3a2c504998a8229da69f25a54c76f46a4 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Feb 2024 12:51:44 +0100 Subject: [PATCH 4/6] lint --- examples/cloud_deployment/gcp/deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/cloud_deployment/gcp/deploy.py b/examples/cloud_deployment/gcp/deploy.py index e9c06da0..27963570 100644 --- a/examples/cloud_deployment/gcp/deploy.py +++ b/examples/cloud_deployment/gcp/deploy.py @@ -1,10 +1,11 @@ import getpass + import typer from prediction_market_agent_tooling.deploy.agent_example import ( DeployableAgent, - DeployableCoinFlipAgent, DeployableAlwaysRaiseAgent, + DeployableCoinFlipAgent, ) from prediction_market_agent_tooling.markets.markets import MarketType From 6cf89e4c40935367cac4490015967b228c93e834 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Feb 2024 13:11:04 +0100 Subject: [PATCH 5/6] remove todo --- examples/cloud_deployment/gcp/deploy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/cloud_deployment/gcp/deploy.py b/examples/cloud_deployment/gcp/deploy.py index 27963570..7b5dff03 100644 --- a/examples/cloud_deployment/gcp/deploy.py +++ b/examples/cloud_deployment/gcp/deploy.py @@ -18,7 +18,6 @@ def main( "always_raise": DeployableAlwaysRaiseAgent, }[agent_name]() agent.deploy_gcp( - # TODO: Switch to main. repository=f"git+https://github.com/gnosis/prediction-market-agent-tooling.git@{branch}", market_type=MarketType.MANIFOLD, labels={ From 53b9c19333173a074e94474c1f0c6774c4f1bc22 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 19 Feb 2024 13:36:29 +0100 Subject: [PATCH 6/6] custom gcp name support --- examples/cloud_deployment/gcp/deploy.py | 6 +++++- prediction_market_agent_tooling/deploy/agent.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/cloud_deployment/gcp/deploy.py b/examples/cloud_deployment/gcp/deploy.py index 7b5dff03..77d293d8 100644 --- a/examples/cloud_deployment/gcp/deploy.py +++ b/examples/cloud_deployment/gcp/deploy.py @@ -11,7 +11,10 @@ def main( - agent_name: str, cron_schedule: str = "0 */2 * * *", branch: str = "main" + agent_name: str, + cron_schedule: str = "0 */2 * * *", + branch: str = "main", + custom_gcp_fname: str | None = None, ) -> None: agent: DeployableAgent = { "coin_flip": DeployableCoinFlipAgent, @@ -30,6 +33,7 @@ def main( }, # Must be in the format "env_var_in_container => secret_name:version", you can create secrets using `gcloud secrets create --labels owner= ` command. memory=256, cron_schedule=cron_schedule, + gcp_fname=custom_gcp_fname, ) diff --git a/prediction_market_agent_tooling/deploy/agent.py b/prediction_market_agent_tooling/deploy/agent.py index 1f40bce8..75b4b4f0 100644 --- a/prediction_market_agent_tooling/deploy/agent.py +++ b/prediction_market_agent_tooling/deploy/agent.py @@ -71,6 +71,7 @@ def deploy_gcp( env_vars: dict[str, str] | None = None, secrets: dict[str, str] | None = None, cron_schedule: str | None = None, + gcp_fname: str | None = None, ) -> None: path_to_agent_file = os.path.relpath(inspect.getfile(self.__class__)) @@ -85,7 +86,7 @@ def main(request) -> str: return "Success" """ - gcp_fname = self.get_gcloud_fname(market_type) + gcp_fname = gcp_fname or self.get_gcloud_fname(market_type) with tempfile.NamedTemporaryFile(mode="w", suffix=".py") as f: f.write(entrypoint_template)