Skip to content

feat: Build docker compose with custom docker images #2296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

feat: Build docker compose with custom docker images #2296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bc68076
Write minimal composer file
nrosa-valory Jan 28, 2025
4e47ce8
Write custom composer file
nrosa-valory Jan 30, 2025
6718a9c
Update documentation
nrosa-valory Feb 5, 2025
62a7bea
Adjust formatting
nrosa-valory Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 92 additions & 78 deletions autonomy/deploy/generators/docker_compose/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
from autonomy.deploy.generators.docker_compose.templates import (
ABCI_NODE_TEMPLATE,
ACN_NODE_TEMPLATE,
CUSTOM_DOCKER_COMPOSE_TEMPLATE,
DOCKER_COMPOSE_TEMPLATE,
HARDHAT_NODE_TEMPLATE,
PORTS,
Expand Down Expand Up @@ -339,99 +340,112 @@ def generate(
image_version: Optional[str] = None,
use_hardhat: bool = False,
use_acn: bool = False,
custom_docker_image_name: Optional[str] = None,
) -> "DockerComposeGenerator":
"""Generate the new configuration."""
network_name = f"service_{self.service_builder.service.name}_localnet"
used_networks = self._find_occupied_networks(network_name)
network = Network(name=network_name, used_subnets=used_networks)
image_version = image_version or self.service_builder.service.agent.hash
if self.dev_mode:
runtime_image = DEVELOPMENT_IMAGE

if custom_docker_image_name is not None:
self.output = CUSTOM_DOCKER_COMPOSE_TEMPLATE.format(
docker_image_name=custom_docker_image_name
)
else:
runtime_image = OAR_IMAGE.format(
image_author=self.image_author,
agent=self.service_builder.service.agent.name,
version=image_version,
network_name = f"service_{self.service_builder.service.name}_localnet"
used_networks = self._find_occupied_networks(network_name)
network = Network(name=network_name, used_subnets=used_networks)
image_version = image_version or self.service_builder.service.agent.hash
if self.dev_mode:
runtime_image = DEVELOPMENT_IMAGE
else:
runtime_image = OAR_IMAGE.format(
image_author=self.image_author,
agent=self.service_builder.service.agent.name,
version=image_version,
)
Comment on lines +359 to +363
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I might be missing something, but it appears that if the docker image is tagged as formatted by OAR_IMAGE ({image_author}/oar-{agent}:{version}), then these changes will not be needed. Have you tried that @nrosa-valory?

And if other runtimes like terndermint are not needed then only those can be made configurable, as suggested by 8ball030

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@OjusWiZard I see your point, but this runtime_image is used to build the agent configuration, which has a lot of other fields we probably dont need and would need to either add checks if they exist or add some default values, then this agent config is used to build the composer and it has many other information, like tendermint nodes, hardhat nodes, acn nodes.

Looking into it I feel like it will be quite messy to try to adapt to this simpler version of a composer, but if you think this is something worth persuing I can give it a try.


agent_vars = self.service_builder.generate_agents()
agents = "".join(
[
build_agent_config(
node_id=i,
container_name=self.service_builder.get_abci_container_name(
index=i
),
build_dir=self.build_dir,
runtime_image=runtime_image,
agent_vars=agent_vars[i],
dev_mode=self.dev_mode,
package_dir=self.packages_dir,
open_aea_dir=self.open_aea_dir,
agent_ports=(
self.service_builder.service.deployment_config.get(
"agent", {}
)
.get("ports", {})
.get(i)
),
network_name=network_name,
network_address=network.next_address,
resources=self.resources,
extra_volumes=self.service_builder.service.deployment_config.get(
"agent", {}
).get(
"volumes"
),
)
for i in range(self.service_builder.service.number_of_agents)
]
)
tendermint_nodes = "".join(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would be sick if this was condiutional.
Not all agents need tendermint

[
build_tendermint_node_config(
node_id=i,
container_name=self.service_builder.get_tm_container_name(
index=i
),
abci_node=self.service_builder.get_abci_container_name(index=i),
dev_mode=self.dev_mode,
log_level=self.service_builder.log_level,
tendermint_ports=(
self.service_builder.service.deployment_config.get(
"tendermint", {}
)
.get("ports", {})
.get(i)
),
network_name=network_name,
network_address=network.next_address,
)
for i in range(self.service_builder.service.number_of_agents)
]
)

agent_vars = self.service_builder.generate_agents()
agents = "".join(
[
build_agent_config(
node_id=i,
container_name=self.service_builder.get_abci_container_name(
index=i
),
build_dir=self.build_dir,
runtime_image=runtime_image,
agent_vars=agent_vars[i],
dev_mode=self.dev_mode,
package_dir=self.packages_dir,
open_aea_dir=self.open_aea_dir,
agent_ports=(
self.service_builder.service.deployment_config.get("agent", {})
.get("ports", {})
.get(i)
),
hardhat_node = ""
if use_hardhat:
hardhat_node = HARDHAT_NODE_TEMPLATE.format(
hardhat_image_name=HARDHAT_IMAGE_NAME,
hardhat_image_version=HARDHAT_IMAGE_VERSION,
network_name=network_name,
network_address=network.next_address,
resources=self.resources,
extra_volumes=self.service_builder.service.deployment_config.get(
"agent", {}
).get("volumes"),
)
for i in range(self.service_builder.service.number_of_agents)
]
)
tendermint_nodes = "".join(
[
build_tendermint_node_config(
node_id=i,
container_name=self.service_builder.get_tm_container_name(index=i),
abci_node=self.service_builder.get_abci_container_name(index=i),
dev_mode=self.dev_mode,
log_level=self.service_builder.log_level,
tendermint_ports=(
self.service_builder.service.deployment_config.get(
"tendermint", {}
)
.get("ports", {})
.get(i)
),

acn_node = ""
if use_acn:
acn_node = ACN_NODE_TEMPLATE.format(
acn_image_name=ACN_IMAGE_NAME,
acn_image_version=ACN_IMAGE_VERSION,
network_name=network_name,
network_address=network.next_address,
)
for i in range(self.service_builder.service.number_of_agents)
]
)

hardhat_node = ""
if use_hardhat:
hardhat_node = HARDHAT_NODE_TEMPLATE.format(
hardhat_image_name=HARDHAT_IMAGE_NAME,
hardhat_image_version=HARDHAT_IMAGE_VERSION,
self.output = DOCKER_COMPOSE_TEMPLATE.format(
abci_nodes=agents,
tendermint_nodes=tendermint_nodes,
hardhat_node=hardhat_node,
acn_node=acn_node,
network_name=network_name,
network_address=network.next_address,
subnet=str(network.subnet),
)

acn_node = ""
if use_acn:
acn_node = ACN_NODE_TEMPLATE.format(
acn_image_name=ACN_IMAGE_NAME,
acn_image_version=ACN_IMAGE_VERSION,
network_name=network_name,
network_address=network.next_address,
)

self.output = DOCKER_COMPOSE_TEMPLATE.format(
abci_nodes=agents,
tendermint_nodes=tendermint_nodes,
hardhat_node=hardhat_node,
acn_node=acn_node,
network_name=network_name,
subnet=str(network.subnet),
)

return self

def _populate_keys(self) -> None:
Expand Down
12 changes: 12 additions & 0 deletions autonomy/deploy/generators/docker_compose/templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,18 @@
"""bash /app/build.sh "{validators}" "{hosts}" "{user}" """
)

CUSTOM_DOCKER_COMPOSE_TEMPLATE: str = """
version: '3.8'

services:
app:
image: {docker_image_name}
ports:
- "8080:80"
env_file:
- .env
"""

DOCKER_COMPOSE_TEMPLATE: str = """version: "2.4"
services:
{hardhat_node}{acn_node}{tendermint_nodes}{abci_nodes}
Expand Down
9 changes: 6 additions & 3 deletions docs/api/deploy/generators/docker_compose/base.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,12 @@ Generate the command to configure tendermint testnet.
#### generate

```python
def generate(image_version: Optional[str] = None,
use_hardhat: bool = False,
use_acn: bool = False) -> "DockerComposeGenerator"
def generate(
image_version: Optional[str] = None,
use_hardhat: bool = False,
use_acn: bool = False,
custom_docker_image_name: Optional[str] = None
) -> "DockerComposeGenerator"
```

Generate the new configuration.
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ mkdocs-monorepo-plugin = "^1.0.5"

[build-system]
requires = ["setuptools", "wheel","poetry-core>=1.0.0"]
build_backend = "setuptools.build_meta"
build-backend = "setuptools.build_meta"
Loading