Skip to content

Commit

Permalink
add testnet and mainnet node for latest code testing (#467)
Browse files Browse the repository at this point in the history
  • Loading branch information
liam-lai authored Mar 4, 2024
1 parent b86c7f6 commit 46ef5e1
Show file tree
Hide file tree
Showing 18 changed files with 663 additions and 9 deletions.
44 changes: 43 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ jobs:
needs: devnet_build_push
environment: devnet
env:
AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- uses: actions/checkout@v4
Expand All @@ -122,6 +122,48 @@ jobs:
done
aws ecs update-service --region ap-southeast-1 --cluster devnet-xdcnode-cluster --service ecs-service-rpc1 --force-new-deployment --no-cli-pager | head -n 10;
rpcnode_terraform_apply:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev-upgrade' && !startsWith(github.ref, 'refs/tags/')
needs: devnet_build_push
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- uses: actions/checkout@v4
- name: Terraform Apply
run: |
cd cicd/terraform
terraform init ${{ env.tf_init_cli_options }}
terraform apply ${{ env.tf_apply_cli_options }}
testnet_dev-upgrade_node:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev-upgrade' && !startsWith(github.ref, 'refs/tags/')
needs: rpcnode_terraform_apply
environment: testnet
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- uses: actions/checkout@v4
- name: ECS Update
run: |
aws ecs update-service --region ap-southeast-1 --cluster testnet-xdcnode-cluster --service ecs-service-testnet-rpc1 --force-new-deployment --no-cli-pager | head -n 10;
mainnet_dev-upgrade_node:
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/dev-upgrade' && !startsWith(github.ref, 'refs/tags/')
needs: rpcnode_terraform_apply
environment: mainnet
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- uses: actions/checkout@v4
- name: ECS Update
run: |
aws ecs update-service --region ap-southeast-1 --cluster mainnet-xdcnode-cluster --service ecs-service-mainnet-rpc1 --force-new-deployment --no-cli-pager | head -n 10;
devnet_send_notification:
runs-on: ubuntu-latest
Expand Down
3 changes: 1 addition & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ profile.cov
**/yarn-error.log
coverage.txt
go.sum
cicd/devnet/terraform/.terraform*
**/.terraform*
cicd/devnet/tmp
.env
cicd/devnet/terraform/node-config.json
2 changes: 1 addition & 1 deletion cicd/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ docker build -f cicd/Dockerfile .
```
## Docker Run
```
docker run -it -e NETWORK=devnet -e PRIVATE_KEYS=$KEY $IMAGE
docker run -it -e NETWORK=devnet -e PRIVATE_KEY=$KEY $IMAGE
``
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"name": "tfXdcNode",
"image": "xinfinorg/${image_environment}:${image_tag}",
"environment": [
{"name": "PRIVATE_KEYS", "value": "${private_keys}"},
{"name": "PRIVATE_KEY", "value": "${private_key}"},
{"name": "LOG_LEVEL", "value": "${log_level}"},
{"name": "NODE_NAME", "value": "${node_name}"},
{"name": "NETWORK", "value": "${chain_network}"}
Expand Down
2 changes: 1 addition & 1 deletion cicd/devnet/terraform/module/region/ecs.tf
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ data template_file devnet_container_definition {
image_environment = "${lookup(each.value, "imageEnvironment", "devnet")}"
image_tag = "${lookup(each.value, "imageTag", "latest")}"
node_name = "${each.key}"
private_keys = "${each.value.pk}"
private_key = "${each.value.pk}"
cloudwatch_group = "tf-${each.key}"
cloudwatch_region = "${var.region}"
log_level = "${lookup(each.value, "logLevel", "${var.logLevel}")}"
Expand Down
2 changes: 1 addition & 1 deletion cicd/mainnet/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ XDC --ethstats ${netstats} --gcmode archive \
--datadir /work/xdcchain --networkid 50 \
-port $port --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \
--rpcport $rpc_port \
--rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,XDPoS \
--rpcapi admin,db,eth,debug,net,shh,txpool,personal,web3,XDPoS \
--rpcvhosts "*" --unlock "${wallet}" --password /work/.pwd --mine \
--gasprice "1" --targetgaslimit "420000000" --verbosity ${log_level} \
--debugdatadir /work/xdcchain \
Expand Down
13 changes: 13 additions & 0 deletions cicd/terraform/.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
log_level=3

# Ohio
us_east_2_start=0
us_east_2_end=36

# Ireland
eu_west_1_start=37
eu_west_1_end=72

# Sydney
ap_southeast_2_start=73
ap_southeast_2_end=108
28 changes: 28 additions & 0 deletions cicd/terraform/iam.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# IAM policies
data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" {
statement {
actions = ["sts:AssumeRole"]

principals {
type = "Service"
identifiers = ["ecs-tasks.amazonaws.com"]
}
}
}

# Create the role
resource "aws_iam_role" "xdc_ecs_tasks_execution_role" {
name = "xdc-ecs-task-execution-role"
assume_role_policy = "${data.aws_iam_policy_document.xdc_ecs_tasks_execution_role.json}"
}

# Attached the AWS managed policies to the new role
resource "aws_iam_role_policy_attachment" "xdc_ecs_tasks_execution_role" {
for_each = toset([
"arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess",
"arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
"arn:aws:iam::aws:policy/AmazonElasticFileSystemsUtils"
])
role = aws_iam_role.xdc_ecs_tasks_execution_role.name
policy_arn = each.value
}
59 changes: 59 additions & 0 deletions cicd/terraform/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 5.13.1"
}
}
}

# Default
provider "aws" {
region = "us-east-1"
}

# WARNING: APSE-1 will only be used to host rpc node
# Workaround to avoid conflicts with existing ecs cluster in existing regions
provider "aws" {
alias = "ap-southeast-1"
region = "ap-southeast-1"
}

module "testnet-rpc" {
source = "./module/region"
region = "ap-southeast-1"
nodeKeys = local.rpcTestnetNodeKeys
enableFixedIp = true
logLevel = local.logLevel
xdc_ecs_tasks_execution_role_arn = aws_iam_role.xdc_ecs_tasks_execution_role.arn

cpu = 1024
memory = 4096

network = "testnet"
vpc_cidr = "10.1.0.0/16"
subnet_cidr = "10.1.0.0/20"
providers = {
aws = aws.ap-southeast-1
}
}

module "mainnet-rpc" {
source = "./module/region"
region = "ap-southeast-1"
nodeKeys = local.rpcMainnetNodeKeys
enableFixedIp = true
logLevel = local.logLevel
xdc_ecs_tasks_execution_role_arn = aws_iam_role.xdc_ecs_tasks_execution_role.arn

cpu = 1024
memory = 4096

network = "mainnet"
vpc_cidr = "10.2.0.0/16"
subnet_cidr = "10.2.0.0/20"
providers = {
aws = aws.ap-southeast-1
}
}

44 changes: 44 additions & 0 deletions cicd/terraform/module/region/container-definition.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
[
{
"name": "tfXdcNode",
"image": "xinfinorg/${image_environment}:${image_tag}",
"environment": [
{"name": "PRIVATE_KEY", "value": "${private_key}"},
{"name": "LOG_LEVEL", "value": "${log_level}"},
{"name": "NODE_NAME", "value": "${node_name}"},
{"name": "NETWORK", "value": "${chain_network}"}
],
"essential": true,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "${cloudwatch_group}",
"awslogs-region": "${cloudwatch_region}",
"awslogs-stream-prefix": "ecs"
}
},
"portMappings": [
{
"hostPort": 8555,
"protocol": "tcp",
"containerPort": 8555
},
{
"hostPort": 8545,
"protocol": "tcp",
"containerPort": 8545
},
{
"hostPort": 30303,
"protocol": "tcp",
"containerPort": 30303
}
],
"mountPoints": [
{
"containerPath": "/work/xdcchain",
"sourceVolume": "efs"
}
]
}
]
96 changes: 96 additions & 0 deletions cicd/terraform/module/region/ecs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
data template_file container_definition {
for_each = var.nodeKeys
template = "${file("${path.module}/container-definition.tpl")}"

vars = {
image_environment = "${lookup(each.value, "imageEnvironment", "devnet")}"
image_tag = "${lookup(each.value, "imageTag", "latest")}"
node_name = "${each.key}"
private_key = "${each.value.pk}"
cloudwatch_group = "tf-${each.key}"
cloudwatch_region = "${var.region}"
log_level = "${lookup(each.value, "logLevel", "${var.logLevel}")}"
chain_network = var.network
}
}

resource "aws_ecs_task_definition" "task_definition_group" {
for_each = var.nodeKeys

family = "${var.network}-${each.key}"
requires_compatibilities = ["FARGATE"]
network_mode = "awsvpc"
container_definitions = data.template_file.container_definition[each.key].rendered
execution_role_arn = var.xdc_ecs_tasks_execution_role_arn
task_role_arn = var.xdc_ecs_tasks_execution_role_arn

# New nodes will consume a lot more CPU usage than existing nodes.
# This is due to sync is resource heavy. Recommending set to below if doing sync:
# CPU = 2048, Memory = 4096
# Please set it back to cpu 256 and memory of 2048 after sync is done to save the cost
# cpu = 256
# memory = 2048
cpu = var.cpu
memory = var.memory
volume {
name = "efs"

efs_volume_configuration {
file_system_id = aws_efs_file_system.efs[each.key].id
root_directory = "/"
transit_encryption = "ENABLED"
authorization_config {
access_point_id = aws_efs_access_point.efs_access_point[each.key].id
iam = "DISABLED"
}
}
}

tags = {
Name = "Tf${var.network}Ecs-${each.key}"
}
}

data "aws_ecs_task_definition" "ecs_task_definition" {
for_each = var.nodeKeys
task_definition = aws_ecs_task_definition.task_definition_group[each.key].family
}

# ECS cluster
resource "aws_ecs_cluster" "ecs_cluster" {
name = "${var.network}-xdcnode-cluster"
tags = {
Name = "Tf${var.network}EcsCluster"
}
}


resource "aws_ecs_service" "ecs_service" {
for_each = var.enableFixedIp ? {} : var.nodeKeys
name = "ecs-service-${each.key}"
cluster = aws_ecs_cluster.ecs_cluster.id
task_definition = "${aws_ecs_task_definition.task_definition_group[each.key].family}:${max(aws_ecs_task_definition.task_definition_group[each.key].revision, data.aws_ecs_task_definition.ecs_task_definition[each.key].revision)}"
launch_type = "FARGATE"
scheduling_strategy = "REPLICA"
desired_count = 1
force_new_deployment = true
deployment_minimum_healthy_percent = 0
deployment_maximum_percent = 100

network_configuration {
subnets = [aws_subnet.subnet.id]
assign_public_ip = true
security_groups = [
aws_default_security_group.xdcnode_security_group.id
]
}

deployment_circuit_breaker {
enable = true
rollback = false
}

tags = {
Name = "Tf${var.network}EcsService-${each.key}"
}
}
Loading

0 comments on commit 46ef5e1

Please sign in to comment.