Skip to content

Commit efa1f45

Browse files
authored
Add support for IPv6-only environments to exec agent (aws#4708)
1 parent f352595 commit efa1f45

14 files changed

+386
-52
lines changed

agent/app/agent.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ func (agent *ecsAgent) start() int {
347347
return exitcodes.ExitError
348348
}
349349
agent.initializeResourceFields(credentialsManager)
350-
return agent.doStart(containerChangeEventStream, credentialsManager, state, imageManager, client, execcmd.NewManager())
350+
return agent.doStart(containerChangeEventStream, credentialsManager, state, imageManager, client, execcmd.NewManager(agent.cfg))
351351
}
352352

353353
// doStart is the worker invoked by start for starting the ECS Agent. This involves

agent/engine/common_integ_testutil.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func setupGMSALinux(cfg *config.Config, state dockerstate.TaskEngineState, t *te
119119

120120
taskEngine := NewDockerTaskEngine(cfg, dockerClient, credentialsManager,
121121
eventstream.NewEventStream("ENGINEINTEGTEST", context.Background()), imageManager, &hostResourceManager, state, metadataManager,
122-
resourceFields, execcmd.NewManager(), engineserviceconnect.NewManager(), daemonManagers)
122+
resourceFields, execcmd.NewManager(cfg), engineserviceconnect.NewManager(), daemonManagers)
123123
taskEngine.MustInit(context.TODO())
124124
return taskEngine, func() {
125125
taskEngine.Shutdown()
@@ -278,7 +278,7 @@ func SetupIntegTestTaskEngine(cfg *config.Config, state dockerstate.TaskEngineSt
278278

279279
taskEngine := NewDockerTaskEngine(cfg, dockerClient, credentialsManager,
280280
eventstream.NewEventStream("ENGINEINTEGTEST", context.Background()), imageManager, &hostResourceManager, state, metadataManager,
281-
nil, execcmd.NewManager(), engineserviceconnect.NewManager(), daemonManagers)
281+
nil, execcmd.NewManager(cfg), engineserviceconnect.NewManager(), daemonManagers)
282282
taskEngine.MustInit(context.TODO())
283283
return taskEngine, func() {
284284
taskEngine.Shutdown()

agent/engine/docker_task_engine.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2034,8 +2034,7 @@ func (engine *DockerTaskEngine) createContainer(task *apitask.Task, container *a
20342034
}
20352035

20362036
if execcmd.IsExecEnabledContainer(container) {
2037-
tID := task.GetID()
2038-
err := engine.execCmdMgr.InitializeContainer(tID, container, hostConfig)
2037+
err := engine.execCmdMgr.InitializeContainer(task, container, hostConfig)
20392038
if err != nil {
20402039
logger.Warn("Error initializing ExecCommandAgent; proceeding to start container without exec feature", logger.Fields{
20412040
field.TaskID: task.GetID(),

agent/engine/engine_sudo_linux_integ_test.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ func TestExecCommandAgent(t *testing.T) {
296296

297297
testExecCmdHostBinDir := "/managed-agents/execute-command/bin"
298298

299-
taskEngine, done, _ := setupEngineForExecCommandAgent(t, testExecCmdHostBinDir)
299+
taskEngine, done, _, cfg := setupEngineForExecCommandAgent(t, testExecCmdHostBinDir)
300300
stateChangeEvents := taskEngine.StateChangeEvents()
301301
defer done()
302302

@@ -323,7 +323,7 @@ func TestExecCommandAgent(t *testing.T) {
323323
cid := containerMap[testTask.Containers[0].Name].DockerID
324324

325325
// session limit is 2
326-
testConfigFileName, _ := execcmd.GetExecAgentConfigFileName(2)
326+
testConfigFileName, _ := execcmd.GetExecAgentConfigFileName(2, cfg, testTask)
327327
testLogConfigFileName, _ := execcmd.GetExecAgentLogConfigFile()
328328
verifyExecCmdAgentExpectedMounts(t, ctx, client, testTaskId, cid, testContainerName, testExecCmdHostBinDir+"/1.0.0.0", testConfigFileName, testLogConfigFileName)
329329
pidA := verifyMockExecCommandAgentIsRunning(t, client, cid)
@@ -391,7 +391,7 @@ func TestManagedAgentEvent(t *testing.T) {
391391

392392
testExecCmdHostBinDir := "/managed-agents/execute-command/bin"
393393

394-
taskEngine, done, _ := setupEngineForExecCommandAgent(t, testExecCmdHostBinDir)
394+
taskEngine, done, _, _ := setupEngineForExecCommandAgent(t, testExecCmdHostBinDir)
395395
defer done()
396396

397397
testTask := createTestExecCommandAgentTask(testTaskId, testContainerName, time.Minute*tc.ManagedAgentLifetime)
@@ -448,7 +448,9 @@ func createTestExecCommandAgentTask(taskId, containerName string, sleepFor time.
448448
// setupEngineForExecCommandAgent creates a new TaskEngine with a custom execcmd.Manager that will attempt to read the
449449
// host binaries from the directory passed as parameter (as opposed to the default directory).
450450
// Additionally, it overrides the engine's monitorExecAgentsInterval to one second.
451-
func setupEngineForExecCommandAgent(t *testing.T, hostBinDir string) (TaskEngine, func(), credentials.Manager) {
451+
func setupEngineForExecCommandAgent(
452+
t *testing.T, hostBinDir string,
453+
) (TaskEngine, func(), credentials.Manager, *config.Config) {
452454
ctx, cancel := context.WithCancel(context.TODO())
453455
defer cancel()
454456

@@ -465,7 +467,7 @@ func setupEngineForExecCommandAgent(t *testing.T, hostBinDir string) (TaskEngine
465467
imageManager := NewImageManager(cfg, dockerClient, state)
466468
imageManager.SetDataClient(data.NewNoopClient())
467469
metadataManager := containermetadata.NewManager(dockerClient, cfg)
468-
execCmdMgr := execcmd.NewManagerWithBinDir(hostBinDir)
470+
execCmdMgr := execcmd.NewManagerWithBinDir(hostBinDir, cfg)
469471
hostResources := getTestHostResources()
470472
hostResourceManager := NewHostResourceManager(hostResources)
471473
daemonManagers := getTestDaemonManagers()
@@ -477,7 +479,7 @@ func setupEngineForExecCommandAgent(t *testing.T, hostBinDir string) (TaskEngine
477479
taskEngine.MustInit(context.TODO())
478480
return taskEngine, func() {
479481
taskEngine.Shutdown()
480-
}, credentialsManager
482+
}, credentialsManager, cfg
481483
}
482484

483485
const (

agent/engine/engine_windows_integ_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,7 @@ func setupGMSA(cfg *config.Config, state dockerstate.TaskEngineState, t *testing
576576

577577
taskEngine := NewDockerTaskEngine(cfg, dockerClient, credentialsManager,
578578
eventstream.NewEventStream("ENGINEINTEGTEST", context.Background()), imageManager, &hostResourceManager, state, metadataManager,
579-
resourceFields, execcmd.NewManager(), engineserviceconnect.NewManager(), getTestDaemonManagers())
579+
resourceFields, execcmd.NewManager(cfg), engineserviceconnect.NewManager(), getTestDaemonManagers())
580580
taskEngine.MustInit(context.TODO())
581581
return taskEngine, func() {
582582
taskEngine.Shutdown()
@@ -814,7 +814,7 @@ func setupEngineForExecCommandAgent(t *testing.T, hostBinDir string) (TaskEngine
814814
imageManager := NewImageManager(cfg, dockerClient, state)
815815
imageManager.SetDataClient(data.NewNoopClient())
816816
metadataManager := containermetadata.NewManager(dockerClient, cfg)
817-
execCmdMgr := execcmd.NewManagerWithBinDir(hostBinDir)
817+
execCmdMgr := execcmd.NewManagerWithBinDir(hostBinDir, cfg)
818818
hostResources := getTestHostResources()
819819
hostResourceManager := NewHostResourceManager(hostResources)
820820

agent/engine/execcmd/manager.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020

2121
apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container"
2222
apitask "github.com/aws/amazon-ecs-agent/agent/api/task"
23+
"github.com/aws/amazon-ecs-agent/agent/config"
2324
"github.com/aws/amazon-ecs-agent/agent/dockerclient/dockerapi"
2425

2526
"github.com/aws/aws-sdk-go-v2/service/ecs/types"
@@ -66,7 +67,7 @@ func (e StartError) Retry() bool {
6667
}
6768

6869
type Manager interface {
69-
InitializeContainer(taskId string, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) error
70+
InitializeContainer(task *apitask.Task, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) error
7071
StartAgent(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) error
7172
RestartAgentIfStopped(ctx context.Context, client dockerapi.DockerClient, task *apitask.Task, container *apicontainer.Container, containerId string) (RestartStatus, error)
7273
}
@@ -77,20 +78,22 @@ type manager struct {
7778
retryMinDelay time.Duration
7879
startRetryTimeout time.Duration
7980
inspectRetryTimeout time.Duration
81+
agentConfig *config.Config
8082
}
8183

82-
func NewManager() *manager {
84+
func NewManager(cfg *config.Config) *manager {
8385
return &manager{
8486
hostBinDir: HostBinDir,
8587
retryMaxDelay: defaultRetryMaxDelay,
8688
retryMinDelay: defaultRetryMinDelay,
8789
startRetryTimeout: defaultStartRetryTimeout,
8890
inspectRetryTimeout: defaultInspectRetryTimeout,
91+
agentConfig: cfg,
8992
}
9093
}
9194

92-
func NewManagerWithBinDir(hostBinDir string) *manager {
93-
m := NewManager()
95+
func NewManagerWithBinDir(hostBinDir string, cfg *config.Config) *manager {
96+
m := NewManager(cfg)
9497
m.hostBinDir = hostBinDir
9598
return m
9699
}

agent/engine/execcmd/manager_init_task.go

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
dockercontainer "github.com/docker/docker/api/types/container"
3131

3232
apicontainer "github.com/aws/amazon-ecs-agent/agent/api/container"
33+
apitask "github.com/aws/amazon-ecs-agent/agent/api/task"
3334
"github.com/pborman/uuid"
3435
)
3536

@@ -50,14 +51,29 @@ var (
5051
execAgentConfigTemplate = `{
5152
"Mgs": {
5253
"Region": "",
53-
"Endpoint": "",
54+
"Endpoint": "%s",
5455
"StopTimeoutMillis": 20000,
5556
"SessionWorkersLimit": %d
5657
},
5758
"Agent": {
5859
"Region": "",
5960
"OrchestrationRootDir": "",
6061
"ContainerMode": true
62+
},
63+
"Ssm": {
64+
"Endpoint": "%s"
65+
},
66+
"Mds": {
67+
"Endpoint": "%s"
68+
},
69+
"S3": {
70+
"Endpoint": "%s"
71+
},
72+
"Kms": {
73+
"Endpoint": "%s"
74+
},
75+
"CloudWatch": {
76+
"Endpoint": "%s"
6177
}
6278
}`
6379

@@ -66,7 +82,9 @@ var (
6682

6783
// InitializeContainer adds the necessary bind mounts in order for the ExecCommandAgent to run properly in the container
6884
// TODO: [ecs-exec] Should we validate the ssm agent binaries & certs are valid and fail here if they're not? (bind mount will succeed even if files don't exist in host)
69-
func (m *manager) InitializeContainer(taskId string, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig) (rErr error) {
85+
func (m *manager) InitializeContainer(
86+
task *apitask.Task, container *apicontainer.Container, hostConfig *dockercontainer.HostConfig,
87+
) (rErr error) {
7088
defer func() {
7189
if rErr != nil {
7290
container.UpdateManagedAgentByName(ExecuteCommandAgentName, apicontainer.ManagedAgentState{
@@ -89,7 +107,8 @@ func (m *manager) InitializeContainer(taskId string, container *apicontainer.Con
89107
return rErr
90108
}
91109

92-
rErr = addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid, sessionWorkersLimit, hostConfig)
110+
rErr = addRequiredBindMounts(task, cn, latestBinVersionDir, uuid, sessionWorkersLimit,
111+
hostConfig, m.agentConfig)
93112
if rErr != nil {
94113
return rErr
95114
}

agent/engine/execcmd/manager_init_task_linux.go

Lines changed: 105 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,15 @@
1616
package execcmd
1717

1818
import (
19+
"errors"
1920
"fmt"
2021
"path/filepath"
2122
"strings"
2223

24+
apitask "github.com/aws/amazon-ecs-agent/agent/api/task"
25+
"github.com/aws/amazon-ecs-agent/agent/config"
26+
"github.com/aws/amazon-ecs-agent/agent/utils/endpoints"
27+
"github.com/aws/amazon-ecs-agent/ecs-agent/logger"
2328
dockercontainer "github.com/docker/docker/api/types/container"
2429
)
2530

@@ -114,25 +119,114 @@ func validConfigExists(configFilePath, expectedHash string) bool {
114119

115120
var GetExecAgentConfigFileName = getAgentConfigFileName
116121

117-
func getAgentConfigFileName(sessionLimit int) (string, error) {
118-
config := fmt.Sprintf(execAgentConfigTemplate, sessionLimit)
122+
// formatSSMAgentConfig creates the SSM agent configuration with the appropriate endpoints
123+
// based on whether we're in an IPv6-only environment
124+
func formatSSMAgentConfig(sessionLimit int, cfg *config.Config, task *apitask.Task) (string, error) {
125+
var (
126+
mgsEndpoint string
127+
ssmEndpoint string
128+
mdsEndpoint string
129+
s3Endpoint string
130+
kmsEndpoint string
131+
cwlEndpoint string
132+
err error
133+
)
134+
135+
// SSM Agent needs to use dualstack endpoints for its dependencies
136+
// if the network only supports IPv6.
137+
useDualStackEndpoints := false
138+
if task.IsNetworkModeAWSVPC() {
139+
// For awsvpc tasks, the task network is used by the SSM Agent
140+
primaryENI := task.GetPrimaryENI()
141+
if primaryENI == nil {
142+
return "", errors.New("awsvpc mode task does not have a primary ENI")
143+
}
144+
useDualStackEndpoints = primaryENI.IPv6Only()
145+
} else {
146+
useDualStackEndpoints = cfg.InstanceIPCompatibility.IsIPv6Only()
147+
}
148+
149+
if useDualStackEndpoints {
150+
// Resolve SSM Messages endpoint
151+
mgsEndpoint, err = endpoints.ResolveSSMMessagesDualStackEndpoint(cfg.AWSRegion)
152+
if err != nil {
153+
return "", fmt.Errorf("failed to resolve SSM Messages endpoint: %w", err)
154+
}
155+
156+
// Resolve SSM endpoint
157+
ssmEndpoint, err = endpoints.ResolveSSMEndpoint(cfg.AWSRegion, true)
158+
if err != nil {
159+
return "", fmt.Errorf("failed to resolve SSM endpoint: %w", err)
160+
}
161+
162+
// Resolve EC2 Messages endpoint
163+
mdsEndpoint, err = endpoints.ResolveEC2MessagesDualStackEndpoint(cfg.AWSRegion)
164+
if err != nil {
165+
return "", fmt.Errorf("failed to resolve EC2 Messages endpoint: %w", err)
166+
}
167+
168+
// Resolve S3 endpoint
169+
s3Endpoint, err = endpoints.ResolveS3Endpoint(cfg.AWSRegion, true)
170+
if err != nil {
171+
return "", fmt.Errorf("failed to resolve S3 endpoint: %w", err)
172+
}
173+
174+
// Resolve KMS endpoint
175+
kmsEndpoint, err = endpoints.ResolveKMSEndpoint(cfg.AWSRegion, true)
176+
if err != nil {
177+
return "", fmt.Errorf("failed to resolve KMS endpoint: %w", err)
178+
}
179+
180+
// Resolve CloudWatch Logs endpoint
181+
cwlEndpoint, err = endpoints.ResolveCloudWatchLogsEndpoint(cfg.AWSRegion, true)
182+
if err != nil {
183+
return "", fmt.Errorf("failed to resolve CloudWatch Logs endpoint: %w", err)
184+
}
185+
186+
logger.Info("Using dualstack endpoints for SSM Agent in IPv6-only environment", logger.Fields{
187+
"region": cfg.AWSRegion,
188+
"mgsEndpoint": mgsEndpoint,
189+
"ssmEndpoint": ssmEndpoint,
190+
"mdsEndpoint": mdsEndpoint,
191+
"s3Endpoint": s3Endpoint,
192+
"kmsEndpoint": kmsEndpoint,
193+
"cwlEndpoint": cwlEndpoint,
194+
})
195+
}
196+
197+
return fmt.Sprintf(execAgentConfigTemplate, mgsEndpoint, sessionLimit, ssmEndpoint,
198+
mdsEndpoint, s3Endpoint, kmsEndpoint, cwlEndpoint), nil
199+
}
200+
201+
func getAgentConfigFileName(sessionLimit int, cfg *config.Config, task *apitask.Task) (string, error) {
202+
// Format the SSM agent config with appropriate endpoints
203+
config, err := formatSSMAgentConfig(sessionLimit, cfg, task)
204+
if err != nil {
205+
return "", err
206+
}
207+
208+
// Generate a hash of the config to use in the filename
119209
hash := getExecAgentConfigHash(config)
120210
configFileName := fmt.Sprintf(execAgentConfigFileNameTemplate, hash)
121-
// check if config file exists already
211+
212+
// Check if config file exists already
122213
configFilePath := filepath.Join(ECSAgentExecConfigDir, configFileName)
123214
if fileExists(configFilePath) && validConfigExists(configFilePath, hash) {
124215
return configFileName, nil
125216
}
126-
// check if config file is a dir; if true, remove it
217+
218+
// Check if config file is a dir; if true, remove it
127219
if isDir(configFilePath) {
128220
if err := removeAll(configFilePath); err != nil {
129221
return "", err
130222
}
131223
}
132-
// config doesn't exist; create a new one
224+
225+
// Config doesn't exist; create a new one
133226
if err := createNewExecAgentConfigFile(config, configFilePath); err != nil {
134227
return "", err
135228
}
229+
136230
return configFileName, nil
137231
}
138232

@@ -142,8 +236,11 @@ func certsExist() bool {
142236

143237
// This function creates any necessary config directories/files and ensures that
144238
// the ssm-agent binaries, configs, logs, and plugin is bind mounted
145-
func addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid string, sessionWorkersLimit int, hostConfig *dockercontainer.HostConfig) error {
146-
configFile, rErr := GetExecAgentConfigFileName(sessionWorkersLimit)
239+
func addRequiredBindMounts(
240+
task *apitask.Task, cn, latestBinVersionDir, uuid string, sessionWorkersLimit int,
241+
hostConfig *dockercontainer.HostConfig, cfg *config.Config,
242+
) error {
243+
configFile, rErr := GetExecAgentConfigFileName(sessionWorkersLimit, cfg, task)
147244
if rErr != nil {
148245
rErr = fmt.Errorf("could not generate ExecAgent Config File: %v", rErr)
149246
return rErr
@@ -191,7 +288,7 @@ func addRequiredBindMounts(taskId, cn, latestBinVersionDir, uuid string, session
191288

192289
// Add ssm log bind mount
193290
hostConfig.Binds = append(hostConfig.Binds, getBindMountMapping(
194-
filepath.Join(HostLogDir, taskId, cn),
291+
filepath.Join(HostLogDir, task.GetID(), cn),
195292
ContainerLogDir))
196293
return nil
197294
}

0 commit comments

Comments
 (0)