Skip to content

Commit f352595

Browse files
xxx0624xxx0624
authored
Update network latency fault check (aws#4726)
1 parent 2ec8a56 commit f352595

File tree

5 files changed

+207
-12
lines changed

5 files changed

+207
-12
lines changed

‎agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go‎

Lines changed: 14 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/fault/v1/types/types.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1376,6 +1376,7 @@ func (h *FaultHandler) startNetworkLatencyFaultForInterface(
13761376
field.CommandOutput: string(cmdOutput[:]),
13771377
field.NetworkInterface: interfaceName,
13781378
})
1379+
13791380
tcAddQdiscLossCommandComposed := nsenterPrefix + fmt.Sprintf(
13801381
tcAddQdiscLatencyCommandString, interfaceName, delayInMs, jitterInMs)
13811382
cmdList = strings.Split(tcAddQdiscLossCommandComposed, " ")
@@ -1631,14 +1632,23 @@ func (h *FaultHandler) checkTCFaultForInterface(
16311632

16321633
// checkLatencyFault parses the tc command output and checks if there's existing network-latency fault running.
16331634
func checkLatencyFault(outputUnmarshalled []map[string]interface{}) (bool, error) {
1635+
packetLossFaultExist, err := checkPacketLossFault(outputUnmarshalled)
1636+
if err != nil {
1637+
return false, err
1638+
}
1639+
1640+
// Make sure no existing packet loss fault because only one tc fault is allowed.
1641+
if packetLossFaultExist {
1642+
return false, nil
1643+
}
1644+
16341645
for _, line := range outputUnmarshalled {
16351646
// Check if field "kind":"netem" exists.
16361647
if line["kind"] == "netem" {
1637-
// Now check if network packet loss fault exists.
16381648
if options := line["options"]; options != nil {
1639-
if delay := options.(map[string]interface{})["delay"]; delay != nil {
1640-
return true, nil
1641-
}
1649+
// We don't check the "delay" field in the output of "options" intentionally because
1650+
// it is not present if the delay is zero and the jitter is non-zero.
1651+
return true, nil
16421652
}
16431653
}
16441654
}

‎ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go‎

Lines changed: 165 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"io"
2626
"net/http"
2727
"net/http/httptest"
28+
"strings"
2829
"testing"
2930
"time"
3031

@@ -62,10 +63,11 @@ const (
6263
nspath = "/some/path"
6364
nspathHost = "host"
6465
// Fault injection tooling errors output
65-
iptablesChainNotFoundError = "iptables: Bad rule (does a matching rule exist in that chain?)."
66-
tcLatencyFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","parent":"1:1","options":{"limit":1000,"delay":{"delay":123456789,"jitter":4567,"correlation":0},"ecn":false,"gap":0}}]`
67-
tcLossFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","dev":"eth0","parent":"1:1","options":{"limit":1000,"loss-random":{"loss":0.06,"correlation":0},"ecn":false,"gap":0}}]`
68-
tcCommandEmptyOutput = `[]`
66+
iptablesChainNotFoundError = "iptables: Bad rule (does a matching rule exist in that chain?)."
67+
tcLatencyFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","parent":"1:1","options":{"limit":1000,"delay":{"delay":123456789,"jitter":4567,"correlation":0},"ecn":false,"gap":0}}]`
68+
tcLatencyFaultWithZeroDelayExistsCommandOutput = `[{"kind":"netem","handle":"10:","parent":"1:1","options":{"limit":1000,"ecn":false,"gap":0}}]`
69+
tcLossFaultExistsCommandOutput = `[{"kind":"netem","handle":"10:","dev":"eth0","parent":"1:1","options":{"limit":1000,"loss-random":{"loss":0.06,"correlation":0},"ecn":false,"gap":0}}]`
70+
tcCommandEmptyOutput = `[]`
6971
// Common Fault injection JSON responses
7072
happyFaultRunningResponse = `{"Status":"running"}`
7173
happyFaultStoppedResponse = `{"Status":"stopped"}`
@@ -234,6 +236,13 @@ var (
234236
"SourcesToFilter": ipSourcesToFilter,
235237
}
236238

239+
happyNetworkLatencyReqBodyWithZeroDelay = map[string]interface{}{
240+
"DelayMilliseconds": 0,
241+
"JitterMilliseconds": jitterMilliseconds,
242+
"Sources": ipSources,
243+
"SourcesToFilter": ipSourcesToFilter,
244+
}
245+
237246
happyNetworkPacketLossReqBody = map[string]interface{}{
238247
"LossPercent": lossPercent,
239248
"Sources": ipSources,
@@ -2175,6 +2184,39 @@ func generateStartNetworkLatencyTestCases() []networkFaultInjectionTestCase {
21752184
},
21762185
expectedResponseJSON: happyFaultRunningResponse,
21772186
},
2187+
{
2188+
name: "zero-delay",
2189+
expectedStatusCode: 200,
2190+
requestBody: happyNetworkLatencyReqBodyWithZeroDelay,
2191+
expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("running"),
2192+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2193+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).Return(happyTaskResponse, nil)
2194+
},
2195+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
2196+
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2197+
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
2198+
2199+
commandList := strings.Split(
2200+
fmt.Sprintf("nsenter --net=/some/path "+tcAddQdiscLatencyCommandString, "eth0", 0, jitterMilliseconds),
2201+
" ")
2202+
gomock.InOrder(
2203+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel),
2204+
// Check existing fault
2205+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD),
2206+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil),
2207+
)
2208+
// Add root handler
2209+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD)
2210+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil)
2211+
// Add lantecy handler
2212+
exec.EXPECT().CommandContext(gomock.Any(), commandList[0], commandList[1:]).Times(1).Return(mockCMD)
2213+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcCommandEmptyOutput), nil)
2214+
// Add targets to the handler
2215+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(5).Return(mockCMD)
2216+
mockCMD.EXPECT().CombinedOutput().Times(5).Return([]byte(tcCommandEmptyOutput), nil)
2217+
},
2218+
expectedResponseJSON: happyFaultRunningResponse,
2219+
},
21782220
{
21792221
name: "no-existing-fault - two interfaces",
21802222
expectedStatusCode: 200,
@@ -2215,6 +2257,23 @@ func generateStartNetworkLatencyTestCases() []networkFaultInjectionTestCase {
22152257
},
22162258
expectedResponseJSON: fmt.Sprintf(errorResponse, latencyFaultAlreadyRunningError),
22172259
},
2260+
{
2261+
name: "existing-network-latency-fault-with-0-delay",
2262+
expectedStatusCode: 409,
2263+
requestBody: happyNetworkLatencyReqBody,
2264+
expectedResponseBody: types.NewNetworkFaultInjectionErrorResponse(latencyFaultAlreadyRunningError),
2265+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2266+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).Return(happyTaskResponse, nil)
2267+
},
2268+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
2269+
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2270+
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
2271+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel)
2272+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD)
2273+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultWithZeroDelayExistsCommandOutput), nil)
2274+
},
2275+
expectedResponseJSON: fmt.Sprintf(errorResponse, latencyFaultAlreadyRunningError),
2276+
},
22182277
{
22192278
name: "existing-network-packet-loss-fault",
22202279
expectedStatusCode: 409,
@@ -2357,6 +2416,21 @@ func generateStartNetworkLatencyTestCases() []networkFaultInjectionTestCase {
23572416
},
23582417
expectedResponseJSON: fmt.Sprintf(errorResponse, fmt.Sprintf(types.MissingRequiredFieldError, "JitterMilliseconds")),
23592418
},
2419+
{
2420+
name: fmt.Sprintf("%s invalid delay and jitter", startNetworkLatencyTestPrefix),
2421+
expectedStatusCode: 400,
2422+
requestBody: map[string]interface{}{
2423+
"DelayMilliseconds": 0,
2424+
"JitterMilliseconds": 0,
2425+
"Sources": ipSources,
2426+
"SourcesToFilter": ipSourcesToFilter,
2427+
},
2428+
expectedResponseBody: types.NewNetworkFaultInjectionErrorResponse(types.ZeroDelayAndJitterError),
2429+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2430+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).Return(state.TaskResponse{}, nil).Times(0)
2431+
},
2432+
expectedResponseJSON: fmt.Sprintf(errorResponse, types.ZeroDelayAndJitterError),
2433+
},
23602434
{
23612435
name: fmt.Sprintf("%s invalid DelayMilliseconds in the request body 1", startNetworkLatencyTestPrefix),
23622436
expectedStatusCode: 400,
@@ -2493,6 +2567,27 @@ func generateStopNetworkLatencyTestCases() []networkFaultInjectionTestCase {
24932567
},
24942568
expectedResponseJSON: happyFaultStoppedResponse,
24952569
},
2570+
{
2571+
name: "existing-network-latency-fault-with-0-delay-happy-request-payload",
2572+
expectedStatusCode: 200,
2573+
requestBody: happyNetworkLatencyReqBody,
2574+
expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("stopped"),
2575+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2576+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).Return(happyTaskResponse, nil)
2577+
},
2578+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
2579+
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2580+
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
2581+
gomock.InOrder(
2582+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel),
2583+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD),
2584+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultWithZeroDelayExistsCommandOutput), nil),
2585+
)
2586+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(2).Return(mockCMD)
2587+
mockCMD.EXPECT().CombinedOutput().Times(2).Return([]byte(""), nil)
2588+
},
2589+
expectedResponseJSON: happyFaultStoppedResponse,
2590+
},
24962591
{
24972592
name: "existing-network-packet-loss-fault-empty-request-payload",
24982593
expectedStatusCode: 200,
@@ -2610,6 +2705,25 @@ func generateCheckNetworkLatencyTestCases() []networkFaultInjectionTestCase {
26102705
},
26112706
expectedResponseJSON: happyFaultRunningResponse,
26122707
},
2708+
{
2709+
name: "existing-network-latency-fault-with-0-delay-happy-request-payload",
2710+
expectedStatusCode: 200,
2711+
requestBody: happyNetworkLatencyReqBody,
2712+
expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("running"),
2713+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
2714+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).Return(happyTaskResponse, nil)
2715+
},
2716+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
2717+
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
2718+
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
2719+
gomock.InOrder(
2720+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel),
2721+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD),
2722+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultWithZeroDelayExistsCommandOutput), nil),
2723+
)
2724+
},
2725+
expectedResponseJSON: happyFaultRunningResponse,
2726+
},
26132727
{
26142728
name: "existing-network-packet-loss-fault-empty-request-payload",
26152729
expectedStatusCode: 200,
@@ -2883,6 +2997,23 @@ func generateStartNetworkPacketLossTestCases() []networkFaultInjectionTestCase {
28832997
},
28842998
expectedResponseJSON: fmt.Sprintf(errorResponse, latencyFaultAlreadyRunningError),
28852999
},
3000+
{
3001+
name: "existing-network-latency-fault-with-0-delay",
3002+
expectedStatusCode: 409,
3003+
requestBody: happyNetworkPacketLossReqBody,
3004+
expectedResponseBody: types.NewNetworkFaultInjectionErrorResponse(latencyFaultAlreadyRunningError),
3005+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
3006+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).Return(happyTaskResponse, nil)
3007+
},
3008+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
3009+
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
3010+
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
3011+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel)
3012+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD)
3013+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultWithZeroDelayExistsCommandOutput), nil)
3014+
},
3015+
expectedResponseJSON: fmt.Sprintf(errorResponse, latencyFaultAlreadyRunningError),
3016+
},
28863017
{
28873018
name: "existing-network-packet-loss-fault",
28883019
expectedStatusCode: 409,
@@ -3280,6 +3411,25 @@ func generateCheckNetworkPacketLossTestCases() []networkFaultInjectionTestCase {
32803411
},
32813412
expectedResponseJSON: happyFaultNotRunningResponse,
32823413
},
3414+
{
3415+
name: "existing-network-latency-fault-with-0-delay-happy-request-payload",
3416+
expectedStatusCode: 200,
3417+
requestBody: happyNetworkPacketLossReqBody,
3418+
expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("not-running"),
3419+
setAgentStateExpectations: func(agentState *mock_state.MockAgentState, netConfigClient *netconfig.NetworkConfigClient) {
3420+
agentState.EXPECT().GetTaskMetadataWithTaskNetworkConfig(endpointId, netConfigClient).Return(happyTaskResponse, nil)
3421+
},
3422+
setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) {
3423+
ctx, cancel := context.WithTimeout(context.Background(), ctxTimeoutDuration)
3424+
mockCMD := mock_execwrapper.NewMockCmd(ctrl)
3425+
gomock.InOrder(
3426+
exec.EXPECT().NewExecContextWithTimeout(gomock.Any(), gomock.Any()).Times(1).Return(ctx, cancel),
3427+
exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(mockCMD),
3428+
mockCMD.EXPECT().CombinedOutput().Times(1).Return([]byte(tcLatencyFaultWithZeroDelayExistsCommandOutput), nil),
3429+
)
3430+
},
3431+
expectedResponseJSON: happyFaultNotRunningResponse,
3432+
},
32833433
{
32843434
name: "existing-network-packet-loss-fault-empty-request-payload",
32853435
expectedStatusCode: 200,
@@ -3451,6 +3601,17 @@ func TestCheckTCFault(t *testing.T) {
34513601
cmd.EXPECT().CombinedOutput().Return([]byte(tcLatencyFaultExistsCommandOutput), nil)
34523602
},
34533603
},
3604+
{
3605+
name: "Latency fault with 0 delay exists",
3606+
deviceNames: []string{"eth0"},
3607+
expectedLatency: true,
3608+
expectedPacketLoss: false,
3609+
expectedError: nil,
3610+
commandExpectations: func(exec *mock_execwrapper.MockExec, cmd *mock_execwrapper.MockCmd) {
3611+
exec.EXPECT().CommandContext(gomock.Any(), "tc", []string{"-j", "q", "show", "dev", "eth0", "parent", "1:1"}).Return(cmd)
3612+
cmd.EXPECT().CombinedOutput().Return([]byte(tcLatencyFaultWithZeroDelayExistsCommandOutput), nil)
3613+
},
3614+
},
34543615
{
34553616
name: "Packet loss fault exists",
34563617
deviceNames: []string{"eth0"},

‎ecs-agent/tmds/handlers/fault/v1/types/types.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020

2121
"github.com/aws/amazon-ecs-agent/ecs-agent/tmds/utils"
2222
"github.com/aws/aws-sdk-go-v2/aws"
23+
"github.com/pkg/errors"
2324
)
2425

2526
const (
@@ -34,6 +35,7 @@ const (
3435
// Request Payload Errors
3536
MissingRequiredFieldError = "required parameter %s is missing"
3637
MissingRequestBodyError = "required request body is missing"
38+
ZeroDelayAndJitterError = "required either DelayMilliseconds or JitterMilliseconds to be non-zero"
3739
InvalidValueError = "invalid value %s for parameter %s"
3840
)
3941

@@ -129,6 +131,11 @@ func (request NetworkLatencyRequest) ValidateRequest() error {
129131
if request.JitterMilliseconds == nil {
130132
return fmt.Errorf(MissingRequiredFieldError, "JitterMilliseconds")
131133
}
134+
135+
if aws.ToUint64(request.DelayMilliseconds) == 0 && aws.ToUint64(request.JitterMilliseconds) == 0 {
136+
return errors.New(ZeroDelayAndJitterError)
137+
}
138+
132139
if len(request.Sources) == 0 {
133140
return fmt.Errorf(MissingRequiredFieldError, "Sources")
134141
}

0 commit comments

Comments
 (0)