Merge branch 'aws:master' into master

Author: singholt

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+## 1.67.2
+* Bug - Fix the generation of network bindings for Service Connect container [#3513](https://github.com/aws/amazon-ecs-agent/pull/3513)
+* Bug - Prevent resetting valid agent state db when IMDS fails on startup [#3509](https://github.com/aws/amazon-ecs-agent/pull/3509)
+
+## 1.67.1
+* Bug - Read git hash from RELEASE_COMMIT file if possible [#3508](https://github.com/aws/amazon-ecs-agent/pull/3508)
+
+## 1.67.0
+* Bug - Don't log errors on instances not using GMSA [#3489](https://github.com/aws/amazon-ecs-agent/pull/3489)
+* Enhancement - Update packaging Readme files with updated instructions to build init files [#3490](https://github.com/aws/amazon-ecs-agent/pull/3490)
+* Bug - Fix unit tests for cgroup v2 [#3491](https://github.com/aws/amazon-ecs-agent/pull/3491)
+* Enhancement - Update readme for ECS_SELINUX_CAPABLE to clarify Z-mode mount only and limited support [#3496](https://github.com/aws/amazon-ecs-agent/pull/3496)
+* Bug - Fix agent short hash version bug [#3497](https://github.com/aws/amazon-ecs-agent/pull/3497)
+* Bug - Use Ubuntu 20.04 for linux GH Unit tests [#3501](https://github.com/aws/amazon-ecs-agent/pull/3501)
+* Feature - Container port range mapping [#3506](https://github.com/aws/amazon-ecs-agent/pull/3506)
+
+## 1.66.2
+* Bug - Add ecs-serviceconnect to CNI and Agent build scripts [#3482](https://github.com/aws/amazon-ecs-agent/pull/3482)
+* Bug - add call to update-version.sh to dockerfree-agent-image [#3484](https://github.com/aws/amazon-ecs-agent/pull/3484)
+
+## 1.66.1
+* Bug - Update ecs agent version short hash to point to built head [#3476](https://github.com/aws/amazon-ecs-agent/pull/3476)
+* Bug - Remove CAP_CHOWN [#3480](https://github.com/aws/amazon-ecs-agent/pull/3480)
+
+## 1.66.0
+* Feature - gMSA on Linux support [#3464](https://github.com/aws/amazon-ecs-agent/pull/3464)
+* Enhancement - Restart AppNet Relay on failure [#3469](Restart AppNet Relay on failure)
+
 ## 1.65.1
 * Enhancement - Add grpc vendor dependencies [#3439](https://github.com/aws/amazon-ecs-agent/pull/3439)
 * Bug - Workaround git-secrets scan issue: awslabs/git-secrets#221 [#3442](https://github.com/aws/amazon-ecs-agent/pull/3442)

Makefile

@@ -52,7 +52,7 @@ static:
 	./scripts/build
 
 static-with-pause:
-	./scripts/build true "" false true
+	./scripts/build true "" true true
 
 # Cross-platform build target for static checks
 xplatform-build:
@@ -377,11 +377,11 @@ amazon-linux-sources.tgz:
 	cp packaging/amazon-linux-ami-integrated/amazon-ecs-volume-plugin.conf amazon-ecs-volume-plugin.conf
 	cp packaging/amazon-linux-ami-integrated/amazon-ecs-volume-plugin.service amazon-ecs-volume-plugin.service
 	cp packaging/amazon-linux-ami-integrated/amazon-ecs-volume-plugin.socket amazon-ecs-volume-plugin.socket
-	tar -czf ./sources.tgz ecs-init scripts misc agent amazon-ecs-cni-plugins amazon-vpc-cni-plugins agent-container VERSION
+	tar -czf ./sources.tgz ecs-init scripts misc agent amazon-ecs-cni-plugins amazon-vpc-cni-plugins agent-container VERSION RELEASE_COMMIT
 
 .amazon-linux-rpm-integrated-done: amazon-linux-sources.tgz
 	test -e SOURCES || ln -s . SOURCES
-	rpmbuild --define "%_topdir $(PWD)" -bb ecs-init.spec
+	rpmbuild --define "%_topdir $(PWD)" -bb ecs-agent.spec
 	find RPMS/ -type f -exec cp {} . \;
 	touch .amazon-linux-rpm-integrated-done
 
@@ -404,8 +404,8 @@ generic-rpm-integrated: .generic-rpm-integrated-done
 VERSION = $(shell cat ecs-init/ECSVERSION)
 
 .generic-deb-integrated-done: get-cni-sources
-	mkdir -p BUILDROOT
 	./scripts/update-version.sh
+	mkdir -p BUILDROOT
 	tar -czf ./amazon-ecs-init_${VERSION}.orig.tar.gz ecs-init scripts README.md
 	cp -r packaging/generic-deb-integrated/debian Makefile ecs-init scripts misc agent agent-container amazon-ecs-cni-plugins amazon-vpc-cni-plugins README.md VERSION GO_VERSION BUILDROOT
 	cd BUILDROOT && dpkg-buildpackage -uc -b
@@ -459,16 +459,17 @@ generic-rpm: .generic-rpm-done
 deb: .deb-done
 
 clean:
-	rm -f misc/certs/host-certs.crt &> /dev/null
-	rm -rf misc/pause-container/image/
-	rm -rf misc/pause-container/rootfs/
-	rm -rf misc/plugins/
-	rm -rf out/
-	rm -rf rootfs/
+	-rm -f misc/certs/host-certs.crt &> /dev/null
+	-rm -rf misc/pause-container/image/
+	-rm -rf misc/pause-container/rootfs/
+	-rm -rf misc/plugins/
+	-rm -rf out/
+	-rm -rf rootfs/
 	-$(MAKE) -C $(ECS_CNI_REPOSITORY_SRC_DIR) clean
 	-rm -f .get-deps-stamp
 	-rm -f .builder-image-stamp
 	-rm -f .out-stamp
+	-rm -f ecs-agent.spec
 	-rm -rf $(PWD)/bin
 	-rm -rf cover.out
 	-rm -rf coverprofile.out

README.md

@@ -146,9 +146,9 @@ additional details on each available environment variable.
 | `ECS_RESERVED_MEMORY` | 32 | Reduction, in MiB, of the memory capacity of the instance that is reported to Amazon ECS. Used by Amazon ECS when placing tasks on container instances. This doesn't reserve memory usage on the instance. | 0 | 0 |
 | `ECS_AVAILABLE_LOGGING_DRIVERS` | `["awslogs","fluentd","gelf","json-file","journald","logentries","splunk","syslog"]` | Which logging drivers are available on the container instance. | `["json-file","none"]` | `["json-file","none"]` |
 | `ECS_DISABLE_PRIVILEGED` | `true` | Whether launching privileged containers is disabled on the container instance. | `false` | `false` |
-| `ECS_SELINUX_CAPABLE` | `true` | Whether SELinux is available on the container instance. | `false` | `false` |
+| `ECS_SELINUX_CAPABLE` | `true` | Whether SELinux is available on the container instance. (Limited support; Z-mode mounts only.) | `false` | `false` |
 | `ECS_APPARMOR_CAPABLE` | `true` | Whether AppArmor is available on the container instance. | `false` | `false` |
-| `ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION` | 10m | Default time to wait to delete containers for a stopped task (see also `ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION_JITTER`). If set to less than 1 minute, the value is ignored.  | 3h | 3h |
+| `ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION` | 10m | Default time to wait to delete containers for a stopped task (see also `ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION_JITTER`). If set to less than 1 second, the value is ignored.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | 3h | 3h |
 | `ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION_JITTER` | 1h | Jitter value for the task engine cleanup wait duration. When specified, the actual cleanup wait duration time for each task will be the duration specified in `ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION` plus a random duration between 0 and the jitter duration. | blank | blank |
 | `ECS_CONTAINER_STOP_TIMEOUT` | 10m | Instance scoped configuration for time to wait for the container to exit normally before being forcibly killed. | 30s | 30s |
 | `ECS_CONTAINER_START_TIMEOUT` | 10m | Timeout before giving up on starting a container. | 3m | 8m |
@@ -204,7 +204,8 @@ additional details on each available environment variable.
 | `ECS_ENABLE_GPU_SUPPORT` | `true` | Whether you use container instances with GPU support. This parameter is specified for the agent. You must also configure your task definitions for GPU. For more information | `false` | `Not applicable` |
 | `HTTP_PROXY` | `10.0.0.131:3128` | The hostname (or IP address) and port number of an HTTP proxy to use for the Amazon ECS agent to connect to the internet. For example, this proxy will be used if your container instances do not have external network access through an Amazon VPC internet gateway or NAT gateway or instance. If this variable is set, you must also set the NO_PROXY variable to filter Amazon EC2 instance metadata and Docker daemon traffic from the proxy. | `null` | `null` |
 | `NO_PROXY` | <For Linux: 169.254.169.254,169.254.170.2,/var/run/docker.sock &#124; For Windows: 169.254.169.254,169.254.170.2,\\.\pipe\docker_engine> | The HTTP traffic that should not be forwarded to the specified HTTP_PROXY. You must specify 169.254.169.254,/var/run/docker.sock to filter Amazon EC2 instance metadata and Docker daemon traffic from the proxy. | `null` | `null` |
-
+| `CREDENTIALS_FETCHER_HOST`   | `unix:///var/credentials-fetcher/socket/credentials_fetcher.sock` | Used to create a connection to the [credentials-fetcher daemon](https://github.com/aws/credentials-fetcher); to support gMSA on Linux. The default is fine for most users, only needs to be modified if user is configuring a custom credentials-fetcher socket path, ie, [CF_UNIX_DOMAIN_SOCKET_DIR](https://github.com/aws/credentials-fetcher#default-environment-variables). | `unix:///var/credentials-fetcher/socket/credentials_fetcher.sock` | Not Applicable |
+| `CREDENTIALS_FETCHER_SECRET_NAME_FOR_DOMAINLESS_GMSA`   | `secretmanager-secretname` | Used to support scaling option for gMSA on Linux [credentials-fetcher daemon](https://github.com/aws/credentials-fetcher). If user is configuring gMSA on a non-domain joined instance, they need to create an Active Directory user with access to retrieve principals for the gMSA account and store it in secrets manager | `secretmanager-secretname` | Not Applicable |
 ### Persistence
 
 When you run the Amazon ECS Container Agent in production, its `datadir` should be persisted between runs of the Docker

VERSION

@@ -1 +1 @@
-1.65.1
+1.67.2

agent/acs/model/api/api-2.json

@@ -618,6 +618,7 @@
       "type":"structure",
       "members":{
         "containerPort":{"shape":"Integer"},
+        "containerPortRange":{"shape":"String"},
         "hostPort":{"shape":"Integer"},
         "protocol":{"shape":"TransportProtocol"}
       }

agent/acs/model/ecsacs/api.go

@@ -15,8 +15,10 @@ package container
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -317,6 +319,13 @@ type Container struct {
 	finishedAt time.Time
 
 	labels map[string]string
+
+	// ContainerHasPortRange is set to true when the container has at least 1 port range requested.
+	ContainerHasPortRange bool
+	// ContainerPortSet is a set of singular container ports that don't belong to a containerPortRange request
+	ContainerPortSet map[int]struct{}
+	// ContainerPortRangeMap is a map of containerPortRange to its associated hostPortRange
+	ContainerPortRangeMap map[string]string
 }
 
 type DependsOn struct {
@@ -1325,6 +1334,44 @@ func (c *Container) UpdateManagedAgentSentStatus(agentName string, status apicon
 	return false
 }
 
+// RequiresCredentialSpec checks if container needs a credentialspec resource
+func (c *Container) RequiresCredentialSpec() bool {
+	credSpec, err := c.getCredentialSpec()
+	if err != nil || credSpec == "" {
+		return false
+	}
+
+	return true
+}
+
+// GetCredentialSpec is used to retrieve the current credentialspec resource
+func (c *Container) GetCredentialSpec() (string, error) {
+	return c.getCredentialSpec()
+}
+
+func (c *Container) getCredentialSpec() (string, error) {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	if c.DockerConfig.HostConfig == nil {
+		return "", errors.New("empty container hostConfig")
+	}
+
+	hostConfig := &dockercontainer.HostConfig{}
+	err := json.Unmarshal([]byte(*c.DockerConfig.HostConfig), hostConfig)
+	if err != nil || len(hostConfig.SecurityOpt) == 0 {
+		return "", errors.New("unable to obtain security options from container hostConfig")
+	}
+
+	for _, opt := range hostConfig.SecurityOpt {
+		if strings.HasPrefix(opt, "credentialspec") {
+			return opt, nil
+		}
+	}
+
+	return "", errors.New("unable to obtain credentialspec")
+}
+
 func (c *Container) GetManagedAgentStatus(agentName string) apicontainerstatus.ManagedAgentStatus {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
@@ -1360,3 +1407,39 @@ func (c *Container) IsContainerTornDown() bool {
 	defer c.lock.RUnlock()
 	return c.ContainerTornDownUnsafe
 }
+
+func (c *Container) SetContainerHasPortRange(containerHasPortRange bool) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.ContainerHasPortRange = containerHasPortRange
+}
+
+func (c *Container) HasPortRange() bool {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	return c.ContainerHasPortRange
+}
+
+func (c *Container) SetContainerPortSet(containerPortSet map[int]struct{}) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.ContainerPortSet = containerPortSet
+}
+
+func (c *Container) GetContainerPortSet() map[int]struct{} {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	return c.ContainerPortSet
+}
+
+func (c *Container) SetContainerPortRangeMap(portRangeMap map[string]string) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.ContainerPortRangeMap = portRangeMap
+}
+
+func (c *Container) GetContainerPortRangeMap() map[string]string {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	return c.ContainerPortRangeMap
+}

agent/api/container/container.go

@@ -970,3 +970,113 @@ func TestUpdateManagedAgentSentStatus(t *testing.T) {
 		})
 	}
 }
+
+func TestRequiresCredentialSpec(t *testing.T) {
+	testCases := []struct {
+		name           string
+		container      *Container
+		expectedOutput bool
+	}{
+		{
+			name:           "hostconfig_nil",
+			container:      &Container{},
+			expectedOutput: false,
+		},
+		{
+			name:           "invalid_case",
+			container:      getContainer("invalid"),
+			expectedOutput: false,
+		},
+		{
+			name:           "empty_sec_opt",
+			container:      getContainer("{\"NetworkMode\":\"bridge\"}"),
+			expectedOutput: false,
+		},
+		{
+			name:           "missing_credentialspec",
+			container:      getContainer("{\"SecurityOpt\": [\"invalid-sec-opt\"]}"),
+			expectedOutput: false,
+		},
+		{
+			name:           "valid_credentialspec_file",
+			container:      getContainer("{\"SecurityOpt\": [\"credentialspec:file://gmsa_gmsa-acct.json\"]}"),
+			expectedOutput: true,
+		},
+		{
+			name:           "valid_credentialspec_s3",
+			container:      getContainer("{\"SecurityOpt\": [\"credentialspec:arn:aws:s3:::${BucketName}/${ObjectName}\"]}"),
+			expectedOutput: true,
+		},
+		{
+			name:           "valid_credentialspec_ssm",
+			container:      getContainer("{\"SecurityOpt\": [\"credentialspec:arn:aws:ssm:region:aws_account_id:parameter/parameter_name\"]}"),
+			expectedOutput: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expectedOutput, tc.container.RequiresCredentialSpec())
+		})
+	}
+}
+
+func TestGetCredentialSpecErr(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		container            *Container
+		expectedOutputString string
+		expectedErrorString  string
+	}{
+		{
+			name:                 "hostconfig_nil",
+			container:            &Container{},
+			expectedOutputString: "",
+			expectedErrorString:  "empty container hostConfig",
+		},
+		{
+			name:                 "invalid_case",
+			container:            getContainer("invalid"),
+			expectedOutputString: "",
+			expectedErrorString:  "unable to obtain security options from container hostConfig",
+		},
+		{
+			name:                 "empty_sec_opt",
+			container:            getContainer("{\"NetworkMode\":\"bridge\"}"),
+			expectedOutputString: "",
+			expectedErrorString:  "unable to obtain security options from container hostConfig",
+		},
+		{
+			name:                 "missing_credentialspec",
+			container:            getContainer("{\"SecurityOpt\": [\"invalid-sec-opt\"]}"),
+			expectedOutputString: "",
+			expectedErrorString:  "unable to obtain credentialspec",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			expectedOutputStr, err := tc.container.GetCredentialSpec()
+			assert.Equal(t, tc.expectedOutputString, expectedOutputStr)
+			assert.EqualError(t, err, tc.expectedErrorString)
+		})
+	}
+}
+
+func TestGetCredentialSpecHappyPath(t *testing.T) {
+	c := getContainer("{\"SecurityOpt\": [\"credentialspec:file://gmsa_gmsa-acct.json\"]}")
+
+	expectedCredentialSpec := "credentialspec:file://gmsa_gmsa-acct.json"
+
+	credentialspec, err := c.GetCredentialSpec()
+	assert.NoError(t, err)
+	assert.EqualValues(t, expectedCredentialSpec, credentialspec)
+}
+
+func getContainer(hostConfig string) *Container {
+	c := &Container{
+		Name: "c",
+	}
+	c.DockerConfig.HostConfig = &hostConfig
+	return c
+}

agent/api/container/container_test.go

@@ -16,22 +16,8 @@
 
 package container
 
-import (
-	"github.com/pkg/errors"
-)
-
 const (
 	// DockerContainerMinimumMemoryInBytes is the minimum amount of
 	// memory to be allocated to a docker container
 	DockerContainerMinimumMemoryInBytes = 4 * 1024 * 1024 // 4MB
 )
-
-// RequiresCredentialSpec checks if container needs a credentialspec resource
-func (c *Container) RequiresCredentialSpec() bool {
-	return false
-}
-
-// GetCredentialSpec is used to retrieve the current credentialspec resource
-func (c *Container) GetCredentialSpec() (string, error) {
-	return "", errors.New("unsupported platform")
-}