From 89856f0d6fb09f80802cf444bb93f05666dfb355 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 10 Aug 2023 08:27:37 +0200 Subject: [PATCH 1/3] update-json-specs.yml: add container_metadata_discovery.json --- .ci/updatecli.d/update-json-specs.yml | 12 ++-- .../json-specs/cgroup_parsing.json | 22 ------- .../container_metadata_discovery.json | 65 +++++++++++++++++++ 3 files changed, 71 insertions(+), 28 deletions(-) delete mode 100644 test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json create mode 100644 test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/container_metadata_discovery.json diff --git a/.ci/updatecli.d/update-json-specs.yml b/.ci/updatecli.d/update-json-specs.yml index 9191ec461..2972b191e 100644 --- a/.ci/updatecli.d/update-json-specs.yml +++ b/.ci/updatecli.d/update-json-specs.yml @@ -24,10 +24,10 @@ sources: - findsubmatch: pattern: "[0-9a-f]{40}" - cgroup_parsing.json: + container_metadata_discovery.json: kind: file spec: - file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/cgroup_parsing.json + file: https://raw.githubusercontent.com/elastic/apm/main/tests/agents/json-specs/container_metadata_discovery.json service_resource_inference.json: kind: file spec: @@ -71,13 +71,13 @@ actions: * https://github.com/elastic/apm/commit/{{ source "sha" }} targets: - cgroup_parsing.json: - name: cgroup_parsing.json + container_metadata_discovery.json: + name: container_metadata_discovery.json scmid: default - sourceid: cgroup_parsing.json + sourceid: container_metadata_discovery.json kind: file spec: - file: test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json + file: test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/container_metadata_discovery.json service_resource_inference.json: name: service_resource_inference.json scmid: default diff --git a/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json b/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json deleted file mode 100644 index f28d87d4c..000000000 --- a/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/cgroup_parsing.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "testUnderscores": { - "groupLine": "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope", - "containerId": "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63", - "podId": "90d81341-92de-11e7-8cf2-507b9d4141fa" - }, - "testOpenshiftForm": { - "groupLine": "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope", - "containerId": "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f", - "podId": "22949dce-fd8b-11ea-8ede-98f2b32c645c" - }, - "testUbuntuCGroup": { - "groupLine": "1:name=systemd:/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope", - "containerId": null, - "podId": null - }, - "testAwsEcsCGroup": { - "groupLine": "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728", - "containerId": "03752a671e744971a862edcee6195646-4015103728", - "podId": null - } -} diff --git a/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/container_metadata_discovery.json b/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/container_metadata_discovery.json new file mode 100644 index 000000000..510fd19d2 --- /dev/null +++ b/test/Elastic.Apm.Tests.Utilities/TestResources/json-specs/container_metadata_discovery.json @@ -0,0 +1,65 @@ +{ + "cgroup_v1_underscores": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope" + ] + }, + "containerId": "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63", + "podId": "90d81341-92de-11e7-8cf2-507b9d4141fa" + }, + "cgroup_v1_openshift": { + "files": { + "/proc/self/cgroup": [ + "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope" + ] + }, + "containerId": "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f", + "podId": "22949dce-fd8b-11ea-8ede-98f2b32c645c" + }, + "cgroup_v1_ubuntu": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope" + ] + }, + "containerId": null, + "podId": null + }, + "cgroup_v1_awsEcs": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728" + ] + }, + "containerId": "03752a671e744971a862edcee6195646-4015103728", + "podId": null + }, + "cgroup_v2": { + "files": { + "/proc/self/cgroup": [ + "0::/" + ], + "/proc/self/mountinfo": [ + "3984 3905 0:73 / / rw,relatime shared:1863 master:1733 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/KEX7CWLHQCXQY2RHPGTXJ3C26N:/var/lib/docker/overlay2/l/2PVS7JRTRSTVZS4KSUAFML3BIV:/var/lib/docker/overlay2/l/52M7ARM4JDVHCJAYUI6JIKBO4B,upperdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/diff,workdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/work", + "3985 3984 0:77 / /proc rw,nosuid,nodev,noexec,relatime shared:1864 - proc proc rw", + "3986 3984 0:78 / /dev rw,nosuid shared:1865 - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "3987 3986 0:79 / /dev/pts rw,nosuid,noexec,relatime shared:1866 - devpts devpts rw,gid=5,mode=620,ptmxmode=666", + "3988 3984 0:80 / /sys ro,nosuid,nodev,noexec,relatime shared:1870 - sysfs sysfs ro", + "3989 3988 0:30 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:1871 - cgroup2 cgroup rw", + "3990 3986 0:76 / /dev/mqueue rw,nosuid,nodev,noexec,relatime shared:1867 - mqueue mqueue rw", + "3991 3986 0:81 / /dev/shm rw,nosuid,nodev,noexec,relatime shared:1868 - tmpfs shm rw,size=65536k,inode64", + "3992 3984 253:1 /var/lib/docker/volumes/9d18ce5b36572d85358fa936afe5a4bf95cca5c822b04941aa08c6118f6e0d33/_data /var rw,relatime shared:1872 master:1 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3993 3984 0:82 / /run rw,nosuid,nodev,noexec,relatime shared:1873 - tmpfs tmpfs rw,inode64", + "3994 3984 0:83 / /tmp rw,nosuid,nodev,noexec,relatime shared:1874 - tmpfs tmpfs rw,inode64", + "3995 3984 253:1 /usr/lib/modules /usr/lib/modules ro,relatime shared:1875 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3996 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/resolv.conf /etc/resolv.conf rw,relatime shared:1876 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3998 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hosts /etc/hosts rw,relatime shared:1878 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro" + ] + }, + "containerId": "6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6", + "podId": null + } +} + From a36f8682fc54a7157ef3f3f8ff57cbb14d6ac6d7 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 5 Jun 2024 14:15:04 +0200 Subject: [PATCH 2/3] Update test cases to cover new format and fallbacks to mountinfo --- src/Elastic.Apm/Helpers/SystemInfoHelper.cs | 61 +++++++++++++-- .../CGroupTestCasesAttribute.cs | 78 +++++++++++++++++++ .../SystemInfoHelperTests.cs | 25 +++--- 3 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs diff --git a/src/Elastic.Apm/Helpers/SystemInfoHelper.cs b/src/Elastic.Apm/Helpers/SystemInfoHelper.cs index b5647da9b..06f568bf5 100644 --- a/src/Elastic.Apm/Helpers/SystemInfoHelper.cs +++ b/src/Elastic.Apm/Helpers/SystemInfoHelper.cs @@ -5,6 +5,7 @@ using System; using System.Data.Common; using System.IO; +using System.Linq; using System.Net; using System.Net.NetworkInformation; using System.Text.RegularExpressions; @@ -28,6 +29,27 @@ internal class SystemInfoHelper public SystemInfoHelper(IApmLogger logger) => _logger = logger.Scoped(nameof(SystemInfoHelper)); + + //3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro + internal void ParseMountInfo(Api.System system, string reportedHostName, string line) + { + + var fields = line.Split(' '); + if (fields.Length <= 3) + return; + + var path = fields[3]; + foreach (var folder in path.Split('/')) + { + //naive implementation to check for guid. + if (folder.Length != 64) continue; + system.Container = new Container { Id = folder }; + } + + } + + // "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728" + // "0::/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod121157b5_c67d_4c3e_9052_cb27bbb711fb.slice/cri-containerd-1cd3449e930b8a28c7595240fa32ba20c84f36d059e5fbe63104ad40057992d1.scope" internal void ParseContainerId(Api.System system, string reportedHostName, string line) { var fields = line.Split(':'); @@ -43,11 +65,13 @@ internal void ParseContainerId(Api.System system, string reportedHostName, strin if (string.IsNullOrWhiteSpace(idPart)) return; - // Legacy, e.g.: /system.slice/docker-.scope + // Legacy, e.g.: /system.slice/docker-.scope or cri-containerd-.scope if (idPart.EndsWith(".scope")) { - idPart = idPart.Substring(0, idPart.Length - ".scope".Length) - .Substring(idPart.IndexOf("-", StringComparison.Ordinal) + 1); + var idParts = idPart.Split(new[] { '-'}, StringSplitOptions.RemoveEmptyEntries); + var containerIdWithScope = idParts.Last(); + + idPart = containerIdWithScope.Substring(0, containerIdWithScope.Length - ".scope".Length); } // Looking for kubernetes info @@ -173,8 +197,10 @@ static string NormalizeHostName(string hostName) => private void ParseContainerInfo(Api.System system, string reportedHostName) { + //0::/ try { + var fallBackToMountInfo = false; using var sr = GetCGroupAsStream(); if (sr is null) { @@ -183,12 +209,33 @@ private void ParseContainerInfo(Api.System system, string reportedHostName) return; } + var i = 0; string line; while ((line = sr.ReadLine()) != null) { + if (line == "0::/" && i == 0) + fallBackToMountInfo = true; ParseContainerId(system, reportedHostName, line); if (system.Container != null) return; + i++; + } + if (!fallBackToMountInfo) + return; + + using var mi = GetMountInfoAsStream(); + if (mi is null) + { + _logger.Debug()?.Log("No /proc/self/mountinfo found - no information to fallback to"); + return; + } + + while ((line = mi.ReadLine()) != null) + { + if (!line.Contains("/etc/hostname")) continue; + ParseMountInfo(system, reportedHostName, line); + if (system.Container != null) + return; } } catch (Exception e) @@ -201,8 +248,12 @@ private void ParseContainerInfo(Api.System system, string reportedHostName) "Failed parsing container id - the agent will not report container id. Likely the application is not running within a container"); } - protected virtual StreamReader GetCGroupAsStream() - => File.Exists("/proc/self/cgroup") ? new StreamReader("/proc/self/cgroup") : null; + protected virtual StreamReader GetCGroupAsStream() => + File.Exists("/proc/self/cgroup") ? new StreamReader("/proc/self/cgroup") : null; + + protected virtual StreamReader GetMountInfoAsStream() => + File.Exists("/proc/self/mountinfo") ? new StreamReader("/proc/self/mountinfo") : null; + internal const string Namespace = "KUBERNETES_NAMESPACE"; internal const string PodName = "KUBERNETES_POD_NAME"; diff --git a/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs b/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs new file mode 100644 index 000000000..2bf0f8a2f --- /dev/null +++ b/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Elastic.Apm.Libraries.Newtonsoft.Json.Linq; +using Xunit.Sdk; + +namespace Elastic.Apm.Tests.Utilities; + +public struct CgroupFiles +{ + public string ProcSelfCgroup; + public string[] MountInfo; +} + +public struct CGroupTestData +{ + public CgroupFiles Files; + public string ContainerId; + public string PodId; +} + + +public class CGroupTestCasesAttribute : DataAttribute +{ + private readonly string _fileName = "./TestResources/json-specs//container_metadata_discovery.json"; + + public override IEnumerable GetData(MethodInfo testMethod) + { + if (!File.Exists(_fileName)) + throw new ArgumentException($"JSON input file {_fileName} does not exist"); + + var jToken = JToken.Parse(File.ReadAllText(_fileName), new JsonLoadSettings + { + CommentHandling = CommentHandling.Ignore + }); + + foreach (var kvp in (JObject)jToken) + { + var name = kvp.Key; + var data = ParseTestData(kvp.Value as JObject); + yield return [name, data]; + } + } + + private static CGroupTestData ParseTestData(JObject jToken) + { + var testData = new CGroupTestData { Files = new CgroupFiles() }; + + foreach (var kvp in jToken) + { + switch (kvp.Key) + { + case "containerId": + testData.ContainerId = kvp.Value?.Value(); + break; + case "podId": + testData.PodId = kvp.Value?.Value(); + break; + case "files": + var o = (JObject)kvp.Value; + var cgroupA = o.Property("/proc/self/cgroup")?.Value as JArray; + testData.Files.ProcSelfCgroup = cgroupA?.Values().FirstOrDefault(); + + var mountInfoA = o.Property("/proc/self/mountinfo")?.Value as JArray; + testData.Files.MountInfo = mountInfoA?.Values().ToArray(); + break; + } + } + return testData; + } +} diff --git a/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs b/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs index 204edfdb2..a20f76d84 100644 --- a/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs +++ b/test/Elastic.Apm.Tests/SystemInfoHelperTests.cs @@ -4,12 +4,14 @@ using System; using System.Collections.Generic; +using System.Linq; using Elastic.Apm.Api.Kubernetes; using Elastic.Apm.Features; using Elastic.Apm.Helpers; using Elastic.Apm.Logging; using Elastic.Apm.Tests.Utilities; using FluentAssertions; +using Newtonsoft.Json; using Xunit; namespace Elastic.Apm.Tests; @@ -61,30 +63,33 @@ public void ParseKubernetesInfo_ShouldReturnNull_WhenNoEnvironmentVariablesAreSe system.Kubernetes.Should().BeNull(); } - public struct CGroupTestData - { - public string GroupLine; - public string ContainerId; - public string PodId; - } - // Remove warning about unused test parameter "name" #pragma warning disable xUnit1026 [Theory] - [JsonFileData("./TestResources/json-specs/cgroup_parsing.json", typeof(CGroupTestData))] + [CGroupTestCases] public void ParseKubernetesInfo_FromCGroupLine(string name, CGroupTestData data) { - var line = data.GroupLine; + data.Files.ProcSelfCgroup.Should().NotBeNull(); + var line = data.Files.ProcSelfCgroup; var containerId = data.ContainerId; var podId = data.PodId; var system = new Api.System(); - _systemInfoHelper.ParseContainerId(system, "hostname", line); + if (line == "0::/") + { + line = data.Files.MountInfo.FirstOrDefault(l => l.Contains("/etc/hostname")); + _systemInfoHelper.ParseMountInfo(system, "hostname", line); + } + else + _systemInfoHelper.ParseContainerId(system, "hostname", line); if (containerId is null) system.Container.Should().BeNull(); else + { + system.Container.Should().NotBeNull("{0}", line); system.Container.Id.Should().Be(containerId); + } if (podId is null) system.Kubernetes.Should().BeNull(); From efe9a78dd56e404bc656aaf8d273d41dbbad2092 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Wed, 5 Jun 2024 14:17:57 +0200 Subject: [PATCH 3/3] formatting --- src/Elastic.Apm/Helpers/SystemInfoHelper.cs | 10 ++++++---- .../CGroupTestCasesAttribute.cs | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Elastic.Apm/Helpers/SystemInfoHelper.cs b/src/Elastic.Apm/Helpers/SystemInfoHelper.cs index 06f568bf5..6c1371aa1 100644 --- a/src/Elastic.Apm/Helpers/SystemInfoHelper.cs +++ b/src/Elastic.Apm/Helpers/SystemInfoHelper.cs @@ -30,7 +30,7 @@ public SystemInfoHelper(IApmLogger logger) => _logger = logger.Scoped(nameof(SystemInfoHelper)); - //3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro + //3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro internal void ParseMountInfo(Api.System system, string reportedHostName, string line) { @@ -42,7 +42,8 @@ internal void ParseMountInfo(Api.System system, string reportedHostName, string foreach (var folder in path.Split('/')) { //naive implementation to check for guid. - if (folder.Length != 64) continue; + if (folder.Length != 64) + continue; system.Container = new Container { Id = folder }; } @@ -68,7 +69,7 @@ internal void ParseContainerId(Api.System system, string reportedHostName, strin // Legacy, e.g.: /system.slice/docker-.scope or cri-containerd-.scope if (idPart.EndsWith(".scope")) { - var idParts = idPart.Split(new[] { '-'}, StringSplitOptions.RemoveEmptyEntries); + var idParts = idPart.Split(new[] { '-' }, StringSplitOptions.RemoveEmptyEntries); var containerIdWithScope = idParts.Last(); idPart = containerIdWithScope.Substring(0, containerIdWithScope.Length - ".scope".Length); @@ -232,7 +233,8 @@ private void ParseContainerInfo(Api.System system, string reportedHostName) while ((line = mi.ReadLine()) != null) { - if (!line.Contains("/etc/hostname")) continue; + if (!line.Contains("/etc/hostname")) + continue; ParseMountInfo(system, reportedHostName, line); if (system.Container != null) return; diff --git a/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs b/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs index 2bf0f8a2f..e73eaa5f7 100644 --- a/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs +++ b/test/Elastic.Apm.Tests.Utilities/CGroupTestCasesAttribute.cs @@ -15,8 +15,8 @@ namespace Elastic.Apm.Tests.Utilities; public struct CgroupFiles { - public string ProcSelfCgroup; - public string[] MountInfo; + public string ProcSelfCgroup; + public string[] MountInfo; } public struct CGroupTestData