Added some cleanup for IPs and memory addresses in telemetry errors (#251)

* Added some cleanup for IPs and memory addresses in telemetry errors
* Locking version of ubuntu so java update doesn't fail tests

Author: MichaelGHSeg

.github/workflows/build.yml

@@ -10,15 +10,15 @@ on:
 jobs:
   cancel_previous:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: styfle/cancel-workflow-action@0.9.1
         with:
           workflow_id: ${{ github.event.workflow.id }}
 
   core-test:
     needs: cancel_previous
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v2
@@ -42,7 +42,7 @@ jobs:
 
   android-test:
     needs: cancel_previous
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v2
@@ -66,7 +66,7 @@ jobs:
 
   destination-test:
     needs: cancel_previous
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v2
@@ -90,7 +90,7 @@ jobs:
 
   security:
     needs: cancel_previous
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/create_jira.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   create_jira:
     name: Create Jira Ticket
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: IssueTracker
     steps:
       - name: Checkout

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: deployment
     steps:
       - uses: actions/checkout@v2

.github/workflows/snapshot.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   snapshot:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: deployment
     steps:
       - uses: actions/checkout@v2

core/src/main/java/com/segment/analytics/kotlin/core/Telemetry.kt

@@ -170,6 +170,22 @@ object Telemetry: Subscriber {
         addRemoteMetric(metric, tags)
     }
 
+    fun cleanErrorValue(value: String): String {
+        var cleanedValue = value
+        // Remove IPs
+        cleanedValue = cleanedValue.replace(Regex("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}[\\d._:port]*"), "_IP")
+        // Remove IPv6
+        cleanedValue = cleanedValue.replace(Regex("[0-9a-fA-F]{2,4}(:[0-9a-fA-F]{0,4}){2,8}[\\d._:port]*"), "_IP")
+        // Remove hex values
+        cleanedValue = cleanedValue.replace(Regex("0x[0-9a-fA-F]+"), "0x00")
+        // Remove hex values that don't have 0x of at least 6 characters
+        cleanedValue = cleanedValue.replace(Regex("[0-9a-fA-F]{6,}"), "0x00")
+        // What even?  Mangled library names probably, e.g. a5.b:_some_error_etc
+        cleanedValue = cleanedValue.replace(Regex("^[a-z][a-z0-9]\\.[a-z]:"), "")
+
+        return cleanedValue
+    }
+
     /**
      * Logs an error metric with the specified tags and log data.
      *
@@ -186,6 +202,10 @@ object Telemetry: Subscriber {
         if (tags.isEmpty()) return
         if (Math.random() > sampleRate) return
 
+        if (tags.containsKey("error")) {
+            tags["error"] = cleanErrorValue(tags["error"]!!)
+        }
+
         var filteredTags = if(sendWriteKeyOnError) {
             tags.toMap()
         } else {

core/src/test/kotlin/com/segment/analytics/kotlin/core/TelemetryTest.kt

@@ -29,6 +29,13 @@ class TelemetryTest {
         queueBytesField.isAccessible = true
         return queueBytesField.get(Telemetry) as Int
     }
+    fun TelemetryQueuePeek(): RemoteMetric {
+        val queueField: Field = Telemetry::class.java.getDeclaredField("queue")
+        queueField.isAccessible = true
+        val queueValue: ConcurrentLinkedQueue<*> = queueField.get(Telemetry) as ConcurrentLinkedQueue<*>
+        return queueValue.peek() as RemoteMetric
+    }
+
     var TelemetryStarted: Boolean
         get() {
             val startedField: Field = Telemetry::class.java.getDeclaredField("started")
@@ -239,4 +246,59 @@ class TelemetryTest {
         }
         assertTrue(TelemetryQueueSize() == Telemetry.maxQueueSize)
     }
-}
+
+    @Test
+    fun `Test error tags are cleaned`() {
+        Telemetry.enable = true
+        Telemetry.start()
+        Telemetry.error(Telemetry.INVOKE_ERROR_METRIC, "error log") {
+            it["error"] = "foo_192.168.0.1:8080"
+        }
+        assertEquals(1, TelemetryQueueSize())
+        assertEquals("foo__IP", TelemetryQueuePeek().tags["error"])
+    }
+
+    @Test
+    fun `Test error tags are cleaned for IPv6`() {
+        Telemetry.enable = true
+        Telemetry.start()
+        Telemetry.error(Telemetry.INVOKE_ERROR_METRIC, "error log") {
+            it["error"] = "foo_2001:0db8:85a3:0000:0000:8a2e:0370:7334"
+        }
+        assertEquals(1, TelemetryQueueSize())
+        assertEquals("foo__IP", TelemetryQueuePeek().tags["error"])
+    }
+
+    @Test
+    fun `Test error tags are cleaned for hex values`() {
+        Telemetry.enable = true
+        Telemetry.start()
+        Telemetry.error(Telemetry.INVOKE_ERROR_METRIC, "error log") {
+            it["error"] = "foo_0x1234567890abcdef_bar"
+        }
+        assertEquals(1, TelemetryQueueSize())
+        assertEquals("foo_0x00_bar", TelemetryQueuePeek().tags["error"])
+    }
+
+    @Test
+    fun `Test error tags are cleaned for sneaky hex values`() {
+        Telemetry.enable = true
+        Telemetry.start()
+        Telemetry.error(Telemetry.INVOKE_ERROR_METRIC, "error log") {
+            it["error"] = "address_deadbeef_face"
+        }
+        assertEquals(1, TelemetryQueueSize())
+        assertEquals("address_0x00_face", TelemetryQueuePeek().tags["error"])
+    }
+
+    @Test
+    fun `Test error tags are cleaned for mangled library names`() {
+        Telemetry.enable = true
+        Telemetry.start()
+        Telemetry.error(Telemetry.INVOKE_ERROR_METRIC, "error log") {
+            it["error"] = "a5.b:_some_error_etc"
+        }
+        assertEquals(1, TelemetryQueueSize())
+        assertEquals("_some_error_etc", TelemetryQueuePeek().tags["error"])
+    }
+}