From 3437ae086d803bcc04a35c3f36595a3c9c0f33ae Mon Sep 17 00:00:00 2001 From: James Forward Date: Tue, 10 Sep 2024 20:10:07 +0100 Subject: [PATCH] feat: JSON Masking --- .../java/org/akhq/configs/DataMasking.java | 6 +- .../org/akhq/configs/DataMaskingMode.java | 12 ++++ .../org/akhq/configs/JsonMaskingFilter.java | 14 ++++ ...ataMaskingFilter.java => RegexFilter.java} | 2 +- src/main/java/org/akhq/models/Record.java | 4 ++ .../akhq/repositories/RecordRepository.java | 8 +-- .../java/org/akhq/utils/AvroSerializer.java | 19 ++++- .../akhq/utils/JsonMaskByDefaultMasker.java | 65 +++++++++++++++++ .../akhq/utils/JsonShowByDefaultMasker.java | 64 +++++++++++++++++ src/main/java/org/akhq/utils/Masker.java | 12 ++++ .../java/org/akhq/utils/MaskerFactory.java | 22 ++++++ src/main/java/org/akhq/utils/NoOpMasker.java | 11 +++ .../{MaskingUtils.java => RegexMasker.java} | 25 +++---- .../akhq/utils/DefaultMaskerSettingTest.java | 29 ++++++++ .../utils/JsonMaskByDefaultMaskerTest.java | 71 +++++++++++++++++++ .../utils/JsonShowByDefaultMaskerTest.java | 71 +++++++++++++++++++ .../java/org/akhq/utils/MaskerTestHelper.java | 28 ++++++++ .../java/org/akhq/utils/MaskingUtilsTest.java | 64 ----------------- .../java/org/akhq/utils/NoOpMaskerTest.java | 47 ++++++++++++ .../java/org/akhq/utils/RegexMaskerTest.java | 67 +++++++++++++++++ ...tion-json-mask-by-default-data-masking.yml | 12 ++++ ...tion-json-show-by-default-data-masking.yml | 12 ++++ .../application-no-op-data-masking.yml | 4 ++ ...yml => application-regex-data-masking.yml} | 1 + 24 files changed, 582 insertions(+), 88 deletions(-) create mode 100644 src/main/java/org/akhq/configs/DataMaskingMode.java create mode 100644 src/main/java/org/akhq/configs/JsonMaskingFilter.java rename src/main/java/org/akhq/configs/{DataMaskingFilter.java => RegexFilter.java} (86%) create mode 100644 src/main/java/org/akhq/utils/JsonMaskByDefaultMasker.java create mode 100644 src/main/java/org/akhq/utils/JsonShowByDefaultMasker.java create mode 100644 src/main/java/org/akhq/utils/Masker.java create mode 100644 src/main/java/org/akhq/utils/MaskerFactory.java create mode 100644 src/main/java/org/akhq/utils/NoOpMasker.java rename src/main/java/org/akhq/utils/{MaskingUtils.java => RegexMasker.java} (54%) create mode 100644 src/test/java/org/akhq/utils/DefaultMaskerSettingTest.java create mode 100644 src/test/java/org/akhq/utils/JsonMaskByDefaultMaskerTest.java create mode 100644 src/test/java/org/akhq/utils/JsonShowByDefaultMaskerTest.java create mode 100644 src/test/java/org/akhq/utils/MaskerTestHelper.java delete mode 100644 src/test/java/org/akhq/utils/MaskingUtilsTest.java create mode 100644 src/test/java/org/akhq/utils/NoOpMaskerTest.java create mode 100644 src/test/java/org/akhq/utils/RegexMaskerTest.java create mode 100644 src/test/resources/application-json-mask-by-default-data-masking.yml create mode 100644 src/test/resources/application-json-show-by-default-data-masking.yml create mode 100644 src/test/resources/application-no-op-data-masking.yml rename src/test/resources/{application-data-masking.yml => application-regex-data-masking.yml} (95%) diff --git a/src/main/java/org/akhq/configs/DataMasking.java b/src/main/java/org/akhq/configs/DataMasking.java index 6fb838e4f..b2ba6a694 100644 --- a/src/main/java/org/akhq/configs/DataMasking.java +++ b/src/main/java/org/akhq/configs/DataMasking.java @@ -9,5 +9,9 @@ @ConfigurationProperties("akhq.security.data-masking") @Data public class DataMasking { - List filters = new ArrayList<>(); + List filters = new ArrayList<>(); + DataMaskingMode mode = DataMaskingMode.REGEX; // set this by default to REGEX for backwards compatibility for current users who haven't defined this property. + List jsonFilters = new ArrayList<>(); + String jsonMaskReplacement = "xxxx"; + boolean cachingEnabled = false; } diff --git a/src/main/java/org/akhq/configs/DataMaskingMode.java b/src/main/java/org/akhq/configs/DataMaskingMode.java new file mode 100644 index 000000000..26e6260f2 --- /dev/null +++ b/src/main/java/org/akhq/configs/DataMaskingMode.java @@ -0,0 +1,12 @@ +package org.akhq.configs; + +public enum DataMaskingMode { + // Use the existing regex-based filtering + REGEX, + // Use filtering where by default all fields of all records are masked, with fields to unmask defined in allowlists + JSON_MASK_BY_DEFAULT, + // Use filtering where by default no fields of any records are masked, with fields to mask explicitly denied + JSON_SHOW_BY_DEFAULT, + // No masker at all, best performance + NONE +} diff --git a/src/main/java/org/akhq/configs/JsonMaskingFilter.java b/src/main/java/org/akhq/configs/JsonMaskingFilter.java new file mode 100644 index 000000000..e5ad0b738 --- /dev/null +++ b/src/main/java/org/akhq/configs/JsonMaskingFilter.java @@ -0,0 +1,14 @@ +package org.akhq.configs; + +import io.micronaut.context.annotation.EachProperty; +import lombok.Data; + +import java.util.List; + +@EachProperty("jsonfilters") +@Data +public class JsonMaskingFilter { + String description = "UNKNOWN"; + String topic = "UNKNOWN"; + List keys = List.of("UNKNOWN"); +} diff --git a/src/main/java/org/akhq/configs/DataMaskingFilter.java b/src/main/java/org/akhq/configs/RegexFilter.java similarity index 86% rename from src/main/java/org/akhq/configs/DataMaskingFilter.java rename to src/main/java/org/akhq/configs/RegexFilter.java index 63075d541..aec809b52 100644 --- a/src/main/java/org/akhq/configs/DataMaskingFilter.java +++ b/src/main/java/org/akhq/configs/RegexFilter.java @@ -5,7 +5,7 @@ @EachProperty("filters") @Data -public class DataMaskingFilter { +public class RegexFilter { String description; String searchRegex; String replacement; diff --git a/src/main/java/org/akhq/models/Record.java b/src/main/java/org/akhq/models/Record.java index 8c09293fa..9f94cf575 100644 --- a/src/main/java/org/akhq/models/Record.java +++ b/src/main/java/org/akhq/models/Record.java @@ -189,6 +189,10 @@ public void setTruncated(Boolean truncated) { this.truncated = truncated; } + public void setTopic(Topic topic) { + this.topic = topic; + } + private String convertToString(byte[] payload, String schemaId, boolean isKey) { if (payload == null) { return null; diff --git a/src/main/java/org/akhq/repositories/RecordRepository.java b/src/main/java/org/akhq/repositories/RecordRepository.java index 83714fc08..cdc4f1d3a 100644 --- a/src/main/java/org/akhq/repositories/RecordRepository.java +++ b/src/main/java/org/akhq/repositories/RecordRepository.java @@ -25,7 +25,7 @@ import org.akhq.modules.schemaregistry.RecordWithSchemaSerializerFactory; import org.akhq.utils.AvroToJsonSerializer; import org.akhq.utils.Debug; -import org.akhq.utils.MaskingUtils; +import org.akhq.utils.Masker; import org.apache.kafka.clients.admin.DeletedRecords; import org.apache.kafka.clients.admin.RecordsToDelete; import org.apache.kafka.clients.consumer.*; @@ -79,7 +79,7 @@ public class RecordRepository extends AbstractRepository { private AvroWireFormatConverter avroWireFormatConverter; @Inject - private MaskingUtils maskingUtils; + private Masker masker; @Value("${akhq.topic-data.poll-timeout:10000}") protected int pollTimeout; @@ -453,7 +453,7 @@ private ConsumerRecords poll(KafkaConsumer consu private Record newRecord(ConsumerRecord record, String clusterId, Topic topic) { SchemaRegistryType schemaRegistryType = this.schemaRegistryRepository.getSchemaRegistryType(clusterId); SchemaRegistryClient client = this.kafkaModule.getRegistryClient(clusterId); - return maskingUtils.maskRecord(new Record( + return masker.maskRecord(new Record( client, record, this.schemaRegistryRepository.getSchemaRegistryType(clusterId), @@ -473,7 +473,7 @@ private Record newRecord(ConsumerRecord record, String clusterId private Record newRecord(ConsumerRecord record, BaseOptions options, Topic topic) { SchemaRegistryType schemaRegistryType = this.schemaRegistryRepository.getSchemaRegistryType(options.clusterId); SchemaRegistryClient client = this.kafkaModule.getRegistryClient(options.clusterId); - return maskingUtils.maskRecord(new Record( + return masker.maskRecord(new Record( client, record, schemaRegistryType, diff --git a/src/main/java/org/akhq/utils/AvroSerializer.java b/src/main/java/org/akhq/utils/AvroSerializer.java index 9304b2295..61003c470 100644 --- a/src/main/java/org/akhq/utils/AvroSerializer.java +++ b/src/main/java/org/akhq/utils/AvroSerializer.java @@ -20,6 +20,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; import java.util.*; import java.util.stream.Collectors; @@ -45,7 +46,19 @@ public class AvroSerializer { private static final TimeConversions.LocalTimestampMillisConversion LOCAL_TIMESTAMP_MILLIS_CONVERSION = new TimeConversions.LocalTimestampMillisConversion(); protected static final String DATE_FORMAT = "yyyy-MM-dd[XXX]"; - protected static final String TIME_FORMAT = "HH:mm[:ss][.SSSSSS][XXX]"; + protected static final DateTimeFormatter TIME_FORMATTER = new DateTimeFormatterBuilder() + .appendPattern("HH:mm") + .optionalStart() + .appendPattern(":ss") + .optionalEnd() + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true) + .optionalEnd() + .optionalStart() + .appendPattern("XXX") + .optionalEnd() + .toFormatter(); + protected static final DateTimeFormatter DATETIME_FORMAT = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ISO_LOCAL_DATE_TIME) .optionalStart() @@ -323,7 +336,7 @@ protected static Instant parseDateTime(String data) { private static Long timeMicrosSerializer(Object data, Schema schema, Schema.Type primitiveType, LogicalType logicalType) { LocalTime value; if (data instanceof String) { - value = LocalTime.parse((String) data, DateTimeFormatter.ofPattern(AvroSerializer.TIME_FORMAT)); + value = LocalTime.parse((String) data, TIME_FORMATTER); } else { value = (LocalTime) data; } @@ -339,7 +352,7 @@ private static Integer timeMillisSerializer(Object data, Schema schema, Schema.T LocalTime value; if (data instanceof String) { - value = LocalTime.parse((String) data, DateTimeFormatter.ofPattern(AvroSerializer.TIME_FORMAT)); + value = LocalTime.parse((String) data, TIME_FORMATTER); } else { value = (LocalTime) data; } diff --git a/src/main/java/org/akhq/utils/JsonMaskByDefaultMasker.java b/src/main/java/org/akhq/utils/JsonMaskByDefaultMasker.java new file mode 100644 index 000000000..a9af6f5c5 --- /dev/null +++ b/src/main/java/org/akhq/utils/JsonMaskByDefaultMasker.java @@ -0,0 +1,65 @@ +package org.akhq.utils; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.akhq.configs.JsonMaskingFilter; +import org.akhq.models.Record; + +import java.util.List; +import java.util.Map; + +@RequiredArgsConstructor +public class JsonMaskByDefaultMasker implements Masker { + + private final List jsonMaskingFilters; + private final String jsonMaskReplacement; + + public Record maskRecord(Record record) { + try { + if(record.getValue().trim().startsWith("{") && record.getValue().trim().endsWith("}")) { + JsonMaskingFilter foundFilter = null; + for (JsonMaskingFilter filter : jsonMaskingFilters) { + if (record.getTopic().getName().equalsIgnoreCase(filter.getTopic())) { + foundFilter = filter; + } + } + if (foundFilter != null) { + return applyMasking(record, foundFilter.getKeys()); + } else { + return applyMasking(record, List.of()); + } + } else { + return record; + } + } catch (Exception e) { + LOG.error("Error masking record", e); + return record; + } + } + + @SneakyThrows + private Record applyMasking(Record record, List keys) { + JsonObject jsonElement = JsonParser.parseString(record.getValue()).getAsJsonObject(); + maskAllExcept(jsonElement, keys); + record.setValue(jsonElement.toString()); + return record; + } + + private void maskAllExcept(JsonObject node, List keys) { + if (node.isJsonObject()) { + JsonObject objectNode = node.getAsJsonObject(); + for(Map.Entry entry : objectNode.entrySet()) { + if(entry.getValue().isJsonObject()) { + maskAllExcept(entry.getValue().getAsJsonObject(), keys); + } else { + if(!keys.contains(entry.getKey())) { + objectNode.addProperty(entry.getKey(), jsonMaskReplacement); + } + } + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/akhq/utils/JsonShowByDefaultMasker.java b/src/main/java/org/akhq/utils/JsonShowByDefaultMasker.java new file mode 100644 index 000000000..1a997837a --- /dev/null +++ b/src/main/java/org/akhq/utils/JsonShowByDefaultMasker.java @@ -0,0 +1,64 @@ +package org.akhq.utils; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.akhq.configs.JsonMaskingFilter; +import org.akhq.models.Record; + +import java.util.List; + +@RequiredArgsConstructor +public class JsonShowByDefaultMasker implements Masker { + + private final List jsonMaskingFilters; + private final String jsonMaskReplacement; + + public Record maskRecord(Record record) { + try { + if(record.getValue().trim().startsWith("{") && record.getValue().trim().endsWith("}")) { + JsonMaskingFilter foundFilter = null; + for (JsonMaskingFilter filter : jsonMaskingFilters) { + if (record.getTopic().getName().equalsIgnoreCase(filter.getTopic())) { + foundFilter = filter; + } + } + if (foundFilter != null) { + return applyMasking(record, foundFilter.getKeys()); + } else { + return record; + } + } else { + return record; + } + } catch (Exception e) { + LOG.error("Error masking record", e); + return record; + } + } + + @SneakyThrows + private Record applyMasking(Record record, List keys) { + JsonObject jsonElement = JsonParser.parseString(record.getValue()).getAsJsonObject(); + for(String key : keys) { + maskField(jsonElement, key.split("\\."), 0); + } + record.setValue(jsonElement.toString()); + return record; + } + + private void maskField(JsonObject node, String[] keys, int index) { + if (index == keys.length - 1) { + if (node.has(keys[index])) { + node.addProperty(keys[index], jsonMaskReplacement); + } + } else { + JsonElement childNode = node.get(keys[index]); + if (childNode != null && childNode.isJsonObject()) { + maskField(childNode.getAsJsonObject(), keys, index + 1); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/akhq/utils/Masker.java b/src/main/java/org/akhq/utils/Masker.java new file mode 100644 index 000000000..30c117e77 --- /dev/null +++ b/src/main/java/org/akhq/utils/Masker.java @@ -0,0 +1,12 @@ +package org.akhq.utils; + +import org.akhq.models.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public interface Masker { + + Logger LOG = LoggerFactory.getLogger(Masker.class); + + Record maskRecord(Record record); +} diff --git a/src/main/java/org/akhq/utils/MaskerFactory.java b/src/main/java/org/akhq/utils/MaskerFactory.java new file mode 100644 index 000000000..ea4eaca2f --- /dev/null +++ b/src/main/java/org/akhq/utils/MaskerFactory.java @@ -0,0 +1,22 @@ +package org.akhq.utils; + +import io.micronaut.context.annotation.Bean; +import io.micronaut.context.annotation.Factory; +import org.akhq.configs.DataMasking; + +@Factory +public class MaskerFactory { + + @Bean + public Masker createMaskingUtil(DataMasking dataMasking) { + if(dataMasking == null) { + return new NoOpMasker(); + } + return switch(dataMasking.getMode()) { + case REGEX -> new RegexMasker(dataMasking.getFilters()); + case JSON_MASK_BY_DEFAULT -> new JsonMaskByDefaultMasker(dataMasking.getJsonFilters(), dataMasking.getJsonMaskReplacement()); + case JSON_SHOW_BY_DEFAULT -> new JsonShowByDefaultMasker(dataMasking.getJsonFilters(), dataMasking.getJsonMaskReplacement()); + case NONE -> new NoOpMasker(); + }; + } +} diff --git a/src/main/java/org/akhq/utils/NoOpMasker.java b/src/main/java/org/akhq/utils/NoOpMasker.java new file mode 100644 index 000000000..4e4ce097a --- /dev/null +++ b/src/main/java/org/akhq/utils/NoOpMasker.java @@ -0,0 +1,11 @@ +package org.akhq.utils; + +import org.akhq.models.Record; + +public class NoOpMasker implements Masker { + + @Override + public Record maskRecord(Record record) { + return record; + } +} diff --git a/src/main/java/org/akhq/utils/MaskingUtils.java b/src/main/java/org/akhq/utils/RegexMasker.java similarity index 54% rename from src/main/java/org/akhq/utils/MaskingUtils.java rename to src/main/java/org/akhq/utils/RegexMasker.java index 71bae921d..1347f9fa7 100644 --- a/src/main/java/org/akhq/utils/MaskingUtils.java +++ b/src/main/java/org/akhq/utils/RegexMasker.java @@ -1,27 +1,22 @@ package org.akhq.utils; -import jakarta.inject.Inject; -import jakarta.inject.Singleton; -import org.akhq.configs.DataMasking; -import org.akhq.configs.DataMaskingFilter; +import lombok.RequiredArgsConstructor; +import org.akhq.configs.RegexFilter; import org.akhq.models.Record; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -@Singleton -public class MaskingUtils { - private static final Logger LOG = LoggerFactory.getLogger(MaskingUtils.class); +import java.util.List; - @Inject - DataMasking dataMasking; +@RequiredArgsConstructor +public class RegexMasker implements Masker { - public Record maskRecord(Record record) { - LOG.trace("masking record"); + private final List filters; + @Override + public Record maskRecord(Record record) { String value = record.getValue(); String key = record.getKey(); - for (DataMaskingFilter filter : dataMasking.getFilters()) { + for (RegexFilter filter : filters) { if (value != null) { value = value.replaceAll(filter.getSearchRegex(), filter.getReplacement()); } @@ -35,4 +30,4 @@ public Record maskRecord(Record record) { return record; } -} \ No newline at end of file +} diff --git a/src/test/java/org/akhq/utils/DefaultMaskerSettingTest.java b/src/test/java/org/akhq/utils/DefaultMaskerSettingTest.java new file mode 100644 index 000000000..5532e049a --- /dev/null +++ b/src/test/java/org/akhq/utils/DefaultMaskerSettingTest.java @@ -0,0 +1,29 @@ +package org.akhq.utils; + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.akhq.configs.DataMasking; +import org.junit.jupiter.api.Test; + +import static org.akhq.configs.DataMaskingMode.REGEX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +@MicronautTest +public class DefaultMaskerSettingTest { + + @Inject + DataMasking dataMasking; + + @Inject + Masker masker; + + @Test + void defaultValuesShouldUseRegexForBackwardsCompatibility() { + assertEquals( + REGEX, + dataMasking.getMode() + ); + assertInstanceOf(RegexMasker.class, masker); + } +} diff --git a/src/test/java/org/akhq/utils/JsonMaskByDefaultMaskerTest.java b/src/test/java/org/akhq/utils/JsonMaskByDefaultMaskerTest.java new file mode 100644 index 000000000..4c6952465 --- /dev/null +++ b/src/test/java/org/akhq/utils/JsonMaskByDefaultMaskerTest.java @@ -0,0 +1,71 @@ +package org.akhq.utils; + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.akhq.models.Record; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +@MicronautTest(environments = "json-mask-by-default-data-masking") +class JsonMaskByDefaultMaskerTest extends MaskerTestHelper { + + @Inject + Masker masker; + + @Test + void shouldUseJsonMaskByDefaultMasker() { + assertInstanceOf(JsonMaskByDefaultMasker.class, masker); + } + + @Test + void shouldMasksRecordValue() { + Record record = sampleRecord( + "my-special-topic", + "some-key", + sampleValue() + ); + + Record maskedRecord = masker.maskRecord(record); + + assertEquals( + "{\"specialId\":\"MySpecialId\",\"firstName\":\"xxxx\",\"lastName\":\"xxxx\",\"age\":\"xxxx\",\"status\":\"ACTIVE\",\"metadata\":{\"comments\":\"xxxx\",\"trusted\":true,\"expired\":\"xxxx\",\"rating\":\"A\"}}", + maskedRecord.getValue() + ); + } + + @Test + void forUndefinedTopicShouldDefaultMaskAllValues() { + Record record = sampleRecord( + "different-topic", + "some-key", + sampleValue() + ); + + Record maskedRecord = masker.maskRecord(record); + + assertEquals( + "{\"specialId\":\"xxxx\",\"firstName\":\"xxxx\",\"lastName\":\"xxxx\",\"age\":\"xxxx\",\"status\":\"xxxx\",\"metadata\":{\"comments\":\"xxxx\",\"trusted\":\"xxxx\",\"expired\":\"xxxx\",\"rating\":\"xxxx\"}}", + maskedRecord.getValue() + ); + } + + private String sampleValue() { + return """ + { + "specialId": "MySpecialId", + "firstName": "Test", + "lastName": "Testington", + "age": 100, + "status": "ACTIVE", + "metadata": { + "comments": "Some comment", + "trusted": true, + "expired": false, + "rating": "A" + } + } + """; + } +} diff --git a/src/test/java/org/akhq/utils/JsonShowByDefaultMaskerTest.java b/src/test/java/org/akhq/utils/JsonShowByDefaultMaskerTest.java new file mode 100644 index 000000000..d48ab4a2b --- /dev/null +++ b/src/test/java/org/akhq/utils/JsonShowByDefaultMaskerTest.java @@ -0,0 +1,71 @@ +package org.akhq.utils; + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.akhq.models.Record; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +@MicronautTest(environments = "json-show-by-default-data-masking") +class JsonShowByDefaultMaskerTest extends MaskerTestHelper { + + @Inject + Masker masker; + + @Test + void shouldUseJsonShowByDefaultMasker() { + assertInstanceOf(JsonShowByDefaultMasker.class, masker); + } + + @Test + void shouldMasksRecordValue() { + Record record = sampleRecord( + "my-special-topic", + "some-key", + sampleValue() + ); + + Record maskedRecord = masker.maskRecord(record); + + assertEquals( + "{\"specialId\":\"xxxx\",\"firstName\":\"Test\",\"lastName\":\"Testington\",\"age\":100,\"status\":\"xxxx\",\"metadata\":{\"comments\":\"Some comment\",\"trusted\":\"xxxx\",\"expired\":false,\"rating\":\"xxxx\"}}", + maskedRecord.getValue() + ); + } + + @Test + void shouldDoNothingForUndefinedTopic() { + Record record = sampleRecord( + "different-topic", + "some-key", + sampleValue() + ); + + Record maskedRecord = masker.maskRecord(record); + + assertEquals( + sampleValue(), + maskedRecord.getValue() + ); + } + + private String sampleValue() { + return """ + { + "specialId": "MySpecialId", + "firstName": "Test", + "lastName": "Testington", + "age": 100, + "status": "ACTIVE", + "metadata": { + "comments": "Some comment", + "trusted": true, + "expired": false, + "rating": "A" + } + } + """; + } +} diff --git a/src/test/java/org/akhq/utils/MaskerTestHelper.java b/src/test/java/org/akhq/utils/MaskerTestHelper.java new file mode 100644 index 000000000..5453278d3 --- /dev/null +++ b/src/test/java/org/akhq/utils/MaskerTestHelper.java @@ -0,0 +1,28 @@ +package org.akhq.utils; + +import org.akhq.models.Record; +import org.akhq.models.Topic; +import org.apache.kafka.clients.admin.TopicDescription; + +import java.util.List; + +public class MaskerTestHelper { + + static Record sampleRecord(String topicName, + String key, + String value) { + Record record = new Record(); + record.setTopic( + new Topic( + new TopicDescription(topicName, true, List.of()), + List.of(), + List.of(), + true, + true + ) + ); + record.setKey(key); + record.setValue(value); + return record; + } +} diff --git a/src/test/java/org/akhq/utils/MaskingUtilsTest.java b/src/test/java/org/akhq/utils/MaskingUtilsTest.java deleted file mode 100644 index 26ff0758b..000000000 --- a/src/test/java/org/akhq/utils/MaskingUtilsTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.akhq.utils; - -import io.micronaut.context.ApplicationContext; -import org.akhq.models.Record; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class MaskingUtilsTest { - - @Test - void shouldMasksRecordValue() { - ApplicationContext ctx = ApplicationContext.run("data-masking"); - MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class); - - Record record = new Record(); - record.setValue("{\"secret-key\":\"my-secret-value\"}"); - - Record maskedRecord = maskingUtils.maskRecord(record); - - assertEquals( - "{\"secret-key\":\"xxxx\"}", - maskedRecord.getValue() - ); - - ctx.close(); - } - - @Test - void shouldMasksRecordKey() { - ApplicationContext ctx = ApplicationContext.run("data-masking"); - MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class); - - Record record = new Record(); - record.setKey("{\"secret-key\":\"my-secret-value\"}"); - - Record maskedRecord = maskingUtils.maskRecord(record); - - assertEquals( - "{\"secret-key\":\"xxxx\"}", - maskedRecord.getKey() - ); - - ctx.close(); - } - - @Test - void shouldReturnGroupsToAllowPartialMasking() { - ApplicationContext ctx = ApplicationContext.run("data-masking"); - MaskingUtils maskingUtils = ctx.getBean(MaskingUtils.class); - - Record record = new Record(); - record.setValue("{\"some-key\":\"+12092503766\"}"); - - Record maskedRecord = maskingUtils.maskRecord(record); - - assertEquals( - "{\"some-key\":\"+120925xxxx\"}", - maskedRecord.getValue() - ); - - ctx.close(); - } -} \ No newline at end of file diff --git a/src/test/java/org/akhq/utils/NoOpMaskerTest.java b/src/test/java/org/akhq/utils/NoOpMaskerTest.java new file mode 100644 index 000000000..725316e89 --- /dev/null +++ b/src/test/java/org/akhq/utils/NoOpMaskerTest.java @@ -0,0 +1,47 @@ +package org.akhq.utils; + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.akhq.models.Record; +import org.junit.jupiter.api.Test; + +import static org.akhq.utils.MaskerTestHelper.sampleRecord; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +@MicronautTest(environments = "no-op-data-masking") +public class NoOpMaskerTest { + + @Inject + Masker masker; + + @Test + void shouldUseNoOpMasker() { + assertInstanceOf(NoOpMasker.class, masker); + } + + @Test + void shouldDoNothing() { + Record record = sampleRecord("some-topic", "some-key", + """ + { + "specialId": "MySpecialId", + "firstName": "Test", + "lastName": "Testington", + "age": 100, + "status": "ACTIVE", + "metadata": { + "comments": "Some comment", + "trusted": true, + "expired": false, + "rating": "A" + } + } + """); + Record maskedRecord = masker.maskRecord(record); + assertEquals( + record, + maskedRecord + ); + } +} diff --git a/src/test/java/org/akhq/utils/RegexMaskerTest.java b/src/test/java/org/akhq/utils/RegexMaskerTest.java new file mode 100644 index 000000000..fa1e403bf --- /dev/null +++ b/src/test/java/org/akhq/utils/RegexMaskerTest.java @@ -0,0 +1,67 @@ +package org.akhq.utils; + +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.akhq.models.Record; +import org.junit.jupiter.api.Test; + +import static org.akhq.utils.MaskerTestHelper.sampleRecord; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +@MicronautTest(environments = "regex-data-masking") +class RegexMaskerTest { + + @Inject + Masker masker; + + @Test + void shouldDefaultToRegexDataMasking() { + assertInstanceOf(RegexMasker.class, masker); + } + + @Test + void shouldMasksRecordValue() { + Record record = sampleRecord( + "some-topic", + "{\"secret-key\":\"my-secret-value\"}", + "{\"secret-key\":\"my-secret-value\"}" + ); + Record maskedRecord = masker.maskRecord(record); + + assertEquals( + "{\"secret-key\":\"xxxx\"}", + maskedRecord.getValue() + ); + } + + @Test + void shouldMasksRecordKey() { + Record record = sampleRecord( + "some-topic", + "{\"secret-key\":\"my-secret-value\"}", + "{\"secret-key\":\"my-secret-value\"}" + ); + Record maskedRecord = masker.maskRecord(record); + + assertEquals( + "{\"secret-key\":\"xxxx\"}", + maskedRecord.getKey() + ); + } + + @Test + void shouldReturnGroupsToAllowPartialMasking() { + Record record = sampleRecord( + "some-topic", + "{\"secret-key\":\"my-secret-value\"}", + "{\"some-key\":\"+12092503766\"}" + ); + Record maskedRecord = masker.maskRecord(record); + + assertEquals( + "{\"some-key\":\"+120925xxxx\"}", + maskedRecord.getValue() + ); + } +} diff --git a/src/test/resources/application-json-mask-by-default-data-masking.yml b/src/test/resources/application-json-mask-by-default-data-masking.yml new file mode 100644 index 000000000..3debf481b --- /dev/null +++ b/src/test/resources/application-json-mask-by-default-data-masking.yml @@ -0,0 +1,12 @@ +akhq: + security: + data-masking: + mode: json_mask_by_default + json-filters: + - description: Unmask non-sensitive values + topic: my-special-topic + keys: + - specialId + - status + - trusted + - rating diff --git a/src/test/resources/application-json-show-by-default-data-masking.yml b/src/test/resources/application-json-show-by-default-data-masking.yml new file mode 100644 index 000000000..19642fc2c --- /dev/null +++ b/src/test/resources/application-json-show-by-default-data-masking.yml @@ -0,0 +1,12 @@ +akhq: + security: + data-masking: + mode: json_show_by_default + json-filters: + - description: Mask sensitive values + topic: my-special-topic + keys: + - specialId + - status + - metadata.trusted + - metadata.rating diff --git a/src/test/resources/application-no-op-data-masking.yml b/src/test/resources/application-no-op-data-masking.yml new file mode 100644 index 000000000..ea871fa7f --- /dev/null +++ b/src/test/resources/application-no-op-data-masking.yml @@ -0,0 +1,4 @@ +akhq: + security: + data-masking: + mode: none diff --git a/src/test/resources/application-data-masking.yml b/src/test/resources/application-regex-data-masking.yml similarity index 95% rename from src/test/resources/application-data-masking.yml rename to src/test/resources/application-regex-data-masking.yml index e39d87e7a..5a00e48c2 100644 --- a/src/test/resources/application-data-masking.yml +++ b/src/test/resources/application-regex-data-masking.yml @@ -1,6 +1,7 @@ akhq: security: data-masking: + mode: regex filters: - description: "Masks value for secret-key fields" search-regex: '"(secret-key)":".*"'