From 4b2786026931df73feb4860c9ea716aa6b250833 Mon Sep 17 00:00:00 2001 From: Ajinkya Date: Mon, 6 Jan 2025 16:46:07 +0530 Subject: [PATCH] added maxmind integration --- .github/workflows/prod.yml | 5 + .github/workflows/staging.yml | 4 + apps/threat-detection-backend/.gitignore | 1 + apps/threat-detection-backend/pom.xml | 13 +- .../java/com/akto/threat/backend/Main.java | 8 +- .../threat/backend/client/IPLookupClient.java | 26 ++ .../service/MaliciousEventService.java | 9 +- .../backend/tasks/FlushMessagesToDB.java | 2 +- .../java/com/akto/runtime/utils/Utils.java | 400 +++++++++--------- 9 files changed, 269 insertions(+), 199 deletions(-) create mode 100644 apps/threat-detection-backend/.gitignore create mode 100644 apps/threat-detection-backend/src/main/java/com/akto/threat/backend/client/IPLookupClient.java diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index 4eb5fea827..6fd5110574 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -66,6 +66,11 @@ jobs: wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json wget -O automated_api_groups.csv https://raw.githubusercontent.com/akto-api-security/akto/master/automated-api-groups/automated-api-groups.csv + - name: Download Maxmind Country database + working-directory: ./apps/threat-detection-backend/resources/src/main/resources + run: | + wget -O Geo-Country.mmdb https://raw.githubusercontent.com/akto-api-security/tests-library/refs/heads/master/resources/Geo-Country.mmdb + - name: Prepare Dashboard polaris UI working-directory: ./apps/dashboard/web/polaris_web run: npm install && export RELEASE_VERSION=${{github.event.inputs.release_version}} && npm run build diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index fce7dcb5b8..9ec45b2474 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -41,6 +41,10 @@ jobs: wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json + - name: Download Maxmind Country database + working-directory: ./apps/threat-detection-backend/resources/src/main/resources + run: | + wget -O Geo-Country.mmdb https://raw.githubusercontent.com/akto-api-security/tests-library/refs/heads/master/resources/Geo-Country.mmdb - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 diff --git a/apps/threat-detection-backend/.gitignore b/apps/threat-detection-backend/.gitignore new file mode 100644 index 0000000000..1ffb9e7e9d --- /dev/null +++ b/apps/threat-detection-backend/.gitignore @@ -0,0 +1 @@ +*.mmdb \ No newline at end of file diff --git a/apps/threat-detection-backend/pom.xml b/apps/threat-detection-backend/pom.xml index 077a5e62cf..b7e4129a9c 100644 --- a/apps/threat-detection-backend/pom.xml +++ b/apps/threat-detection-backend/pom.xml @@ -79,6 +79,13 @@ ${vertex.version} + + + com.maxmind.geoip2 + geoip2 + 2.15.0 + + @@ -121,10 +128,6 @@ src/main/resources - true - - **/version.txt - @@ -204,4 +207,4 @@ - + \ No newline at end of file diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/Main.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/Main.java index dc5de50490..6b8ecc2ce6 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/Main.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/Main.java @@ -7,6 +7,7 @@ import com.akto.kafka.KafkaConfig; import com.akto.kafka.KafkaConsumerConfig; import com.akto.kafka.KafkaProducerConfig; +import com.akto.threat.backend.client.IPLookupClient; import com.akto.threat.backend.service.MaliciousEventService; import com.akto.threat.backend.service.ThreatActorService; import com.akto.threat.backend.service.ThreatApiService; @@ -17,6 +18,7 @@ import com.mongodb.WriteConcern; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; +import java.io.File; import org.bson.codecs.configuration.CodecRegistry; import org.bson.codecs.pojo.PojoCodecProvider; @@ -53,10 +55,14 @@ public static void main(String[] args) throws Exception { KafkaProducerConfig.newBuilder().setBatchSize(100).setLingerMs(1000).build()) .build(); + String maxmindPath = Main.class.getClassLoader().getResource("Geo-Country.mmdb").getPath(); + + IPLookupClient ipLookupClient = new IPLookupClient(new File(maxmindPath)); + new FlushMessagesToDB(internalKafkaConfig, threatProtectionMongo).run(); MaliciousEventService maliciousEventService = - new MaliciousEventService(internalKafkaConfig, threatProtectionMongo); + new MaliciousEventService(internalKafkaConfig, threatProtectionMongo, ipLookupClient); ThreatActorService threatActorService = new ThreatActorService(threatProtectionMongo); ThreatApiService threatApiService = new ThreatApiService(threatProtectionMongo); diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/client/IPLookupClient.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/client/IPLookupClient.java new file mode 100644 index 0000000000..10b64166ab --- /dev/null +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/client/IPLookupClient.java @@ -0,0 +1,26 @@ +package com.akto.threat.backend.client; + +import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.model.CountryResponse; +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.util.Optional; + +public class IPLookupClient { + private final DatabaseReader db; + + public IPLookupClient(File dbFile) throws IOException { + this.db = new DatabaseReader.Builder(dbFile).build(); + } + + public Optional getCountryISOCodeGivenIp(String ip) { + try { + InetAddress ipAddr = InetAddress.getByName(ip); + CountryResponse resp = db.country(ipAddr); + return Optional.of(resp.getCountry().getIsoCode()); + } catch (Exception e) { + return Optional.empty(); + } + } +} diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java index 803e8ccb98..f69640a7fa 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/service/MaliciousEventService.java @@ -11,6 +11,7 @@ import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListMaliciousRequestsRequest; import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListMaliciousRequestsResponse; import com.akto.proto.generated.threat_detection.service.malicious_alert_service.v1.RecordMaliciousEventRequest; +import com.akto.threat.backend.client.IPLookupClient; import com.akto.threat.backend.constants.KafkaTopic; import com.akto.threat.backend.constants.MongoDBCollection; import com.akto.threat.backend.db.AggregateSampleMaliciousEventModel; @@ -33,10 +34,13 @@ public class MaliciousEventService { private final Kafka kafka; private MongoClient mongoClient; + private IPLookupClient ipLookupClient; - public MaliciousEventService(KafkaConfig kafkaConfig, MongoClient mongoClient) { + public MaliciousEventService( + KafkaConfig kafkaConfig, MongoClient mongoClient, IPLookupClient ipLookupClient) { this.kafka = new Kafka(kafkaConfig); this.mongoClient = mongoClient; + this.ipLookupClient = ipLookupClient; } public void recordMaliciousEvent(String accountId, RecordMaliciousEventRequest request) { @@ -64,7 +68,8 @@ public void recordMaliciousEvent(String accountId, RecordMaliciousEventRequest r .setLatestApiCollectionId(evt.getLatestApiCollectionId()) .setEventType(maliciousEventType) .setLatestApiIp(evt.getLatestApiIp()) - .setCountry("US") + .setCountry( + this.ipLookupClient.getCountryISOCodeGivenIp(evt.getLatestApiIp()).orElse("")) .setCategory(evt.getCategory()) .setSubCategory(evt.getSubCategory()) .build(); diff --git a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/tasks/FlushMessagesToDB.java b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/tasks/FlushMessagesToDB.java index 543793b3ac..abcb98452d 100644 --- a/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/tasks/FlushMessagesToDB.java +++ b/apps/threat-detection-backend/src/main/java/com/akto/threat/backend/tasks/FlushMessagesToDB.java @@ -71,8 +71,8 @@ public void run() { private void processRecords(ConsumerRecords records) { records.forEach( r -> { - String message = r.value(); try { + String message = r.value(); writeMessage(message); } catch (JsonProcessingException e) { System.out.println("Error while parsing message" + e); diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java index 628cc440aa..ffba6d1876 100644 --- a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -1,17 +1,7 @@ package com.akto.runtime.utils; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.StringJoiner; -import java.util.regex.Pattern; - -import org.apache.commons.lang3.math.NumberUtils; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static com.akto.dto.RawApi.convertHeaders; +import static com.akto.util.HttpRequestResponseUtils.GRPC_CONTENT_TYPE; import com.akto.dto.HttpRequestParams; import com.akto.dto.HttpResponseParams; @@ -24,213 +14,243 @@ import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONObject; import com.mongodb.BasicDBObject; -import static com.akto.dto.RawApi.convertHeaders; - -import static com.akto.util.HttpRequestResponseUtils.GRPC_CONTENT_TYPE; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.StringJoiner; +import java.util.regex.Pattern; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Utils { - private static final Logger logger = LoggerFactory.getLogger(Utils.class); + private static final Logger logger = LoggerFactory.getLogger(Utils.class); - private static int debugPrintCounter = 500; - public static void printL(Object o) { - if (debugPrintCounter > 0) { - debugPrintCounter--; - logger.info(o.toString()); - } - } + private static int debugPrintCounter = 500; - public static Properties configProperties(String kafkaBrokerUrl, String groupIdConfig, int maxPollRecordsConfig) { - Properties properties = new Properties(); - properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokerUrl); - properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); - properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArrayDeserializer"); - properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig); - properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupIdConfig); - properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); - - return properties; + public static void printL(Object o) { + if (debugPrintCounter > 0) { + debugPrintCounter--; + logger.info(o.toString()); } - - public static String convertOriginalReqRespToString(OriginalHttpRequest request, OriginalHttpResponse response, int responseTime) { - BasicDBObject ret = convertOriginalReqRespToStringUtil(request, response); - ret.append("responseTime", responseTime); - return ret.toString(); + } + + public static Properties configProperties( + String kafkaBrokerUrl, String groupIdConfig, int maxPollRecordsConfig) { + Properties properties = new Properties(); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokerUrl); + properties.put( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringDeserializer"); + properties.put( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + "org.apache.kafka.common.serialization.StringDeserializer"); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupIdConfig); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + + return properties; + } + + public static String convertOriginalReqRespToString( + OriginalHttpRequest request, OriginalHttpResponse response, int responseTime) { + BasicDBObject ret = convertOriginalReqRespToStringUtil(request, response); + ret.append("responseTime", responseTime); + return ret.toString(); + } + + public static String convertOriginalReqRespToString( + OriginalHttpRequest request, OriginalHttpResponse response) { + return convertOriginalReqRespToStringUtil(request, response).toString(); + } + + public static BasicDBObject convertOriginalReqRespToStringUtil( + OriginalHttpRequest request, OriginalHttpResponse response) { + BasicDBObject req = new BasicDBObject(); + if (request != null) { + req.put("url", request.getUrl()); + req.put("method", request.getMethod()); + req.put("type", request.getType()); + req.put("queryParams", request.getQueryParams()); + req.put("body", request.getBody()); + req.put("headers", convertHeaders(request.getHeaders())); } - public static String convertOriginalReqRespToString(OriginalHttpRequest request, OriginalHttpResponse response) { - return convertOriginalReqRespToStringUtil(request, response).toString(); + BasicDBObject resp = new BasicDBObject(); + if (response != null) { + resp.put("statusCode", response.getStatusCode()); + resp.put("body", response.getBody()); + resp.put("headers", convertHeaders(response.getHeaders())); } - public static BasicDBObject convertOriginalReqRespToStringUtil(OriginalHttpRequest request, OriginalHttpResponse response) { - BasicDBObject req = new BasicDBObject(); - if (request != null) { - req.put("url", request.getUrl()); - req.put("method", request.getMethod()); - req.put("type", request.getType()); - req.put("queryParams", request.getQueryParams()); - req.put("body", request.getBody()); - req.put("headers", convertHeaders(request.getHeaders())); - } - - BasicDBObject resp = new BasicDBObject(); - if (response != null) { - resp.put("statusCode", response.getStatusCode()); - resp.put("body", response.getBody()); - resp.put("headers", convertHeaders(response.getHeaders())); - } - - BasicDBObject ret = new BasicDBObject(); - ret.put("request", req); - ret.put("response", resp); - - return ret; + BasicDBObject ret = new BasicDBObject(); + ret.put("request", req); + ret.put("response", resp); + + return ret; + } + + public static String convertToSampleMessage(String message) throws Exception { + JSONObject jsonObject = JSON.parseObject(message); + JSONObject request = (JSONObject) jsonObject.get("request"); + JSONObject response = (JSONObject) jsonObject.get("response"); + + JSONObject sampleMessage = new JSONObject(); + if (request != null) { + if (request.get("body") != null) { + sampleMessage.put("requestPayload", request.get("body")); + } + if (request.get("headers") != null) { + sampleMessage.put("requestHeaders", request.get("headers")); + } + // TODO: add query params to url + if (request.get("url") != null) { + sampleMessage.put("path", request.get("url")); + } + if (request.get("method") != null) { + sampleMessage.put("method", request.get("method")); + } + if (request.get("type") != null) { + sampleMessage.put("type", request.get("type")); + } } - - public static String convertToSampleMessage(String message) throws Exception { - JSONObject jsonObject = JSON.parseObject(message); - JSONObject request = (JSONObject) jsonObject.get("request"); - JSONObject response = (JSONObject) jsonObject.get("response"); - - JSONObject sampleMessage = new JSONObject(); - if(request != null) { - if(request.get("body") != null) { - sampleMessage.put("requestPayload", request.get("body")); - } - if(request.get("headers") != null) { - sampleMessage.put("requestHeaders", request.get("headers")); - } - // TODO: add query params to url - if(request.get("url") != null) { - sampleMessage.put("path", request.get("url")); - } - if(request.get("method") != null) { - sampleMessage.put("method", request.get("method")); - } - if(request.get("type") != null) { - sampleMessage.put("type", request.get("type")); - } - } - if(response != null) { - if(response.get("body") != null) { - sampleMessage.put("responsePayload", response.get("body")); - } - if(response.get("headers") != null) { - sampleMessage.put("responseHeaders", response.get("headers")); - } - if(response.get("statusCode") != null) { - sampleMessage.put("statusCode", (Integer)response.getInteger("statusCode")); - } - - } - return sampleMessage.toJSONString(); + if (response != null) { + if (response.get("body") != null) { + sampleMessage.put("responsePayload", response.get("body")); + } + if (response.get("headers") != null) { + sampleMessage.put("responseHeaders", response.get("headers")); + } + if (response.get("statusCode") != null) { + sampleMessage.put("statusCode", (Integer) response.getInteger("statusCode")); + } } - - public static Map parseCookie(List cookieList){ - Map cookieMap = new HashMap<>(); - if(cookieList==null)return cookieMap; - for (String cookieValues : cookieList) { - String[] cookies = cookieValues.split(";"); - for (String cookie : cookies) { - cookie=cookie.trim(); - String[] cookieFields = cookie.split("="); - boolean twoCookieFields = cookieFields.length == 2; - if (twoCookieFields) { - if(!cookieMap.containsKey(cookieFields[0])){ - cookieMap.put(cookieFields[0], cookieFields[1]); - } - } - } + return sampleMessage.toJSONString(); + } + + public static Map parseCookie(List cookieList) { + Map cookieMap = new HashMap<>(); + if (cookieList == null) return cookieMap; + for (String cookieValues : cookieList) { + String[] cookies = cookieValues.split(";"); + for (String cookie : cookies) { + cookie = cookie.trim(); + String[] cookieFields = cookie.split("="); + boolean twoCookieFields = cookieFields.length == 2; + if (twoCookieFields) { + if (!cookieMap.containsKey(cookieFields[0])) { + cookieMap.put(cookieFields[0], cookieFields[1]); + } } - return cookieMap; + } } + return cookieMap; + } - private static int GRPC_DEBUG_COUNTER = 50; - - public static HttpResponseParams parseKafkaMessage(String message) throws Exception { + private static int GRPC_DEBUG_COUNTER = 50; - //convert java object to JSON format + public static HttpResponseParams parseKafkaMessage(String message) throws Exception { - JSONObject jsonObject = JSON.parseObject(message); + // convert java object to JSON format - String method = jsonObject.getString("method"); - String url = jsonObject.getString("path"); - String type = jsonObject.getString("type"); - Map> requestHeaders = OriginalHttpRequest.buildHeadersMap(jsonObject, "requestHeaders"); + JSONObject jsonObject = JSON.parseObject(message); - String rawRequestPayload = jsonObject.getString("requestPayload"); - String requestPayload = HttpRequestResponseUtils.rawToJsonString(rawRequestPayload,requestHeaders); + String method = jsonObject.getString("method"); + String url = jsonObject.getString("path"); + String type = jsonObject.getString("type"); + Map> requestHeaders = + OriginalHttpRequest.buildHeadersMap(jsonObject, "requestHeaders"); - if (GRPC_DEBUG_COUNTER > 0) { - String acceptableContentType = HttpRequestResponseUtils.getAcceptableContentType(requestHeaders); - if (acceptableContentType != null && rawRequestPayload.length() > 0) { - // only if request payload is of FORM_URL_ENCODED_CONTENT_TYPE we convert it to json - if (acceptableContentType.equals(GRPC_CONTENT_TYPE)) { - logger.info("grpc kafka payload:" + message,LogDb.RUNTIME); - GRPC_DEBUG_COUNTER--; - } - } - } + String rawRequestPayload = jsonObject.getString("requestPayload"); + String requestPayload = + HttpRequestResponseUtils.rawToJsonString(rawRequestPayload, requestHeaders); - String apiCollectionIdStr = jsonObject.getOrDefault("akto_vxlan_id", "0").toString(); - int apiCollectionId = 0; - if (NumberUtils.isDigits(apiCollectionIdStr)) { - apiCollectionId = NumberUtils.toInt(apiCollectionIdStr, 0); + if (GRPC_DEBUG_COUNTER > 0) { + String acceptableContentType = + HttpRequestResponseUtils.getAcceptableContentType(requestHeaders); + if (acceptableContentType != null && rawRequestPayload.length() > 0) { + // only if request payload is of FORM_URL_ENCODED_CONTENT_TYPE we convert it to json + if (acceptableContentType.equals(GRPC_CONTENT_TYPE)) { + logger.info("grpc kafka payload:" + message, LogDb.RUNTIME); + GRPC_DEBUG_COUNTER--; } - - HttpRequestParams requestParams = new HttpRequestParams( - method,url,type, requestHeaders, requestPayload, apiCollectionId - ); - - int statusCode = jsonObject.getInteger("statusCode"); - String status = jsonObject.getString("status"); - Map> responseHeaders = OriginalHttpRequest.buildHeadersMap(jsonObject, "responseHeaders"); - String payload = jsonObject.getString("responsePayload"); - payload = HttpRequestResponseUtils.rawToJsonString(payload, responseHeaders); - payload = JSONUtils.parseIfJsonP(payload); - int time = jsonObject.getInteger("time"); - String accountId = jsonObject.getString("akto_account_id"); - String sourceIP = jsonObject.getString("ip"); - String destIP = jsonObject.getString("destIp"); - String direction = jsonObject.getString("direction"); - - String isPendingStr = (String) jsonObject.getOrDefault("is_pending", "false"); - boolean isPending = !isPendingStr.toLowerCase().equals("false"); - String sourceStr = (String) jsonObject.getOrDefault("source", HttpResponseParams.Source.OTHER.name()); - HttpResponseParams.Source source = HttpResponseParams.Source.valueOf(sourceStr); - - return new HttpResponseParams( - type,statusCode, status, responseHeaders, payload, requestParams, time, accountId, isPending, source, message, sourceIP, destIP, direction - ); + } } - public static Pattern createRegexPatternFromList(List discardedUrlList){ - StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); - for (String extension : discardedUrlList) { - if(extension.startsWith("CONTENT-TYPE")){ - continue; - } - joiner.add(extension); - } - String regex = joiner.toString(); - - Pattern pattern = Pattern.compile(regex); - return pattern; + String apiCollectionIdStr = jsonObject.getOrDefault("akto_vxlan_id", "0").toString(); + int apiCollectionId = 0; + if (NumberUtils.isDigits(apiCollectionIdStr)) { + apiCollectionId = NumberUtils.toInt(apiCollectionIdStr, 0); } - public static HttpResponseParams convertRawApiToHttpResponseParams(RawApi rawApi, HttpResponseParams originalHttpResponseParams){ + HttpRequestParams requestParams = + new HttpRequestParams(method, url, type, requestHeaders, requestPayload, apiCollectionId); + + int statusCode = jsonObject.getInteger("statusCode"); + String status = jsonObject.getString("status"); + Map> responseHeaders = + OriginalHttpRequest.buildHeadersMap(jsonObject, "responseHeaders"); + String payload = jsonObject.getString("responsePayload"); + payload = HttpRequestResponseUtils.rawToJsonString(payload, responseHeaders); + payload = JSONUtils.parseIfJsonP(payload); + int time = jsonObject.getInteger("time"); + String accountId = jsonObject.getString("akto_account_id"); + String sourceIP = jsonObject.getString("ip"); + String destIP = jsonObject.getString("destIp"); + String direction = jsonObject.getString("direction"); + + String isPendingStr = (String) jsonObject.getOrDefault("is_pending", "false"); + boolean isPending = !isPendingStr.toLowerCase().equals("false"); + String sourceStr = + (String) jsonObject.getOrDefault("source", HttpResponseParams.Source.OTHER.name()); + HttpResponseParams.Source source = HttpResponseParams.Source.valueOf(sourceStr); + + return new HttpResponseParams( + type, + statusCode, + status, + responseHeaders, + payload, + requestParams, + time, + accountId, + isPending, + source, + message, + sourceIP, + destIP, + direction); + } + + public static Pattern createRegexPatternFromList(List discardedUrlList) { + StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); + for (String extension : discardedUrlList) { + if (extension.startsWith("CONTENT-TYPE")) { + continue; + } + joiner.add(extension); + } + String regex = joiner.toString(); - HttpRequestParams ogRequestParams = originalHttpResponseParams.getRequestParams(); - OriginalHttpRequest modifiedRequest = rawApi.getRequest(); + Pattern pattern = Pattern.compile(regex); + return pattern; + } - ogRequestParams.setHeaders(modifiedRequest.getHeaders()); - ogRequestParams.setUrl(modifiedRequest.getFullUrlWithParams()); - ogRequestParams.setPayload(modifiedRequest.getBody()); + public static HttpResponseParams convertRawApiToHttpResponseParams( + RawApi rawApi, HttpResponseParams originalHttpResponseParams) { - originalHttpResponseParams.setRequestParams(ogRequestParams); + HttpRequestParams ogRequestParams = originalHttpResponseParams.getRequestParams(); + OriginalHttpRequest modifiedRequest = rawApi.getRequest(); - return originalHttpResponseParams; - } + ogRequestParams.setHeaders(modifiedRequest.getHeaders()); + ogRequestParams.setUrl(modifiedRequest.getFullUrlWithParams()); + ogRequestParams.setPayload(modifiedRequest.getBody()); + originalHttpResponseParams.setRequestParams(ogRequestParams); + return originalHttpResponseParams; + } }