Skip to content

Commit

Permalink
Merge pull request #42 from Moesif/add-gzip-support
Browse files Browse the repository at this point in the history
Add: Support for gzip body
  • Loading branch information
dkm199 authored May 8, 2020
2 parents 0621933 + 11162b7 commit 5884d9c
Show file tree
Hide file tree
Showing 7 changed files with 365 additions and 312 deletions.
184 changes: 92 additions & 92 deletions moesifdjango/client_ip.py
Original file line number Diff line number Diff line change
@@ -1,95 +1,95 @@
import re


def is_ip(value):
if not value is None:
ipv4 = r"^(?:(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\.){3}(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])$"
ipv6 = r"^((?=.*::)(?!.*::.+::)(::)?([\dA-F]{1,4}:(:|\b)|){5}|([\dA-F]{1,4}:){6})((([\dA-F]{1,4}((?!\3)::|:\b|$))|(?!\2\3)){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})$/i"
return re.match(ipv4, value) or re.match(ipv6, value)


def get_client_ip_from_x_forwarded_for(value):
try:

if not value or value is None:
return None

if not isinstance(value, str):
print("Expected a string, got -" + str(type(value)))
else:
# x-forwarded-for may return multiple IP addresses in the format:
# "client IP, proxy 1 IP, proxy 2 IP"
# Therefore, the right-most IP address is the IP address of the most recent proxy
# and the left-most IP address is the IP address of the originating client.
# source: http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/x-forwarded-headers.html
# Azure Web App's also adds a port for some reason, so we'll only use the first part (the IP)
forwardedIps = []

for e in value.split(','):
ip = e.strip()
if ':' in ip:
splitted = ip.split(':')
if (len(splitted) == 2):
forwardedIps.append(splitted[0])
forwardedIps.append(ip)

# Sometimes IP addresses in this header can be 'unknown' (http://stackoverflow.com/a/11285650).
# Therefore taking the left-most IP address that is not unknown
# A Squid configuration directive can also set the value to "unknown" (http://www.squid-cache.org/Doc/config/forwarded_for/)
return next(item for item in forwardedIps if is_ip(item))
except StopIteration:
return value.encode('utf-8')


def get_client_ip(request):
try:
# Standard headers used by Amazon EC2, Heroku, and others.
if 'HTTP_X_CLIENT_IP' in request.META:
if is_ip(request.META.get('HTTP_X_CLIENT_IP')):
return request.META.get('HTTP_X_CLIENT_IP')

# Load-balancers (AWS ELB) or proxies.
if 'HTTP_X_FORWARDED_FOR' in request.META:
x_forwarded_for = get_client_ip_from_x_forwarded_for(request.META.get('HTTP_X_FORWARDED_FOR'))
if is_ip(x_forwarded_for):
return x_forwarded_for

# Cloudflare.
# @see https://support.cloudflare.com/hc/en-us/articles/200170986-How-does-Cloudflare-handle-HTTP-Request-headers-
# CF-Connecting-IP - applied to every request to the origin.
if 'HTTP_CF_CONNECTING_IP' in request.META:
if is_ip(request.META.get('HTTP_CF_CONNECTING_IP')):
return request.META.get('HTTP_CF_CONNECTING_IP')

# Akamai and Cloudflare: True-Client-IP.
if 'HTTP_TRUE_CLIENT_IP' in request.META:
if is_ip(request.META.get('HTTP_TRUE_CLIENT_IP')):
return request.META.get('HTTP_TRUE_CLIENT_IP')

# Default nginx proxy/fcgi; alternative to x-forwarded-for, used by some proxies.
if 'HTTP_X_REAL_IP' in request.META:
if is_ip(request.META.get('HTTP_X_REAL_IP')):
return request.META.get('HTTP_X_REAL_IP')

# (Rackspace LB and Riverbed's Stingray)
# http://www.rackspace.com/knowledge_center/article/controlling-access-to-linux-cloud-sites-based-on-the-client-ip-address
# https://splash.riverbed.com/docs/DOC-1926
if 'HTTP_X_CLUSTER_CLIENT_IP' in request.META:
if is_ip(request.META.get('HTTP_X_CLUSTER_CLIENT_IP')):
return request.META.get('HTTP_X_CLUSTER_CLIENT_IP')

if 'HTTP_X_FORWARDED' in request.META:
if is_ip(request.META.get('HTTP_X_FORWARDED')):
return request.META.get('HTTP_X_FORWARDED')

if 'HTTP_FORWARDED_FOR' in request.META:
if is_ip(request.META.get('HTTP_FORWARDED_FOR')):
return request.META.get('HTTP_FORWARDED_FOR')

if 'HTTP_FORWARDED' in request.META:
if is_ip(request.META.get('HTTP_FORWARDED')):
return request.META.get('HTTP_FORWARDED')

return request.META.get('REMOTE_ADDR')
except:
return request.META.get('REMOTE_ADDR')
class ClientIp:

def is_ip(self, value):
if not value is None:
ipv4 = r"^(?:(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\.){3}(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])$"
ipv6 = r"^((?=.*::)(?!.*::.+::)(::)?([\dA-F]{1,4}:(:|\b)|){5}|([\dA-F]{1,4}:){6})((([\dA-F]{1,4}((?!\3)::|:\b|$))|(?!\2\3)){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})$/i"
return re.match(ipv4, value) or re.match(ipv6, value)

def get_client_ip_from_x_forwarded_for(self, value):
try:

if not value or value is None:
return None

if not isinstance(value, str):
print("Expected a string, got -" + str(type(value)))
else:
# x-forwarded-for may return multiple IP addresses in the format:
# "client IP, proxy 1 IP, proxy 2 IP"
# Therefore, the right-most IP address is the IP address of the most recent proxy
# and the left-most IP address is the IP address of the originating client.
# source: http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/x-forwarded-headers.html
# Azure Web App's also adds a port for some reason, so we'll only use the first part (the IP)
forwardedIps = []

for e in value.split(','):
ip = e.strip()
if ':' in ip:
splitted = ip.split(':')
if (len(splitted) == 2):
forwardedIps.append(splitted[0])
forwardedIps.append(ip)

# Sometimes IP addresses in this header can be 'unknown' (http://stackoverflow.com/a/11285650).
# Therefore taking the left-most IP address that is not unknown
# A Squid configuration directive can also set the value to "unknown" (http://www.squid-cache.org/Doc/config/forwarded_for/)
return next(item for item in forwardedIps if is_ip(item))
except StopIteration:
return value.encode('utf-8')

def get_client_ip(self, request):
try:
# Standard headers used by Amazon EC2, Heroku, and others.
if 'HTTP_X_CLIENT_IP' in request.META:
if is_ip(request.META.get('HTTP_X_CLIENT_IP')):
return request.META.get('HTTP_X_CLIENT_IP')

# Load-balancers (AWS ELB) or proxies.
if 'HTTP_X_FORWARDED_FOR' in request.META:
x_forwarded_for = get_client_ip_from_x_forwarded_for(request.META.get('HTTP_X_FORWARDED_FOR'))
if is_ip(x_forwarded_for):
return x_forwarded_for

# Cloudflare.
# @see https://support.cloudflare.com/hc/en-us/articles/200170986-How-does-Cloudflare-handle-HTTP-Request-headers-
# CF-Connecting-IP - applied to every request to the origin.
if 'HTTP_CF_CONNECTING_IP' in request.META:
if is_ip(request.META.get('HTTP_CF_CONNECTING_IP')):
return request.META.get('HTTP_CF_CONNECTING_IP')

# Akamai and Cloudflare: True-Client-IP.
if 'HTTP_TRUE_CLIENT_IP' in request.META:
if is_ip(request.META.get('HTTP_TRUE_CLIENT_IP')):
return request.META.get('HTTP_TRUE_CLIENT_IP')

# Default nginx proxy/fcgi; alternative to x-forwarded-for, used by some proxies.
if 'HTTP_X_REAL_IP' in request.META:
if is_ip(request.META.get('HTTP_X_REAL_IP')):
return request.META.get('HTTP_X_REAL_IP')

# (Rackspace LB and Riverbed's Stingray)
# http://www.rackspace.com/knowledge_center/article/controlling-access-to-linux-cloud-sites-based-on-the-client-ip-address
# https://splash.riverbed.com/docs/DOC-1926
if 'HTTP_X_CLUSTER_CLIENT_IP' in request.META:
if is_ip(request.META.get('HTTP_X_CLUSTER_CLIENT_IP')):
return request.META.get('HTTP_X_CLUSTER_CLIENT_IP')

if 'HTTP_X_FORWARDED' in request.META:
if is_ip(request.META.get('HTTP_X_FORWARDED')):
return request.META.get('HTTP_X_FORWARDED')

if 'HTTP_FORWARDED_FOR' in request.META:
if is_ip(request.META.get('HTTP_FORWARDED_FOR')):
return request.META.get('HTTP_FORWARDED_FOR')

if 'HTTP_FORWARDED' in request.META:
if is_ip(request.META.get('HTTP_FORWARDED')):
return request.META.get('HTTP_FORWARDED')

return request.META.get('REMOTE_ADDR')
except:
return request.META.get('REMOTE_ADDR')
64 changes: 31 additions & 33 deletions moesifdjango/middleware.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@
from moesifpythonrequest.start_capture.start_capture import StartCapture
from datetime import datetime, timedelta
from .app_config import AppConfig
from .parse_body import ParseBody
from .update_companies import Company
from .update_users import User
from .client_ip import ClientIp
import uuid

# Logger Config
Expand Down Expand Up @@ -90,6 +94,10 @@ def __init__(self, get_response):
self.regex_content_type = re.compile(r'^CONTENT_TYPE$')
self.regex_content_length = re.compile(r'^CONTENT_LENGTH$')
self.app_config = AppConfig()
self.parse_body = ParseBody()
self.client_ip = ClientIp()
self.user = User()
self.company = Company()
self.config = self.app_config.get_config(self.api_client, self.DEBUG)
self.sampling_percentage = 100
self.config_etag = None
Expand Down Expand Up @@ -180,20 +188,17 @@ def flatten_to_string(value):

req_body = None
req_body_transfer_encoding = None
if self.LOG_BODY:
try:
# print("about to serialize request body" + request.body)
if self.DEBUG:
print("about to process request body")
if request._mo_body:
req_body = json.loads(request._mo_body)
req_body = mask_body(req_body, self.middleware_settings.get('REQUEST_BODY_MASKS'))
except:
if request._mo_body:
req_body = base64.standard_b64encode(request._mo_body).decode(encoding="UTF-8")
req_body_transfer_encoding = 'base64'
if self.LOG_BODY and request._mo_body:
if isinstance(request._mo_body, str):
req_body, req_body_transfer_encoding = self.parse_body.parse_string_body(request._mo_body,
self.parse_body.transform_headers(req_headers),
self.middleware_settings.get('REQUEST_BODY_MASKS'))
else:
req_body, req_body_transfer_encoding = self.parse_body.parse_bytes_body(request._mo_body,
self.parse_body.transform_headers(req_headers),
self.middleware_settings.get('REQUEST_BODY_MASKS'))

ip_address = get_client_ip(request)
ip_address = self.client_ip.get_client_ip(request)
uri = request.scheme + "://" + request.get_host() + request.get_full_path()

def mapper(key):
Expand All @@ -211,22 +216,15 @@ def mapper(key):
rsp_body = None
rsp_body_transfer_encoding = None
if self.LOG_BODY and isinstance(response, HttpResponse) and response.content:
if self.DEBUG:
print("about to process response")
print(response.content)
try:
rsp_body = json.loads(response.content)
if self.DEBUG:
print("json parsed succesfully")
rsp_body = mask_body(rsp_body, self.middleware_settings.get('RESPONSE_BODY_MASKS'))

except:
if self.DEBUG:
print("could not json parse, so base64 encode")
rsp_body = base64.standard_b64encode(response.content).decode(encoding="UTF-8")
rsp_body_transfer_encoding = 'base64'
if self.DEBUG:
print("base64 encoded body: " + str(rsp_body))
if isinstance(response.content, str):
rsp_body, rsp_body_transfer_encoding = self.parse_body.parse_string_body(response.content,
self.parse_body.transform_headers(rsp_headers),
self.middleware_settings.get('RESPONSE_BODY_MASKS'))
else:
print("Response bytes body")
rsp_body, rsp_body_transfer_encoding = self.parse_body.parse_bytes_body(response.content,
self.parse_body.transform_headers(rsp_headers),
self.middleware_settings.get('RESPONSE_BODY_MASKS'))

rsp_time = timezone.now()

Expand Down Expand Up @@ -357,13 +355,13 @@ def sending_event():
return response

def update_user(self, user_profile):
update_user(user_profile, self.api_client, self.DEBUG)
self.user.update_user(user_profile, self.api_client, self.DEBUG)

def update_users_batch(self, user_profiles):
update_users_batch(user_profiles, self.api_client, self.DEBUG)
self.user.update_users_batch(user_profiles, self.api_client, self.DEBUG)

def update_company(self, company_profile):
update_company(company_profile, self.api_client, self.DEBUG)
self.company.update_company(company_profile, self.api_client, self.DEBUG)

def update_companies_batch(self, companies_profiles):
update_companies_batch(companies_profiles, self.api_client, self.DEBUG)
self.company.update_companies_batch(companies_profiles, self.api_client, self.DEBUG)
60 changes: 30 additions & 30 deletions moesifdjango/middleware_pre19.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
from moesifpythonrequest.start_capture.start_capture import StartCapture
from datetime import datetime, timedelta
from .app_config import AppConfig
from .parse_body import ParseBody
from .update_companies import Company
from .update_users import User
from .client_ip import ClientIp
import uuid

class MoesifMiddlewarePre19(object):
Expand Down Expand Up @@ -53,6 +57,10 @@ def __init__(self):
self.regex_content_type = re.compile(r'^CONTENT_TYPE$')
self.regex_content_length = re.compile(r'^CONTENT_LENGTH$')
self.app_config = AppConfig()
self.parse_body = ParseBody()
self.client_ip = ClientIp()
self.user = User()
self.company = Company()
self.config = self.app_config.get_config(self.api_client, self.DEBUG)
self.sampling_percentage = 100
self.config_etag = None
Expand Down Expand Up @@ -144,20 +152,18 @@ def flatten_to_string(value):

req_body = None
req_body_transfer_encoding = None
if self.LOG_BODY:
try:
# print("about to serialize request body" + request.body)
if self.DEBUG:
print("about to process request body")
if request._mo_body:
req_body = json.loads(request._mo_body)
except:
if request._mo_body:
req_body = base64.standard_b64encode(request._mo_body).decode(encoding="UTF-8")
req_body_transfer_encoding = 'base64'
if self.LOG_BODY and request._mo_body:
if isinstance(request._mo_body, str):
req_body, req_body_transfer_encoding = self.parse_body.parse_string_body(request._mo_body,
self.parse_body.transform_headers(req_headers),
self.middleware_settings.get('REQUEST_BODY_MASKS'))
else:
req_body, req_body_transfer_encoding = self.parse_body.parse_bytes_body(request._mo_body,
self.parse_body.transform_headers(req_headers),
self.middleware_settings.get('REQUEST_BODY_MASKS'))


ip_address = get_client_ip(request)
ip_address = self.client_ip.get_client_ip(request)
uri = request.scheme + "://" + request.get_host() + request.get_full_path()

def mapper(key):
Expand All @@ -173,20 +179,14 @@ def mapper(key):
rsp_body = None
rsp_body_transfer_encoding = None
if self.LOG_BODY and isinstance(response, HttpResponse) and response.content:
if self.DEBUG:
print("about to process response")
print(response.content)
try:
rsp_body = json.loads(response.content)
if self.DEBUG:
print("jason parsed succesfully")
except:
if self.DEBUG:
print("could not json parse, so base64 encode")
rsp_body = base64.standard_b64encode(response.content).decode(encoding="UTF-8")
rsp_body_transfer_encoding = 'base64'
if self.DEBUG:
print("base64 encoded body: " + str(rsp_body))
if isinstance(response.content, str):
rsp_body, rsp_body_transfer_encoding = self.parse_body.parse_string_body(response.content,
self.parse_body.transform_headers(rsp_headers),
self.middleware_settings.get('RESPONSE_BODY_MASKS'))
else:
rsp_body, rsp_body_transfer_encoding = self.parse_body.parse_bytes_body(response.content,
self.parse_body.transform_headers(rsp_headers),
self.middleware_settings.get('RESPONSE_BODY_MASKS'))


rsp_time = timezone.now()
Expand Down Expand Up @@ -308,13 +308,13 @@ def sending_event():
return response

def update_user(self, user_profile):
update_user(user_profile, self.api_client, self.DEBUG)
self.user.update_user(user_profile, self.api_client, self.DEBUG)

def update_users_batch(self, user_profiles):
update_users_batch(user_profiles, self.api_client, self.DEBUG)
self.user.update_users_batch(user_profiles, self.api_client, self.DEBUG)

def update_company(self, company_profile):
update_company(company_profile, self.api_client, self.DEBUG)
self.company.update_company(company_profile, self.api_client, self.DEBUG)

def update_companies_batch(self, companies_profiles):
update_companies_batch(companies_profiles, self.api_client, self.DEBUG)
self.company.update_companies_batch(companies_profiles, self.api_client, self.DEBUG)
Loading

0 comments on commit 5884d9c

Please sign in to comment.