Skip to content

Commit

Permalink
Merge pull request #3 from aws-samples/2-md5-hash
Browse files Browse the repository at this point in the history
Add AWSIPPrefixes.md5 attribute
  • Loading branch information
cmlccie authored Oct 2, 2021
2 parents 35097ba + 6ce8419 commit 6e3fa1a
Show file tree
Hide file tree
Showing 10 changed files with 402 additions and 134 deletions.
1 change: 1 addition & 0 deletions awsipranges/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# limitations under the License.

from awsipranges.data_loading import get_ranges # noqa: F401
from awsipranges.exceptions import AWSIPRangesException, HTTPError # noqa: F401
from awsipranges.models.awsipprefix import ( # noqa: F401
AWSIPPrefix,
AWSIPv4Prefix,
Expand Down
24 changes: 19 additions & 5 deletions awsipranges/data_loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,27 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

import hashlib
import json
import urllib.request
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Union
from typing import Any, Dict, Optional, Tuple, Union

from awsipranges.config import (
AWS_IP_ADDRESS_RANGES_URL,
CREATE_DATE_FORMAT,
CREATE_DATE_TIMEZONE,
)
from awsipranges.exceptions import raise_for_status
from awsipranges.models.awsipprefix import aws_ip_prefix
from awsipranges.models.awsipprefixes import AWSIPPrefixes
from awsipranges.utils import check_type


def get_json_data(
cafile: Union[str, Path, None] = None, capath: Union[str, Path, None] = None
) -> Dict[str, Any]:
) -> Tuple[Dict[str, Any], Optional[str]]:
"""Retrieve and parse the AWS IP address ranges JSON file."""
check_type("cafile", cafile, (str, Path), optional=True)
cafile = Path(cafile) if isinstance(cafile, str) else cafile
Expand All @@ -38,8 +40,19 @@ def get_json_data(
with urllib.request.urlopen(
AWS_IP_ADDRESS_RANGES_URL, cafile=cafile, capath=capath
) as response:
response_data = json.load(response)
return response_data
raise_for_status(response)

response_bytes = response.read()
response_data = json.loads(response_bytes)

if "md5" in hashlib.algorithms_available:
md5_hash = hashlib.md5()
md5_hash.update(response_bytes)
md5_hex_digest = md5_hash.hexdigest()
else:
md5_hex_digest = None

return response_data, md5_hex_digest


def get_ranges(cafile: Path = None, capath: Path = None) -> AWSIPPrefixes:
Expand Down Expand Up @@ -78,7 +91,7 @@ def get_ranges(cafile: Path = None, capath: Path = None) -> AWSIPPrefixes:
The AWS IP address ranges in a `AWSIPPrefixes` collection.
"""
json_data = get_json_data(cafile=cafile, capath=capath)
json_data, json_md5 = get_json_data(cafile=cafile, capath=capath)

assert "syncToken" in json_data
assert "createDate" in json_data
Expand All @@ -92,4 +105,5 @@ def get_ranges(cafile: Path = None, capath: Path = None) -> AWSIPPrefixes:
).replace(tzinfo=CREATE_DATE_TIMEZONE),
ipv4_prefixes=(aws_ip_prefix(record) for record in json_data["prefixes"]),
ipv6_prefixes=(aws_ip_prefix(record) for record in json_data["ipv6_prefixes"]),
md5=json_md5,
)
77 changes: 77 additions & 0 deletions awsipranges/exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
"""Custom exceptions."""

from typing import Optional, Tuple


class AWSIPRangesException(Exception):
"""Base class for all awsipranges exceptions."""


class HTTPError(AWSIPRangesException):
"""An HTTP/HTTPS error."""

args: Tuple[object, ...]
status: Optional[int]
reason: Optional[str]

def __init__(self, *args, status: Optional[int], reason: Optional[str]):
super(HTTPError, self).__init__(*args)
self.args = args
self.status = status
self.reason = reason

def __repr__(self):
return (
f"{self.__class__.__name__}("
f"{', '.join([repr(arg) for arg in self.args])}, "
f"status={self.status!r}, "
f"reason={self.reason!r}"
f")"
)

def __str__(self):
msg = []
if self.status:
msg.append(str(self.status))

if self.reason:
msg.append(self.reason)

if self.args:
if msg:
msg.append("-")
msg += [str(arg) for arg in self.args]

return " ".join(msg)


def raise_for_status(response):
"""Raise an HTTPError on 4xx and 5xx status codes."""
# Get the status code
if hasattr(response, "status"):
status = int(response.status)
elif hasattr(response, "code"):
status = int(response.code)
elif hasattr(response, "getstatus"):
status = int(response.getstatus())
else:
raise ValueError(
f"Response object {response!r} does not contain a status code."
)

# Get the URL
if hasattr(response, "url"):
url = response.url
elif hasattr(response, "geturl"):
url = response.geturl()
else:
raise ValueError(f"Response object {response!r} does not contain a url.")

# Get the reason, if available
reason = response.reason if hasattr(response, "reason") else None

if 400 <= status < 500:
raise HTTPError(f"Client error for URL: {url}", status=status, reason=reason)

if 500 <= status < 600:
raise HTTPError(f"Server error for URL: {url}", status=status, reason=reason)
53 changes: 33 additions & 20 deletions awsipranges/models/awsipprefixes.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ class AWSIPPrefixes(object):
_create_date: Optional[datetime]
_ipv4_prefixes: Tuple[AWSIPv4Prefix, ...]
_ipv6_prefixes: Tuple[AWSIPv6Prefix, ...]
_md5: Optional[str]

_regions: Optional[FrozenSet[str]] = None
_network_border_groups: Optional[FrozenSet[str]] = None
Expand All @@ -46,18 +47,21 @@ def __init__(
create_date: Optional[datetime] = None,
ipv4_prefixes: Iterable[AWSIPv4Prefix] = None,
ipv6_prefixes: Iterable[AWSIPv6Prefix] = None,
md5: Optional[str] = None,
) -> None:
super().__init__()

check_type("sync_token", sync_token, str, optional=True)
check_type("create_date", create_date, datetime, optional=True)
check_type("ipv4_prefixes", ipv4_prefixes, Iterable)
check_type("ipv6_prefixes", ipv6_prefixes, Iterable)
check_type("md5", md5, str, optional=True)

self._sync_token = sync_token
self._create_date = create_date
self._ipv4_prefixes = self._process_prefixes(ipv4_prefixes)
self._ipv6_prefixes = self._process_prefixes(ipv6_prefixes)
self._md5 = md5

@staticmethod
def _process_prefixes(
Expand Down Expand Up @@ -153,13 +157,22 @@ def ipv6_prefixes(self) -> Tuple[AWSIPv6Prefix, ...]:
"""The IPv6 prefixes in the collection."""
return self._ipv6_prefixes

@property
def md5(self) -> Optional[str]:
"""The MD5 cryptographic hash value of the ip-ranges.json file.
You can use this value to verify the integrity of the downloaded file.
"""
return self._md5

def __repr__(self) -> str:
return pprint.pformat(
{
"sync_token": self.sync_token,
"create_date": self.create_date,
"ipv4_prefixes": self.ipv4_prefixes,
"ipv6_prefixes": self.ipv6_prefixes,
"md5": self.md5,
}
)

Expand Down Expand Up @@ -247,23 +260,23 @@ def get(
],
default=None,
) -> Union[AWSIPv4Prefix, AWSIPv6Prefix]:
"""Get the AWS IP address prefix that contains the IPv4 or IPv6 item.
"""Get the AWS IP address prefix that contains the IPv4 or IPv6 key.
Returns the longest-match prefix that contains the provided item or the
value of the `default=` parameter if the item is not found in the
Returns the longest-match prefix that contains the provided key or the
value of the `default=` parameter if the key is not found in the
collection.
**Parameters:**
- **item** (str, IPv4Address, IPv6Address, IPv4Network, IPv6Network,
IPv4Interface, IPv6Interface, AWSIPv4Prefix, AWSIPv6Prefix) - the item
to retrieve from the collection
- **default** - the value to return if the item is not found in the
- **key** (str, IPv4Address, IPv6Address, IPv4Network, IPv6Network,
IPv4Interface, IPv6Interface, AWSIPv4Prefix, AWSIPv6Prefix) - the IP
address or network to retrieve from the collection
- **default** - the value to return if the key is not found in the
collection
**Returns:**
The `AWSIPv4Prefix` or `AWSIPv6Prefix` that contains the provided `item`.
The `AWSIPv4Prefix` or `AWSIPv6Prefix` that contains the provided key.
"""
try:
return self[key]
Expand All @@ -272,7 +285,7 @@ def get(

def get_prefix_and_supernets(
self,
item: Union[
key: Union[
str,
IPv4Address,
IPv6Address,
Expand All @@ -285,32 +298,32 @@ def get_prefix_and_supernets(
],
default=None,
) -> Optional[Tuple[Union[AWSIPv4Prefix, AWSIPv6Prefix], ...]]:
"""Get the prefix and supernets that contain the IPv4 or IPv6 item.
"""Get the prefix and supernets that contain the IPv4 or IPv6 key.
Returns a tuple that contains the longest-match prefix and supernets
that contains the provided or the value of the `default=` parameter if
the item is not found in the collection.
that contains the provided key or the value of the `default=` parameter
if the key is not found in the collection.
The tuple is sorted by prefix length in ascending order (shorter prefixes
come before longer prefixes).
**Parameters:**
- **item** (str, IPv4Address, IPv6Address, IPv4Network, IPv6Network,
IPv4Interface, IPv6Interface, AWSIPv4Prefix, AWSIPv6Prefix) - the item
to retrieve from the collection
- **default** - the value to return if the item is not found in the
- **key** (str, IPv4Address, IPv6Address, IPv4Network, IPv6Network,
IPv4Interface, IPv6Interface, AWSIPv4Prefix, AWSIPv6Prefix) - the IP
address or network to retrieve from the collection
- **default** - the value to return if the key is not found in the
collection
**Returns:**
A tuple of the `AWSIPv4Prefix`es or `AWSIPv6Prefix`es that contains the
provided `item`.
provided key.
"""
if isinstance(item, (AWSIPv4Prefix, AWSIPv6Prefix)):
network = item.prefix
if isinstance(key, (AWSIPv4Prefix, AWSIPv6Prefix)):
network = key.prefix
else:
network = ip_network(item, strict=False)
network = ip_network(key, strict=False)

prefixes = list()
for supernet in supernets(network):
Expand Down
34 changes: 18 additions & 16 deletions docs/quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ To get started, make sure you have:
- `awsipranges` [installed](./index.md#installation)
- `awsipranges` [upgraded to the latest version](./index.md#installation)

Then dive-in with this QuickStart to start using the AWS IP address ranges as native Python objects!
Then dive-in with this Quickstart to begin working with the AWS IP address ranges as native Python objects!

## Verify server TLS certificates

Expand All @@ -27,15 +27,15 @@ To verify the TLS certificate presented by the Amazon IP ranges server (ip-range

- Stacking (concatenating) the files into a single certificate bundle (single file)

- Storing them in a directory using the OpenSSL hash filenames
- Storing them in a directory using OpenSSL hash filenames

You can do this with the [`c_rehash` script](https://www.openssl.org/docs/man1.1.0/man1/rehash.html) included in many OpenSSL distributions:
You can do this with the [`c_rehash` script](https://www.openssl.org/docs/man1.1.0/man1/rehash.html) included in many OpenSSL distributions:

```shell
❯ c_rehash amazon_root_certificates/
```
```shell
❯ c_rehash amazon_root_certificates/
```

> ***Tip***: See [`tests/unit/test_data_loading.py`](https://github.com/aws-samples/awsipranges/blob/main/tests/unit/test_data_loading.py) in the `awsipranges` repository for sample Python functions that download the Amazon Root CA certificates and prepare the certificates as both a stacked certificate bundle file and as a directory with OpenSSL hash filenames.
> ***Tip***: See [`tests/unit/test_data_loading.py`](https://github.com/aws-samples/awsipranges/blob/main/tests/unit/test_data_loading.py) in the `awsipranges` repository for sample Python functions that download the Amazon Root CA certificates and prepare the certificates as both a stacked certificate bundle file and as a directory with OpenSSL hash filenames.

3. Pass the path to the prepared certificates to the `awsipranges.get_ranges()` function using the `cafile` or `capath` parameters:
Expand Down Expand Up @@ -64,17 +64,20 @@ One line of code (okay, two if you count the import statement) is all it takes t

The [`awsipranges.get_ranges()`](./api.md#get_ranges) function returns an [`AWSIPPrefixes`](./api.md#awsipprefixes) object, which is a structured collection of AWS IP prefixes.

You can access the `create_date` and `sync_token` attributes of the `AWSIPPrefixes` collection to check the version of the downloaded JSON file:
You can access the `create_date` and `sync_token` attributes of the `AWSIPPrefixes` collection to check the version of the downloaded JSON file and verify the integrity of the file with the `md5` attribute:

```python
>>> aws_ip_ranges.create_date
datetime.datetime(2021, 9, 16, 17, 43, 14, tzinfo=datetime.timezone.utc)
datetime.datetime(2021, 10, 1, 16, 33, 13, tzinfo=datetime.timezone.utc)
>>> aws_ip_ranges.sync_token
'1631814194'
'1633105993'
>>> aws_ip_ranges.md5
'59e4cd7f4757a9f380c626d772a5eef2'
```

And you can access the IPv4 and IPv6 address prefixes with the `ipv4_prefixes` and `ipv6_prefixes` attributes:
You can access the IPv4 and IPv6 address prefixes with the `ipv4_prefixes` and `ipv6_prefixes` attributes:

```python
>>> aws_ip_ranges.ipv4_prefixes
Expand All @@ -86,7 +89,6 @@ And you can access the IPv4 and IPv6 address prefixes with the `ipv4_prefixes` a
...)
>>> aws_ip_ranges.ipv6_prefixes
Out[7]:
(AWSIPv6Prefix('2400:6500:0:9::1/128', region='ap-southeast-3', network_border_group='ap-southeast-3', services=('AMAZON',)),
AWSIPv6Prefix('2400:6500:0:9::2/128', region='ap-southeast-3', network_border_group='ap-southeast-3', services=('AMAZON',)),
AWSIPv6Prefix('2400:6500:0:9::3/128', region='ap-southeast-3', network_border_group='ap-southeast-3', services=('AMAZON',)),
Expand Down Expand Up @@ -149,7 +151,7 @@ The AWS IP address ranges contain supernet and subnet prefixes, so an IP address
The `filter()` method allows you to select a subset of AWS IP prefixes from the collection. You can filter on `regions`, `network_border_groups`, IP `versions` (4, 6), and `services`. The `filter()` method returns a new `AWSIPPrefixes` object that contains the subset of IP prefixes that match your filter criteria.
You may pass a single value (`regions='eu-central-2'`) or a sequence of values (`regions=['eu-central-1', 'eu-central-2']`) to the filter parameters. The `filter()` method will return the prefixes that match all the provided parameters; selecting prefixes where the prefix's attributes are contained in the provided set of values.
You may pass a single value (`regions='eu-central-2'`) or a sequence of values (`regions=['eu-central-1', 'eu-central-2']`) to the filter parameters. The `filter()` method returns the prefixes that match all the provided parameters; selecting prefixes where the prefix's attributes intersect the provided set of values.

For example, `filter(regions=['eu-central-1', 'eu-central-2'], services='EC2', versions=4)` will select all IP version `4` prefixes that have `EC2` in the prefix's list of services and are in the `eu-central-1` or `eu-central-2` Regions.
Expand Down Expand Up @@ -195,9 +197,9 @@ frozenset({'AMAZON',
*My router/firewall wants IP networks in a net-mask or host-mask format. Do the AWS IP prefix objects provide a way for me to get the prefix in the format I need?*
[`AWSIPv4Prefix`](./api.md#awsipv4prefix) and [`AWSIPv6Prefix`](./api.md#awsipv6prefix) objects are proxies around [`IPv4Network`](https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv4Network) and [`IPv6Network`](https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv6Network) objects from the Python standard library (see the [`ipaddress`](https://docs.python.org/3/library/ipaddress.html) module). They support all the attributes and methods available on the `IPv4Network` and `IPv6Network` objects. They also inherit additional attributes (like `region`, `network_border_group`, and `services`) and additional functionality from the [`AWSIPPrefix`](./api.md#awsipprefix) base class.
[`AWSIPv4Prefix`](./api.md#awsipv4prefix) and [`AWSIPv6Prefix`](./api.md#awsipv6prefix) objects are proxies around [`IPv4Network`](https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv4Network) and [`IPv6Network`](https://docs.python.org/3/library/ipaddress.html#ipaddress.IPv6Network) objects from the Python standard library (see the [`ipaddress`](https://docs.python.org/3/library/ipaddress.html) module). They support all the attributes and methods available on the `IPv4Network` and `IPv6Network` objects. They also inherit additional attributes (like `region`, `network_border_group`, and `services`) and additional functionality from the [`AWSIPPrefix`](./api.md#awsipprefix) base class.
Combining the functionality provided by the standard library objects with the rich collection capabilities provided by the `awsipranges` library allows you to complete what could be a complex task easily:
Combining the functionality provided by the standard library objects with the rich collection capabilities provided by the `awsipranges` library allows you to complete complex tasks easily:
Like adding routes to the `DYNAMODB` prefixes in the `eu-west-1` Region to a router:
Expand All @@ -211,7 +213,7 @@ ip route 52.94.26.0 255.255.254.0 1.1.1.1
ip route 52.119.240.0 255.255.248.0 1.1.1.1
```
Or, configuring an access control list to allow traffic to `S3` prefixes in `eu-north-1`:
Or, configuring an access control list to allow traffic to the `S3` prefixes in `eu-north-1`:
```python
>>> for prefix in aws_ip_ranges.filter(regions='eu-north-1', services='S3', versions=4):
Expand Down
Loading

0 comments on commit 6e3fa1a

Please sign in to comment.