Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Misc cleanup #185

Merged
merged 8 commits into from
Jan 30, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions docs/conf.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# maxminddb documentation build configuration file, created by
# sphinx-quickstart on Tue Apr 9 13:34:57 2013.
Expand All @@ -12,8 +11,8 @@
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import sys

sys.path.insert(0, os.path.abspath(".."))
import maxminddb
Expand Down
10 changes: 5 additions & 5 deletions examples/benchmark.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-

import argparse
import maxminddb
import random
import socket
import struct
import timeit

import maxminddb

parser = argparse.ArgumentParser(description="Benchmark maxminddb.")
parser.add_argument("--count", default=250000, type=int, help="number of lookups")
parser.add_argument("--mode", default=0, type=int, help="reader mode to use")
Expand All @@ -19,9 +19,9 @@
reader = maxminddb.open_database(args.file, args.mode)


def lookup_ip_address():
def lookup_ip_address() -> None:
ip = socket.inet_ntoa(struct.pack("!L", random.getrandbits(32)))
record = reader.get(str(ip))
reader.get(str(ip))


elapsed = timeit.timeit(
Expand All @@ -30,4 +30,4 @@ def lookup_ip_address():
number=args.count,
)

print("{:,}".format(int(args.count / elapsed)), "lookups per second")
print(f"{int(args.count / elapsed):,}", "lookups per second")
7 changes: 4 additions & 3 deletions maxminddb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@


__all__ = [
"InvalidDatabaseError",
"MODE_AUTO",
"MODE_FD",
"MODE_FILE",
"MODE_MEMORY",
"MODE_MMAP",
"MODE_MMAP_EXT",
"InvalidDatabaseError",
"Reader",
"open_database",
]
Expand All @@ -37,7 +37,7 @@ def open_database(
database: Union[AnyStr, int, os.PathLike, IO],
mode: int = MODE_AUTO,
) -> Reader:
"""Open a MaxMind DB database
"""Open a MaxMind DB database.

Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2 database
Expand All @@ -51,6 +51,7 @@ def open_database(
a path. This mode implies MODE_MEMORY.
* MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that
order. Default mode.

"""
if mode not in (
MODE_AUTO,
Expand All @@ -70,7 +71,7 @@ def open_database(

if not has_extension:
raise ValueError(
"MODE_MMAP_EXT requires the maxminddb.extension module to be available"
"MODE_MMAP_EXT requires the maxminddb.extension module to be available",
)

# The C type exposes the same API as the Python Reader, so for type
Expand Down
2 changes: 1 addition & 1 deletion maxminddb/const.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Constants used in the API"""
"""Constants used in the API."""

MODE_AUTO = 0
MODE_MMAP_EXT = 1
Expand Down
21 changes: 13 additions & 8 deletions maxminddb/decoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"""

import struct
from typing import cast, Dict, List, Tuple, Union
from typing import Dict, List, Tuple, Union, cast

try:
# pylint: disable=unused-import
Expand All @@ -23,20 +23,21 @@


class Decoder: # pylint: disable=too-few-public-methods
"""Decoder for the data section of the MaxMind DB"""
"""Decoder for the data section of the MaxMind DB."""

def __init__(
self,
database_buffer: Union[FileBuffer, "mmap.mmap", bytes],
pointer_base: int = 0,
pointer_test: bool = False,
) -> None:
"""Created a Decoder for a MaxMind DB
"""Created a Decoder for a MaxMind DB.

Arguments:
database_buffer -- an mmap'd MaxMind DB file.
pointer_base -- the base number to use when decoding a pointer
pointer_test -- used for internal unit testing of pointer code

"""
self._pointer_test = pointer_test
self._buffer = database_buffer
Expand Down Expand Up @@ -138,10 +139,11 @@ def _decode_utf8_string(self, size: int, offset: int) -> Tuple[str, int]:
}

def decode(self, offset: int) -> Tuple[Record, int]:
"""Decode a section of the data section starting at offset
"""Decode a section of the data section starting at offset.

Arguments:
offset -- the location of the data structure to decode

"""
new_offset = offset + 1
ctrl_byte = self._buffer[offset]
Expand All @@ -154,7 +156,7 @@ def decode(self, offset: int) -> Tuple[Record, int]:
decoder = self._type_decoder[type_num]
except KeyError as ex:
raise InvalidDatabaseError(
f"Unexpected type number ({type_num}) encountered"
f"Unexpected type number ({type_num}) encountered",
) from ex

(size, new_offset) = self._size_from_ctrl_byte(ctrl_byte, new_offset, type_num)
Expand All @@ -166,7 +168,7 @@ def _read_extended(self, offset: int) -> Tuple[int, int]:
if type_num < 7:
raise InvalidDatabaseError(
"Something went horribly wrong in the decoder. An "
f"extended type resolved to a type number < 8 ({type_num})"
f"extended type resolved to a type number < 8 ({type_num})",
)
return type_num, offset + 1

Expand All @@ -175,11 +177,14 @@ def _verify_size(expected: int, actual: int) -> None:
if expected != actual:
raise InvalidDatabaseError(
"The MaxMind DB file's data section contains bad data "
"(unknown data type or corrupt data)"
"(unknown data type or corrupt data)",
)

def _size_from_ctrl_byte(
self, ctrl_byte: int, offset: int, type_num: int
self,
ctrl_byte: int,
offset: int,
type_num: int,
) -> Tuple[int, int]:
size = ctrl_byte & 0x1F
if type_num == 1 or size < 29:
Expand Down
27 changes: 16 additions & 11 deletions maxminddb/extension.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,25 @@ This module contains the C extension database reader and related classes.

"""

# pylint: disable=E0601,E0602
from ipaddress import IPv4Address, IPv6Address
from os import PathLike
from typing import Any, AnyStr, Dict, IO, List, Optional, Tuple, Union
from typing import IO, Any, AnyStr, Optional, Tuple, Union

from maxminddb import MODE_AUTO
from maxminddb.types import Record

class Reader:
"""
A C extension implementation of a reader for the MaxMind DB format. IP
"""A C extension implementation of a reader for the MaxMind DB format. IP
addresses can be looked up using the ``get`` method.
"""

closed: bool = ...

def __init__(
self, database: Union[AnyStr, int, PathLike, IO], mode: int = MODE_AUTO
self,
database: Union[AnyStr, int, PathLike, IO],
mode: int = MODE_AUTO,
) -> None:
"""Reader for the MaxMind DB file format

Expand All @@ -30,6 +33,7 @@ class Reader:
file, or a file descriptor in the case of MODE_FD.
mode -- mode to open the database with. The only supported modes are
MODE_AUTO and MODE_MMAP_EXT.

"""

def close(self) -> None:
Expand All @@ -38,25 +42,26 @@ class Reader:
def get(self, ip_address: Union[str, IPv6Address, IPv4Address]) -> Optional[Record]:
"""Return the record for the ip_address in the MaxMind DB


Arguments:
ip_address -- an IP address in the standard string notation

"""

def get_with_prefix_len(
self, ip_address: Union[str, IPv6Address, IPv4Address]
self,
ip_address: Union[str, IPv6Address, IPv4Address],
) -> Tuple[Optional[Record], int]:
"""Return a tuple with the record and the associated prefix length


Arguments:
ip_address -- an IP address in the standard string notation

"""

def metadata(self) -> "Metadata":
def metadata(self) -> Metadata:
"""Return the metadata associated with the MaxMind DB file"""

def __enter__(self) -> "Reader": ...
def __enter__(self) -> Reader: ...
def __exit__(self, *args) -> None: ...

# pylint: disable=too-few-public-methods
Expand Down Expand Up @@ -85,7 +90,7 @@ class Metadata:
A string identifying the database type, e.g., "GeoIP2-City".
"""

description: Dict[str, str]
description: dict[str, str]
"""
A map from locales to text descriptions of the database.
"""
Expand All @@ -97,7 +102,7 @@ class Metadata:
both IPv4 and IPv6 lookups.
"""

languages: List[str]
languages: list[str]
"""
A list of locale codes supported by the databse.
"""
Expand Down
12 changes: 6 additions & 6 deletions maxminddb/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@


class FileBuffer:
"""A slice-able file reader"""
"""A slice-able file reader."""

def __init__(self, database: str) -> None:
# pylint: disable=consider-using-with
Expand All @@ -28,31 +28,31 @@ def __getitem__(self, key: Union[slice, int]):
raise TypeError("Invalid argument type.")

def rfind(self, needle: bytes, start: int) -> int:
"""Reverse find needle from start"""
"""Reverse find needle from start."""
pos = self._read(self._size - start - 1, start).rfind(needle)
if pos == -1:
return pos
return start + pos

def size(self) -> int:
"""Size of file"""
"""Size of file."""
return self._size

def close(self) -> None:
"""Close file"""
"""Close file."""
self._handle.close()

if hasattr(os, "pread"):

def _read(self, buffersize: int, offset: int) -> bytes:
"""read that uses pread"""
"""Read that uses pread."""
# pylint: disable=no-member
return os.pread(self._handle.fileno(), buffersize, offset) # type: ignore

else:

def _read(self, buffersize: int, offset: int) -> bytes:
"""read with a lock
"""Read with a lock.

This lock is necessary as after a fork, the different processes
will share the same file table entry, even if we dup the fd, and
Expand Down
Loading
Loading