From ccac7d1bd4a66434af4573200d390d8188f9d0a9 Mon Sep 17 00:00:00 2001 From: JonnyWong16 <9099342+JonnyWong16@users.noreply.github.com> Date: Mon, 27 Apr 2020 17:34:51 -0700 Subject: [PATCH] Remove maxminddb library --- lib/maxminddb/__init__.py | 46 --- lib/maxminddb/compat.py | 33 -- lib/maxminddb/const.py | 7 - lib/maxminddb/decoder.py | 173 --------- lib/maxminddb/errors.py | 11 - lib/maxminddb/extension/maxminddb.c | 570 ---------------------------- lib/maxminddb/file.py | 66 ---- lib/maxminddb/reader.py | 223 ----------- 8 files changed, 1129 deletions(-) delete mode 100644 lib/maxminddb/__init__.py delete mode 100644 lib/maxminddb/compat.py delete mode 100644 lib/maxminddb/const.py delete mode 100644 lib/maxminddb/decoder.py delete mode 100644 lib/maxminddb/errors.py delete mode 100644 lib/maxminddb/extension/maxminddb.c delete mode 100644 lib/maxminddb/file.py delete mode 100644 lib/maxminddb/reader.py diff --git a/lib/maxminddb/__init__.py b/lib/maxminddb/__init__.py deleted file mode 100644 index 7c6008b3..00000000 --- a/lib/maxminddb/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# pylint:disable=C0111 -import os - -import maxminddb.reader - -try: - import maxminddb.extension -except ImportError: - maxminddb.extension = None - -from maxminddb.const import (MODE_AUTO, MODE_MMAP, MODE_MMAP_EXT, MODE_FILE, - MODE_MEMORY) -from maxminddb.decoder import InvalidDatabaseError - - -def open_database(database, mode=MODE_AUTO): - """Open a Maxmind DB database - - Arguments: - database -- A path to a valid MaxMind DB file such as a GeoIP2 - database file. - mode -- mode to open the database with. Valid mode are: - * MODE_MMAP_EXT - use the C extension with memory map. - * MODE_MMAP - read from memory map. Pure Python. - * MODE_FILE - read database as standard file. Pure Python. - * MODE_MEMORY - load database into memory. Pure Python. - * MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that - order. Default mode. - """ - if (mode == MODE_AUTO and maxminddb.extension and - hasattr(maxminddb.extension, 'Reader')) or mode == MODE_MMAP_EXT: - return maxminddb.extension.Reader(database) - elif mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY): - return maxminddb.reader.Reader(database, mode) - raise ValueError('Unsupported open mode: {0}'.format(mode)) - - -def Reader(database): # pylint: disable=invalid-name - """This exists for backwards compatibility. Use open_database instead""" - return open_database(database) - -__title__ = 'maxminddb' -__version__ = '1.2.1' -__author__ = 'Gregory Oschwald' -__license__ = 'Apache License, Version 2.0' -__copyright__ = 'Copyright 2014 Maxmind, Inc.' diff --git a/lib/maxminddb/compat.py b/lib/maxminddb/compat.py deleted file mode 100644 index 8e2a81c5..00000000 --- a/lib/maxminddb/compat.py +++ /dev/null @@ -1,33 +0,0 @@ -import sys - -import ipaddress - -# pylint: skip-file - -if sys.version_info[0] == 2: - def compat_ip_address(address): - if isinstance(address, bytes): - address = address.decode() - return ipaddress.ip_address(address) - - int_from_byte = ord - - FileNotFoundError = IOError - - def int_from_bytes(b): - if b: - return int(b.encode("hex"), 16) - return 0 - - byte_from_int = chr -else: - def compat_ip_address(address): - return ipaddress.ip_address(address) - - int_from_byte = lambda x: x - - FileNotFoundError = FileNotFoundError - - int_from_bytes = lambda x: int.from_bytes(x, 'big') - - byte_from_int = lambda x: bytes([x]) diff --git a/lib/maxminddb/const.py b/lib/maxminddb/const.py deleted file mode 100644 index 59ea84b6..00000000 --- a/lib/maxminddb/const.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Constants used in the API""" - -MODE_AUTO = 0 -MODE_MMAP_EXT = 1 -MODE_MMAP = 2 -MODE_FILE = 4 -MODE_MEMORY = 8 diff --git a/lib/maxminddb/decoder.py b/lib/maxminddb/decoder.py deleted file mode 100644 index e8f223a8..00000000 --- a/lib/maxminddb/decoder.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -maxminddb.decoder -~~~~~~~~~~~~~~~~~ - -This package contains code for decoding the MaxMind DB data section. - -""" -from __future__ import unicode_literals - -import struct - -from maxminddb.compat import byte_from_int, int_from_bytes -from maxminddb.errors import InvalidDatabaseError - - -class Decoder(object): # pylint: disable=too-few-public-methods - - """Decoder for the data section of the MaxMind DB""" - - def __init__(self, database_buffer, pointer_base=0, pointer_test=False): - """Created a Decoder for a MaxMind DB - - Arguments: - database_buffer -- an mmap'd MaxMind DB file. - pointer_base -- the base number to use when decoding a pointer - pointer_test -- used for internal unit testing of pointer code - """ - self._pointer_test = pointer_test - self._buffer = database_buffer - self._pointer_base = pointer_base - - def _decode_array(self, size, offset): - array = [] - for _ in range(size): - (value, offset) = self.decode(offset) - array.append(value) - return array, offset - - def _decode_boolean(self, size, offset): - return size != 0, offset - - def _decode_bytes(self, size, offset): - new_offset = offset + size - return self._buffer[offset:new_offset], new_offset - - # pylint: disable=no-self-argument - # |-> I am open to better ways of doing this as long as it doesn't involve - # lots of code duplication. - def _decode_packed_type(type_code, type_size, pad=False): - # pylint: disable=protected-access, missing-docstring - def unpack_type(self, size, offset): - if not pad: - self._verify_size(size, type_size) - new_offset = offset + type_size - packed_bytes = self._buffer[offset:new_offset] - if pad: - packed_bytes = packed_bytes.rjust(type_size, b'\x00') - (value,) = struct.unpack(type_code, packed_bytes) - return value, new_offset - return unpack_type - - def _decode_map(self, size, offset): - container = {} - for _ in range(size): - (key, offset) = self.decode(offset) - (value, offset) = self.decode(offset) - container[key] = value - return container, offset - - _pointer_value_offset = { - 1: 0, - 2: 2048, - 3: 526336, - 4: 0, - } - - def _decode_pointer(self, size, offset): - pointer_size = ((size >> 3) & 0x3) + 1 - new_offset = offset + pointer_size - pointer_bytes = self._buffer[offset:new_offset] - packed = pointer_bytes if pointer_size == 4 else struct.pack( - b'!c', byte_from_int(size & 0x7)) + pointer_bytes - unpacked = int_from_bytes(packed) - pointer = unpacked + self._pointer_base + \ - self._pointer_value_offset[pointer_size] - if self._pointer_test: - return pointer, new_offset - (value, _) = self.decode(pointer) - return value, new_offset - - def _decode_uint(self, size, offset): - new_offset = offset + size - uint_bytes = self._buffer[offset:new_offset] - return int_from_bytes(uint_bytes), new_offset - - def _decode_utf8_string(self, size, offset): - new_offset = offset + size - return self._buffer[offset:new_offset].decode('utf-8'), new_offset - - _type_decoder = { - 1: _decode_pointer, - 2: _decode_utf8_string, - 3: _decode_packed_type(b'!d', 8), # double, - 4: _decode_bytes, - 5: _decode_uint, # uint16 - 6: _decode_uint, # uint32 - 7: _decode_map, - 8: _decode_packed_type(b'!i', 4, pad=True), # int32 - 9: _decode_uint, # uint64 - 10: _decode_uint, # uint128 - 11: _decode_array, - 14: _decode_boolean, - 15: _decode_packed_type(b'!f', 4), # float, - } - - def decode(self, offset): - """Decode a section of the data section starting at offset - - Arguments: - offset -- the location of the data structure to decode - """ - new_offset = offset + 1 - (ctrl_byte,) = struct.unpack(b'!B', self._buffer[offset:new_offset]) - type_num = ctrl_byte >> 5 - # Extended type - if not type_num: - (type_num, new_offset) = self._read_extended(new_offset) - - if type_num not in self._type_decoder: - raise InvalidDatabaseError('Unexpected type number ({type}) ' - 'encountered'.format(type=type_num)) - - (size, new_offset) = self._size_from_ctrl_byte( - ctrl_byte, new_offset, type_num) - return self._type_decoder[type_num](self, size, new_offset) - - def _read_extended(self, offset): - (next_byte,) = struct.unpack(b'!B', self._buffer[offset:offset + 1]) - type_num = next_byte + 7 - if type_num < 7: - raise InvalidDatabaseError( - 'Something went horribly wrong in the decoder. An ' - 'extended type resolved to a type number < 8 ' - '({type})'.format(type=type_num)) - return type_num, offset + 1 - - def _verify_size(self, expected, actual): - if expected != actual: - raise InvalidDatabaseError( - 'The MaxMind DB file\'s data section contains bad data ' - '(unknown data type or corrupt data)' - ) - - def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num): - size = ctrl_byte & 0x1f - if type_num == 1: - return size, offset - bytes_to_read = 0 if size < 29 else size - 28 - - new_offset = offset + bytes_to_read - size_bytes = self._buffer[offset:new_offset] - - # Using unpack rather than int_from_bytes as it is about 200 lookups - # per second faster here. - if size == 29: - size = 29 + struct.unpack(b'!B', size_bytes)[0] - elif size == 30: - size = 285 + struct.unpack(b'!H', size_bytes)[0] - elif size > 30: - size = struct.unpack( - b'!I', size_bytes.rjust(4, b'\x00'))[0] + 65821 - - return size, new_offset diff --git a/lib/maxminddb/errors.py b/lib/maxminddb/errors.py deleted file mode 100644 index f04ff028..00000000 --- a/lib/maxminddb/errors.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -maxminddb.errors -~~~~~~~~~~~~~~~~ - -This module contains custom errors for the MaxMind DB reader -""" - - -class InvalidDatabaseError(RuntimeError): - - """This error is thrown when unexpected data is found in the database.""" diff --git a/lib/maxminddb/extension/maxminddb.c b/lib/maxminddb/extension/maxminddb.c deleted file mode 100644 index 9e4d45e2..00000000 --- a/lib/maxminddb/extension/maxminddb.c +++ /dev/null @@ -1,570 +0,0 @@ -#include -#include -#include "structmember.h" - -#define __STDC_FORMAT_MACROS -#include - -static PyTypeObject Reader_Type; -static PyTypeObject Metadata_Type; -static PyObject *MaxMindDB_error; - -typedef struct { - PyObject_HEAD /* no semicolon */ - MMDB_s *mmdb; -} Reader_obj; - -typedef struct { - PyObject_HEAD /* no semicolon */ - PyObject *binary_format_major_version; - PyObject *binary_format_minor_version; - PyObject *build_epoch; - PyObject *database_type; - PyObject *description; - PyObject *ip_version; - PyObject *languages; - PyObject *node_count; - PyObject *record_size; -} Metadata_obj; - -static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list); -static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list); - -#if PY_MAJOR_VERSION >= 3 - #define MOD_INIT(name) PyMODINIT_FUNC PyInit_ ## name(void) - #define RETURN_MOD_INIT(m) return (m) - #define FILE_NOT_FOUND_ERROR PyExc_FileNotFoundError -#else - #define MOD_INIT(name) PyMODINIT_FUNC init ## name(void) - #define RETURN_MOD_INIT(m) return - #define PyInt_FromLong PyLong_FromLong - #define FILE_NOT_FOUND_ERROR PyExc_IOError -#endif - -#ifdef __GNUC__ - # define UNUSED(x) UNUSED_ ## x __attribute__((__unused__)) -#else - # define UNUSED(x) UNUSED_ ## x -#endif - -static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - char *filename; - int mode = 0; - - static char *kwlist[] = {"database", "mode", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", kwlist, &filename, &mode)) { - return -1; - } - - if (mode != 0 && mode != 1) { - PyErr_Format(PyExc_ValueError, "Unsupported open mode (%i). Only " - "MODE_AUTO and MODE_MMAP_EXT are supported by this extension.", - mode); - return -1; - } - - if (0 != access(filename, R_OK)) { - PyErr_Format(FILE_NOT_FOUND_ERROR, - "No such file or directory: '%s'", - filename); - return -1; - } - - MMDB_s *mmdb = (MMDB_s *)malloc(sizeof(MMDB_s)); - if (NULL == mmdb) { - PyErr_NoMemory(); - return -1; - } - - Reader_obj *mmdb_obj = (Reader_obj *)self; - if (!mmdb_obj) { - free(mmdb); - PyErr_NoMemory(); - return -1; - } - - uint16_t status = MMDB_open(filename, MMDB_MODE_MMAP, mmdb); - - if (MMDB_SUCCESS != status) { - free(mmdb); - PyErr_Format( - MaxMindDB_error, - "Error opening database file (%s). Is this a valid MaxMind DB file?", - filename - ); - return -1; - } - - mmdb_obj->mmdb = mmdb; - return 0; -} - -static PyObject *Reader_get(PyObject *self, PyObject *args) -{ - char *ip_address = NULL; - - Reader_obj *mmdb_obj = (Reader_obj *)self; - if (!PyArg_ParseTuple(args, "s", &ip_address)) { - return NULL; - } - - MMDB_s *mmdb = mmdb_obj->mmdb; - - if (NULL == mmdb) { - PyErr_SetString(PyExc_ValueError, - "Attempt to read from a closed MaxMind DB."); - return NULL; - } - - int gai_error = 0; - int mmdb_error = MMDB_SUCCESS; - MMDB_lookup_result_s result = - MMDB_lookup_string(mmdb, ip_address, &gai_error, - &mmdb_error); - - if (0 != gai_error) { - PyErr_Format(PyExc_ValueError, - "'%s' does not appear to be an IPv4 or IPv6 address.", - ip_address); - return NULL; - } - - if (MMDB_SUCCESS != mmdb_error) { - PyObject *exception; - if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_error) { - exception = PyExc_ValueError; - } else { - exception = MaxMindDB_error; - } - PyErr_Format(exception, "Error looking up %s. %s", - ip_address, MMDB_strerror(mmdb_error)); - return NULL; - } - - if (!result.found_entry) { - Py_RETURN_NONE; - } - - MMDB_entry_data_list_s *entry_data_list = NULL; - int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list); - if (MMDB_SUCCESS != status) { - PyErr_Format(MaxMindDB_error, - "Error while looking up data for %s. %s", - ip_address, MMDB_strerror(status)); - MMDB_free_entry_data_list(entry_data_list); - return NULL; - } - - MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - PyObject *py_obj = from_entry_data_list(&entry_data_list); - MMDB_free_entry_data_list(original_entry_data_list); - return py_obj; -} - -static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) -{ - Reader_obj *mmdb_obj = (Reader_obj *)self; - - if (NULL == mmdb_obj->mmdb) { - PyErr_SetString(PyExc_IOError, - "Attempt to read from a closed MaxMind DB."); - return NULL; - } - - MMDB_entry_data_list_s *entry_data_list; - MMDB_get_metadata_as_entry_data_list(mmdb_obj->mmdb, &entry_data_list); - MMDB_entry_data_list_s *original_entry_data_list = entry_data_list; - - PyObject *metadata_dict = from_entry_data_list(&entry_data_list); - MMDB_free_entry_data_list(original_entry_data_list); - if (NULL == metadata_dict || !PyDict_Check(metadata_dict)) { - PyErr_SetString(MaxMindDB_error, - "Error decoding metadata."); - return NULL; - } - - PyObject *args = PyTuple_New(0); - if (NULL == args) { - Py_DECREF(metadata_dict); - return NULL; - } - - PyObject *metadata = PyObject_Call((PyObject *)&Metadata_Type, args, - metadata_dict); - - Py_DECREF(metadata_dict); - return metadata; -} - -static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args)) -{ - Reader_obj *mmdb_obj = (Reader_obj *)self; - - if (NULL != mmdb_obj->mmdb) { - MMDB_close(mmdb_obj->mmdb); - free(mmdb_obj->mmdb); - mmdb_obj->mmdb = NULL; - } - - Py_RETURN_NONE; -} - -static void Reader_dealloc(PyObject *self) -{ - Reader_obj *obj = (Reader_obj *)self; - if (NULL != obj->mmdb) { - Reader_close(self, NULL); - } - - PyObject_Del(self); -} - -static int Metadata_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - - PyObject - *binary_format_major_version, - *binary_format_minor_version, - *build_epoch, - *database_type, - *description, - *ip_version, - *languages, - *node_count, - *record_size; - - static char *kwlist[] = { - "binary_format_major_version", - "binary_format_minor_version", - "build_epoch", - "database_type", - "description", - "ip_version", - "languages", - "node_count", - "record_size", - NULL - }; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOOOOOO", kwlist, - &binary_format_major_version, - &binary_format_minor_version, - &build_epoch, - &database_type, - &description, - &ip_version, - &languages, - &node_count, - &record_size)) { - return -1; - } - - Metadata_obj *obj = (Metadata_obj *)self; - - obj->binary_format_major_version = binary_format_major_version; - obj->binary_format_minor_version = binary_format_minor_version; - obj->build_epoch = build_epoch; - obj->database_type = database_type; - obj->description = description; - obj->ip_version = ip_version; - obj->languages = languages; - obj->node_count = node_count; - obj->record_size = record_size; - - Py_INCREF(obj->binary_format_major_version); - Py_INCREF(obj->binary_format_minor_version); - Py_INCREF(obj->build_epoch); - Py_INCREF(obj->database_type); - Py_INCREF(obj->description); - Py_INCREF(obj->ip_version); - Py_INCREF(obj->languages); - Py_INCREF(obj->node_count); - Py_INCREF(obj->record_size); - - return 0; -} - -static void Metadata_dealloc(PyObject *self) -{ - Metadata_obj *obj = (Metadata_obj *)self; - Py_DECREF(obj->binary_format_major_version); - Py_DECREF(obj->binary_format_minor_version); - Py_DECREF(obj->build_epoch); - Py_DECREF(obj->database_type); - Py_DECREF(obj->description); - Py_DECREF(obj->ip_version); - Py_DECREF(obj->languages); - Py_DECREF(obj->node_count); - Py_DECREF(obj->record_size); - PyObject_Del(self); -} - -static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list) -{ - if (NULL == entry_data_list || NULL == *entry_data_list) { - PyErr_SetString( - MaxMindDB_error, - "Error while looking up data. Your database may be corrupt or you have found a bug in libmaxminddb." - ); - return NULL; - } - - switch ((*entry_data_list)->entry_data.type) { - case MMDB_DATA_TYPE_MAP: - return from_map(entry_data_list); - case MMDB_DATA_TYPE_ARRAY: - return from_array(entry_data_list); - case MMDB_DATA_TYPE_UTF8_STRING: - return PyUnicode_FromStringAndSize( - (*entry_data_list)->entry_data.utf8_string, - (*entry_data_list)->entry_data.data_size - ); - case MMDB_DATA_TYPE_BYTES: - return PyByteArray_FromStringAndSize( - (const char *)(*entry_data_list)->entry_data.bytes, - (Py_ssize_t)(*entry_data_list)->entry_data.data_size); - case MMDB_DATA_TYPE_DOUBLE: - return PyFloat_FromDouble((*entry_data_list)->entry_data.double_value); - case MMDB_DATA_TYPE_FLOAT: - return PyFloat_FromDouble((*entry_data_list)->entry_data.float_value); - case MMDB_DATA_TYPE_UINT16: - return PyLong_FromLong( (*entry_data_list)->entry_data.uint16); - case MMDB_DATA_TYPE_UINT32: - return PyLong_FromLong((*entry_data_list)->entry_data.uint32); - case MMDB_DATA_TYPE_BOOLEAN: - return PyBool_FromLong((*entry_data_list)->entry_data.boolean); - case MMDB_DATA_TYPE_UINT64: - return PyLong_FromUnsignedLongLong( - (*entry_data_list)->entry_data.uint64); - case MMDB_DATA_TYPE_UINT128: - return from_uint128(*entry_data_list); - case MMDB_DATA_TYPE_INT32: - return PyLong_FromLong((*entry_data_list)->entry_data.int32); - default: - PyErr_Format(MaxMindDB_error, - "Invalid data type arguments: %d", - (*entry_data_list)->entry_data.type); - return NULL; - } - return NULL; -} - -static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list) -{ - PyObject *py_obj = PyDict_New(); - if (NULL == py_obj) { - PyErr_NoMemory(); - return NULL; - } - - const uint32_t map_size = (*entry_data_list)->entry_data.data_size; - - uint i; - // entry_data_list cannot start out NULL (see from_entry_data_list). We - // check it in the loop because it may become NULL. - // coverity[check_after_deref] - for (i = 0; i < map_size && entry_data_list; i++) { - *entry_data_list = (*entry_data_list)->next; - - PyObject *key = PyUnicode_FromStringAndSize( - (char *)(*entry_data_list)->entry_data.utf8_string, - (*entry_data_list)->entry_data.data_size - ); - - *entry_data_list = (*entry_data_list)->next; - - PyObject *value = from_entry_data_list(entry_data_list); - if (NULL == value) { - Py_DECREF(key); - Py_DECREF(py_obj); - return NULL; - } - PyDict_SetItem(py_obj, key, value); - Py_DECREF(value); - Py_DECREF(key); - } - - return py_obj; -} - -static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list) -{ - const uint32_t size = (*entry_data_list)->entry_data.data_size; - - PyObject *py_obj = PyList_New(size); - if (NULL == py_obj) { - PyErr_NoMemory(); - return NULL; - } - - uint i; - // entry_data_list cannot start out NULL (see from_entry_data_list). We - // check it in the loop because it may become NULL. - // coverity[check_after_deref] - for (i = 0; i < size && entry_data_list; i++) { - *entry_data_list = (*entry_data_list)->next; - PyObject *value = from_entry_data_list(entry_data_list); - if (NULL == value) { - Py_DECREF(py_obj); - return NULL; - } - // PyList_SetItem 'steals' the reference - PyList_SetItem(py_obj, i, value); - } - return py_obj; -} - -static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list) -{ - uint64_t high = 0; - uint64_t low = 0; -#if MMDB_UINT128_IS_BYTE_ARRAY - int i; - for (i = 0; i < 8; i++) { - high = (high << 8) | entry_data_list->entry_data.uint128[i]; - } - - for (i = 8; i < 16; i++) { - low = (low << 8) | entry_data_list->entry_data.uint128[i]; - } -#else - high = entry_data_list->entry_data.uint128 >> 64; - low = (uint64_t)entry_data_list->entry_data.uint128; -#endif - - char *num_str = malloc(33); - if (NULL == num_str) { - PyErr_NoMemory(); - return NULL; - } - - snprintf(num_str, 33, "%016" PRIX64 "%016" PRIX64, high, low); - - PyObject *py_obj = PyLong_FromString(num_str, NULL, 16); - - free(num_str); - return py_obj; -} - -static PyMethodDef Reader_methods[] = { - { "get", Reader_get, METH_VARARGS, - "Get record for IP address" }, - { "metadata", Reader_metadata, METH_NOARGS, - "Returns metadata object for database" }, - { "close", Reader_close, METH_NOARGS, "Closes database"}, - { NULL, NULL, 0, NULL } -}; - -static PyTypeObject Reader_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_basicsize = sizeof(Reader_obj), - .tp_dealloc = Reader_dealloc, - .tp_doc = "Reader object", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_methods = Reader_methods, - .tp_name = "Reader", - .tp_init = Reader_init, -}; - -static PyMethodDef Metadata_methods[] = { - { NULL, NULL, 0, NULL } -}; - -/* *INDENT-OFF* */ -static PyMemberDef Metadata_members[] = { - { "binary_format_major_version", T_OBJECT, offsetof( - Metadata_obj, binary_format_major_version), READONLY, NULL }, - { "binary_format_minor_version", T_OBJECT, offsetof( - Metadata_obj, binary_format_minor_version), READONLY, NULL }, - { "build_epoch", T_OBJECT, offsetof(Metadata_obj, build_epoch), - READONLY, NULL }, - { "database_type", T_OBJECT, offsetof(Metadata_obj, database_type), - READONLY, NULL }, - { "description", T_OBJECT, offsetof(Metadata_obj, description), - READONLY, NULL }, - { "ip_version", T_OBJECT, offsetof(Metadata_obj, ip_version), - READONLY, NULL }, - { "languages", T_OBJECT, offsetof(Metadata_obj, languages), READONLY, - NULL }, - { "node_count", T_OBJECT, offsetof(Metadata_obj, node_count), - READONLY, NULL }, - { "record_size", T_OBJECT, offsetof(Metadata_obj, record_size), - READONLY, NULL }, - { NULL, 0, 0, 0, NULL } -}; -/* *INDENT-ON* */ - -static PyTypeObject Metadata_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_basicsize = sizeof(Metadata_obj), - .tp_dealloc = Metadata_dealloc, - .tp_doc = "Metadata object", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_members = Metadata_members, - .tp_methods = Metadata_methods, - .tp_name = "Metadata", - .tp_init = Metadata_init -}; - -static PyMethodDef MaxMindDB_methods[] = { - { NULL, NULL, 0, NULL } -}; - - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef MaxMindDB_module = { - PyModuleDef_HEAD_INIT, - .m_name = "extension", - .m_doc = "This is a C extension to read MaxMind DB file format", - .m_methods = MaxMindDB_methods, -}; -#endif - -MOD_INIT(extension){ - PyObject *m; - -#if PY_MAJOR_VERSION >= 3 - m = PyModule_Create(&MaxMindDB_module); -#else - m = Py_InitModule("extension", MaxMindDB_methods); -#endif - - if (!m) { - RETURN_MOD_INIT(NULL); - } - - Reader_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&Reader_Type)) { - RETURN_MOD_INIT(NULL); - } - Py_INCREF(&Reader_Type); - PyModule_AddObject(m, "Reader", (PyObject *)&Reader_Type); - - Metadata_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&Metadata_Type)) { - RETURN_MOD_INIT(NULL); - } - PyModule_AddObject(m, "extension", (PyObject *)&Metadata_Type); - - PyObject* error_mod = PyImport_ImportModule("maxminddb.errors"); - if (error_mod == NULL) { - RETURN_MOD_INIT(NULL); - } - - MaxMindDB_error = PyObject_GetAttrString(error_mod, "InvalidDatabaseError"); - Py_DECREF(error_mod); - - if (MaxMindDB_error == NULL) { - RETURN_MOD_INIT(NULL); - } - - Py_INCREF(MaxMindDB_error); - - /* We primarily add it to the module for backwards compatibility */ - PyModule_AddObject(m, "InvalidDatabaseError", MaxMindDB_error); - - RETURN_MOD_INIT(m); -} diff --git a/lib/maxminddb/file.py b/lib/maxminddb/file.py deleted file mode 100644 index 2e01e756..00000000 --- a/lib/maxminddb/file.py +++ /dev/null @@ -1,66 +0,0 @@ -"""For internal use only. It provides a slice-like file reader.""" - -import os - -try: - # pylint: disable=no-name-in-module - from multiprocessing import Lock -except ImportError: - from threading import Lock - - -class FileBuffer(object): - - """A slice-able file reader""" - - def __init__(self, database): - self._handle = open(database, 'rb') - self._size = os.fstat(self._handle.fileno()).st_size - if not hasattr(os, 'pread'): - self._lock = Lock() - - def __getitem__(self, key): - if isinstance(key, slice): - return self._read(key.stop - key.start, key.start) - elif isinstance(key, int): - return self._read(1, key) - else: - raise TypeError("Invalid argument type.") - - def rfind(self, needle, start): - """Reverse find needle from start""" - pos = self._read(self._size - start - 1, start).rfind(needle) - if pos == -1: - return pos - return start + pos - - def size(self): - """Size of file""" - return self._size - - def close(self): - """Close file""" - self._handle.close() - - if hasattr(os, 'pread'): - - def _read(self, buffersize, offset): - """read that uses pread""" - # pylint: disable=no-member - return os.pread(self._handle.fileno(), buffersize, offset) - - else: - - def _read(self, buffersize, offset): - """read with a lock - - This lock is necessary as after a fork, the different processes - will share the same file table entry, even if we dup the fd, and - as such the same offsets. There does not appear to be a way to - duplicate the file table entry and we cannot re-open based on the - original path as that file may have replaced with another or - unlinked. - """ - with self._lock: - self._handle.seek(offset) - return self._handle.read(buffersize) diff --git a/lib/maxminddb/reader.py b/lib/maxminddb/reader.py deleted file mode 100644 index b45f31e2..00000000 --- a/lib/maxminddb/reader.py +++ /dev/null @@ -1,223 +0,0 @@ -""" -maxminddb.reader -~~~~~~~~~~~~~~~~ - -This module contains the pure Python database reader and related classes. - -""" -from __future__ import unicode_literals - -try: - import mmap -except ImportError: - # pylint: disable=invalid-name - mmap = None - -import struct - -from maxminddb.compat import byte_from_int, int_from_byte, compat_ip_address -from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY -from maxminddb.decoder import Decoder -from maxminddb.errors import InvalidDatabaseError -from maxminddb.file import FileBuffer - - -class Reader(object): - - """ - Instances of this class provide a reader for the MaxMind DB format. IP - addresses can be looked up using the ``get`` method. - """ - - _DATA_SECTION_SEPARATOR_SIZE = 16 - _METADATA_START_MARKER = b"\xAB\xCD\xEFMaxMind.com" - - _ipv4_start = None - - def __init__(self, database, mode=MODE_AUTO): - """Reader for the MaxMind DB file format - - Arguments: - database -- A path to a valid MaxMind DB file such as a GeoIP2 - database file. - mode -- mode to open the database with. Valid mode are: - * MODE_MMAP - read from memory map. - * MODE_FILE - read database as standard file. - * MODE_MEMORY - load database into memory. - * MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default. - """ - # pylint: disable=redefined-variable-type - if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP: - with open(database, 'rb') as db_file: - self._buffer = mmap.mmap( - db_file.fileno(), 0, access=mmap.ACCESS_READ) - self._buffer_size = self._buffer.size() - elif mode in (MODE_AUTO, MODE_FILE): - self._buffer = FileBuffer(database) - self._buffer_size = self._buffer.size() - elif mode == MODE_MEMORY: - with open(database, 'rb') as db_file: - self._buffer = db_file.read() - self._buffer_size = len(self._buffer) - else: - raise ValueError('Unsupported open mode ({0}). Only MODE_AUTO, ' - ' MODE_FILE, and MODE_MEMORY are support by the pure Python ' - 'Reader'.format(mode)) - - metadata_start = self._buffer.rfind(self._METADATA_START_MARKER, - max(0, self._buffer_size - - 128 * 1024)) - - if metadata_start == -1: - self.close() - raise InvalidDatabaseError('Error opening database file ({0}). ' - 'Is this a valid MaxMind DB file?' - ''.format(database)) - - metadata_start += len(self._METADATA_START_MARKER) - metadata_decoder = Decoder(self._buffer, metadata_start) - (metadata, _) = metadata_decoder.decode(metadata_start) - self._metadata = Metadata( - **metadata) # pylint: disable=bad-option-value - - self._decoder = Decoder(self._buffer, self._metadata.search_tree_size - + self._DATA_SECTION_SEPARATOR_SIZE) - - def metadata(self): - """Return the metadata associated with the MaxMind DB file""" - return self._metadata - - def get(self, ip_address): - """Return the record for the ip_address in the MaxMind DB - - - Arguments: - ip_address -- an IP address in the standard string notation - """ - - address = compat_ip_address(ip_address) - - if address.version == 6 and self._metadata.ip_version == 4: - raise ValueError('Error looking up {0}. You attempted to look up ' - 'an IPv6 address in an IPv4-only database.'.format( - ip_address)) - pointer = self._find_address_in_tree(address) - - return self._resolve_data_pointer(pointer) if pointer else None - - def _find_address_in_tree(self, ip_address): - packed = ip_address.packed - - bit_count = len(packed) * 8 - node = self._start_node(bit_count) - - for i in range(bit_count): - if node >= self._metadata.node_count: - break - bit = 1 & (int_from_byte(packed[i >> 3]) >> 7 - (i % 8)) - node = self._read_node(node, bit) - if node == self._metadata.node_count: - # Record is empty - return 0 - elif node > self._metadata.node_count: - return node - - raise InvalidDatabaseError('Invalid node in search tree') - - def _start_node(self, length): - if self._metadata.ip_version != 6 or length == 128: - return 0 - - # We are looking up an IPv4 address in an IPv6 tree. Skip over the - # first 96 nodes. - if self._ipv4_start: - return self._ipv4_start - - node = 0 - for _ in range(96): - if node >= self._metadata.node_count: - break - node = self._read_node(node, 0) - self._ipv4_start = node - return node - - def _read_node(self, node_number, index): - base_offset = node_number * self._metadata.node_byte_size - - record_size = self._metadata.record_size - if record_size == 24: - offset = base_offset + index * 3 - node_bytes = b'\x00' + self._buffer[offset:offset + 3] - elif record_size == 28: - (middle,) = struct.unpack( - b'!B', self._buffer[base_offset + 3:base_offset + 4]) - if index: - middle &= 0x0F - else: - middle = (0xF0 & middle) >> 4 - offset = base_offset + index * 4 - node_bytes = byte_from_int( - middle) + self._buffer[offset:offset + 3] - elif record_size == 32: - offset = base_offset + index * 4 - node_bytes = self._buffer[offset:offset + 4] - else: - raise InvalidDatabaseError( - 'Unknown record size: {0}'.format(record_size)) - return struct.unpack(b'!I', node_bytes)[0] - - def _resolve_data_pointer(self, pointer): - resolved = pointer - self._metadata.node_count + \ - self._metadata.search_tree_size - - if resolved > self._buffer_size: - raise InvalidDatabaseError( - "The MaxMind DB file's search tree is corrupt") - - (data, _) = self._decoder.decode(resolved) - return data - - def close(self): - """Closes the MaxMind DB file and returns the resources to the system""" - # pylint: disable=unidiomatic-typecheck - if type(self._buffer) not in (str, bytes): - self._buffer.close() - - -class Metadata(object): - - """Metadata for the MaxMind DB reader""" - - # pylint: disable=too-many-instance-attributes - def __init__(self, **kwargs): - """Creates new Metadata object. kwargs are key/value pairs from spec""" - # Although I could just update __dict__, that is less obvious and it - # doesn't work well with static analysis tools and some IDEs - self.node_count = kwargs['node_count'] - self.record_size = kwargs['record_size'] - self.ip_version = kwargs['ip_version'] - self.database_type = kwargs['database_type'] - self.languages = kwargs['languages'] - self.binary_format_major_version = kwargs[ - 'binary_format_major_version'] - self.binary_format_minor_version = kwargs[ - 'binary_format_minor_version'] - self.build_epoch = kwargs['build_epoch'] - self.description = kwargs['description'] - - @property - def node_byte_size(self): - """The size of a node in bytes""" - return self.record_size // 4 - - @property - def search_tree_size(self): - """The size of the search tree""" - return self.node_count * self.node_byte_size - - def __repr__(self): - args = ', '.join('%s=%r' % x for x in self.__dict__.items()) - return '{module}.{class_name}({data})'.format( - module=self.__module__, - class_name=self.__class__.__name__, - data=args)