github/plexpy
1
0
Fork 0
mirror of https://github.com/Tautulli/Tautulli.git synced 2025-07-06 13:11:15 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

IP address lookup using MaxMind GeoLite2 database

Browse source
This commit is contained in:
JonnyWong16 2016-06-30 21:19:54 -07:00
parent f5bfa67c69
commit c96b1eb09d
22 changed files with 2886 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -15,6 +15,7 @@
version.lock
logs/*
cache/*
*.mmdb
# HTTPS Cert/Key #
##################

62
data/interfaces/default/ip_address_modal.html
View file

@ -3,37 +3,38 @@
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true"><i class="fa fa-remove"></i></button>
<h4 class="modal-title" id="myModalLabel">
% if data:
<strong><span id="modal_header_ip_address">
<i class="fa fa-spin fa-refresh"></i>&nbspLoading Details...
% if data:
<i class="fa fa-map-marker"></i> IP Address: ${data}
% else:
<i class="fa fa-exclamation-circle"></i> Invalid IP Address
% endif
</span></strong>
% else:
<i class="fa fa-exclamation-circle"></i>&nbspInvalid IP Address</span></strong>
% endif
</h4>
</div>
<div class="modal-body" id="modal-text">
<div class="col-sm-6">
<div id="ip_error" style="display: none; text-align: center;"></div>
<div class="col-sm-12">
<h4><strong>Location Details</strong></h4>
</div>
<div class="col-sm-6">
<ul class="list-unstyled">
<li>Country: <strong><span id="country"></span></strong></li>
<li>Region: <strong><span id="region"></span></strong></li>
<li>City: <strong><span id="city"></span></strong></li>
</ul>
</div>
<div class="col-sm-6">
<ul class="list-unstyled">
<li>Timezone: <strong><span id="timezone"></span></strong></li>
<li>Latitude: <strong><span id="lat"></span></strong></li>
<li>Longitude: <strong><span id="lon"></span></strong></li>
</ul>
</div>
<div class="col-sm-6">
<h4><strong>Connection Details</strong></h4>
<ul class="list-unstyled">
<li>Organization: <strong><span id="organization"></span></strong></li>
</ul>
</div>
</div>
<div class="modal-footer">
<% from plexpy.helpers import anon_url %>
<span class="text-muted">Telize service written by <a href="${anon_url('https://github.com/fcambus/telize')}" target="_blank">Frederic Cambus</a>.</span>
<span class="text-muted">GeoLite2 data created by <a href="${anon_url('http://www.maxmind.com')}" target="_blank">MaxMind</a>.</span>
</div>
</div>
</div>
@ -42,25 +43,26 @@
<script>
function getUserLocation(ip_address) {
$.ajax({
url: 'https://telize.myhtpc.co.za/geoip/' + ip_address,
cache: true,
async: true,
url: 'get_geoip_lookup',
type: 'GET',
dataType: 'json',
error: function(){
$('#modal_header_ip_address').html("Request failed. Server may be too busy.");
data: { ip_address: ip_address },
cache: true,
async: true,
error: function () {
$('#ip_error').html('<i class="fa fa-exclamation-circle"></i> Request failed.<br /><br />').show();
},
success: function(data) {
$('#modal_header_ip_address').html('<i class="fa fa-map-marker"></i> IP Address: ' + ip_address);
$('#country').html(data.country);
$('#city').html(data.city);
$('#region').html(data.region);
$('#timezone').html(data.timezone);
$('#lat').html(data.latitude);
$('#lon').html(data.longitude);
$('#organization').html(data.organization);
},
timeout: 5000
success: function (data) {
if ('error' in data) {
$('#ip_error').html('<i class="fa fa-exclamation-circle"></i> ' + data.error + '<br /><br />').show();
} else {
$('#country').html(data.country);
$('#city').html(data.city);
$('#region').html(data.region);
$('#timezone').html(data.timezone);
$('#lat').html(data.latitude);
$('#lon').html(data.longitude);
}
}
});
}
getUserLocation('${data}');

21
data/interfaces/default/settings.html
View file

@ -94,6 +94,14 @@
<td>Cache Directory:</td>
<td>${config['cache_dir']}</td>
</tr>
<tr>
<td>GeoLite2 Database:</td>
% if config['geoip_db']:
<td>${config['geoip_db']} | <a class="no-highlight" href="#" id="install_geoip_db">Reinstall / Update</a></td>
% else:
<td><a class="no-highlight" href="#" id="install_geoip_db">Click here to install the database.</a></td>
% endif
</tr>
% if plexpy.ARGS:
<tr>
<td>Arguments:</td>
@ -2236,10 +2244,13 @@ $(document).ready(function() {
}
getSchedulerTable();
function confirmAjaxCall (url, msg) {
$("#confirm-message").text(msg);
function confirmAjaxCall (url, msg, loader) {
$("#confirm-message").html(msg);
$('#confirm-modal').modal();
$('#confirm-modal').one('click', '#confirm-button', function () {
if (loader) {
showMsg(loader, true, false)
}
$.ajax({
url: url,
type: 'POST',
@ -2286,6 +2297,12 @@ $(document).ready(function() {
confirmAjaxCall(url, msg);
});
$("#install_geoip_db").click(function () {
var msg = 'Are you sure you want to install the GeoLite2 database?<br /><br />' +
'The database uses ~65MB in your PlexPy directory.';
var url = 'install_geoip_db';
confirmAjaxCall(url, msg, 'Installing GeoLite2 database.');
});
$('#api_key').click(function(){ $('#api_key').select() });
$("#generate_api").click(function() {

7
lib/geoip2/__init__.py Normal file
View file

@ -0,0 +1,7 @@
# pylint:disable=C0111
__title__ = 'geoip2'
__version__ = '2.4.0'
__author__ = 'Gregory Oschwald'
__license__ = 'Apache License, Version 2.0'
__copyright__ = 'Copyright (c) 2013-2016 Maxmind, Inc.'

17
lib/geoip2/compat.py Normal file
View file

@ -0,0 +1,17 @@
"""Intended for internal use only."""
import sys
import ipaddress
# pylint: skip-file
if sys.version_info[0] == 2:
def compat_ip_address(address):
"""Intended for internal use only."""
if isinstance(address, bytes):
address = address.decode()
return ipaddress.ip_address(address)
else:
def compat_ip_address(address):
"""Intended for internal use only."""
return ipaddress.ip_address(address)

199
lib/geoip2/database.py Normal file
View file

@ -0,0 +1,199 @@
"""
======================
GeoIP2 Database Reader
======================
"""
import inspect
import maxminddb
# pylint: disable=unused-import
from maxminddb import (MODE_AUTO, MODE_MMAP, MODE_MMAP_EXT, MODE_FILE,
MODE_MEMORY)
import geoip2
import geoip2.models
import geoip2.errors
class Reader(object):
"""GeoIP2 database Reader object.
Instances of this class provide a reader for the GeoIP2 database format.
IP addresses can be looked up using the ``country`` and ``city`` methods.
The basic API for this class is the same for every database. First, you
create a reader object, specifying a file name. You then call the method
corresponding to the specific database, passing it the IP address you want
to look up.
If the request succeeds, the method call will return a model class for the
method you called. This model in turn contains multiple record classes,
each of which represents part of the data returned by the database. If the
database does not contain the requested information, the attributes on the
record class will have a ``None`` value.
If the address is not in the database, an
``geoip2.errors.AddressNotFoundError`` exception will be thrown. If the
database is corrupt or invalid, a ``maxminddb.InvalidDatabaseError`` will
be thrown.
"""
def __init__(self, filename, locales=None, mode=MODE_AUTO):
"""Create GeoIP2 Reader.
:param filename: The path to the GeoIP2 database.
:param locales: This is list of locale codes. This argument will be
passed on to record classes to use when their name properties are
called. The default value is ['en'].
The order of the locales is significant. When a record class has
multiple names (country, city, etc.), its name property will return
the name in the first locale that has one.
Note that the only locale which is always present in the GeoIP2
data is "en". If you do not include this locale, the name property
may end up returning None even when the record has an English name.
Currently, the valid locale codes are:
* de -- German
* en -- English names may still include accented characters if that
is the accepted spelling in English. In other words, English does
not mean ASCII.
* es -- Spanish
* fr -- French
* ja -- Japanese
* pt-BR -- Brazilian Portuguese
* ru -- Russian
* zh-CN -- Simplified Chinese.
:param mode: The mode to open the database with. Valid mode are:
* MODE_MMAP_EXT - use the C extension with memory map.
* MODE_MMAP - read from memory map. Pure Python.
* MODE_FILE - read database as standard file. Pure Python.
* MODE_MEMORY - load database into memory. Pure Python.
* MODE_AUTO - try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order.
Default.
"""
if locales is None:
locales = ['en']
self._db_reader = maxminddb.open_database(filename, mode)
self._locales = locales
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.close()
def country(self, ip_address):
"""Get the Country object for the IP address.
:param ip_address: IPv4 or IPv6 address as a string.
:returns: :py:class:`geoip2.models.Country` object
"""
return self._model_for(geoip2.models.Country, 'Country', ip_address)
def city(self, ip_address):
"""Get the City object for the IP address.
:param ip_address: IPv4 or IPv6 address as a string.
:returns: :py:class:`geoip2.models.City` object
"""
return self._model_for(geoip2.models.City, 'City', ip_address)
def anonymous_ip(self, ip_address):
"""Get the AnonymousIP object for the IP address.
:param ip_address: IPv4 or IPv6 address as a string.
:returns: :py:class:`geoip2.models.AnonymousIP` object
"""
return self._flat_model_for(geoip2.models.AnonymousIP,
'GeoIP2-Anonymous-IP', ip_address)
def connection_type(self, ip_address):
"""Get the ConnectionType object for the IP address.
:param ip_address: IPv4 or IPv6 address as a string.
:returns: :py:class:`geoip2.models.ConnectionType` object
"""
return self._flat_model_for(geoip2.models.ConnectionType,
'GeoIP2-Connection-Type', ip_address)
def domain(self, ip_address):
"""Get the Domain object for the IP address.
:param ip_address: IPv4 or IPv6 address as a string.
:returns: :py:class:`geoip2.models.Domain` object
"""
return self._flat_model_for(geoip2.models.Domain, 'GeoIP2-Domain',
ip_address)
def enterprise(self, ip_address):
"""Get the Enterprise object for the IP address.
:param ip_address: IPv4 or IPv6 address as a string.
:returns: :py:class:`geoip2.models.Enterprise` object
"""
return self._model_for(geoip2.models.Enterprise, 'Enterprise',
ip_address)
def isp(self, ip_address):
"""Get the ISP object for the IP address.
:param ip_address: IPv4 or IPv6 address as a string.
:returns: :py:class:`geoip2.models.ISP` object
"""
return self._flat_model_for(geoip2.models.ISP, 'GeoIP2-ISP',
ip_address)
def _get(self, database_type, ip_address):
if database_type not in self.metadata().database_type:
caller = inspect.stack()[2][3]
raise TypeError("The %s method cannot be used with the "
"%s database" %
(caller, self.metadata().database_type))
record = self._db_reader.get(ip_address)
if record is None:
raise geoip2.errors.AddressNotFoundError(
"The address %s is not in the database." % ip_address)
return record
def _model_for(self, model_class, types, ip_address):
record = self._get(types, ip_address)
record.setdefault('traits', {})['ip_address'] = ip_address
return model_class(record, locales=self._locales)
def _flat_model_for(self, model_class, types, ip_address):
record = self._get(types, ip_address)
record['ip_address'] = ip_address
return model_class(record)
def metadata(self):
"""The metadata for the open database.
:returns: :py:class:`maxminddb.reader.Metadata` object
"""
return self._db_reader.metadata()
def close(self):
"""Closes the GeoIP2 database."""
self._db_reader.close()

51
lib/geoip2/errors.py Normal file
View file

@ -0,0 +1,51 @@
"""
Errors
======
"""
class GeoIP2Error(RuntimeError):
"""There was a generic error in GeoIP2.
This class represents a generic error. It extends :py:exc:`RuntimeError`
and does not add any additional attributes.
"""
class AddressNotFoundError(GeoIP2Error):
"""The address you were looking up was not found."""
class AuthenticationError(GeoIP2Error):
"""There was a problem authenticating the request."""
class HTTPError(GeoIP2Error):
"""There was an error when making your HTTP request.
This class represents an HTTP transport error. It extends
:py:exc:`GeoIP2Error` and adds attributes of its own.
:ivar http_status: The HTTP status code returned
:ivar uri: The URI queried
"""
def __init__(self, message, http_status=None, uri=None):
super(HTTPError, self).__init__(message)
self.http_status = http_status
self.uri = uri
class InvalidRequestError(GeoIP2Error):
"""The request was invalid."""
class OutOfQueriesError(GeoIP2Error):
"""Your account is out of funds for the service queried."""
class PermissionRequiredError(GeoIP2Error):
"""Your account does not have permission to access this service."""

16
lib/geoip2/mixins.py Normal file
View file

@ -0,0 +1,16 @@
"""This package contains utility mixins"""
# pylint: disable=too-few-public-methods
from abc import ABCMeta
class SimpleEquality(object):
"""Naive __dict__ equality mixin"""
__metaclass__ = ABCMeta
def __eq__(self, other):
return (isinstance(other, self.__class__) and
self.__dict__ == other.__dict__)
def __ne__(self, other):
return not self.__eq__(other)

472
lib/geoip2/models.py Normal file
View file

@ -0,0 +1,472 @@
"""
Models
======
These classes provide models for the data returned by the GeoIP2
web service and databases.
The only difference between the City and Insights model classes is which
fields in each record may be populated. See
http://dev.maxmind.com/geoip/geoip2/web-services for more details.
"""
# pylint: disable=too-many-instance-attributes,too-few-public-methods
from abc import ABCMeta
import geoip2.records
from geoip2.mixins import SimpleEquality
class Country(SimpleEquality):
"""Model for the GeoIP2 Precision: Country and the GeoIP2 Country database.
This class provides the following attributes:
.. attribute:: continent
Continent object for the requested IP address.
:type: :py:class:`geoip2.records.Continent`
.. attribute:: country
Country object for the requested IP address. This record represents the
country where MaxMind believes the IP is located.
:type: :py:class:`geoip2.records.Country`
.. attribute:: maxmind
Information related to your MaxMind account.
:type: :py:class:`geoip2.records.MaxMind`
.. attribute:: registered_country
The registered country object for the requested IP address. This record
represents the country where the ISP has registered a given IP block in
and may differ from the user's country.
:type: :py:class:`geoip2.records.Country`
.. attribute:: represented_country
Object for the country represented by the users of the IP address
when that country is different than the country in ``country``. For
instance, the country represented by an overseas military base.
:type: :py:class:`geoip2.records.RepresentedCountry`
.. attribute:: traits
Object with the traits of the requested IP address.
:type: :py:class:`geoip2.records.Traits`
"""
def __init__(self, raw_response, locales=None):
if locales is None:
locales = ['en']
self._locales = locales
self.continent = \
geoip2.records.Continent(locales,
**raw_response.get('continent', {}))
self.country = \
geoip2.records.Country(locales,
**raw_response.get('country', {}))
self.registered_country = \
geoip2.records.Country(locales,
**raw_response.get('registered_country',
{}))
self.represented_country \
= geoip2.records.RepresentedCountry(locales,
**raw_response.get(
'represented_country', {}))
self.maxmind = \
geoip2.records.MaxMind(**raw_response.get('maxmind', {}))
self.traits = geoip2.records.Traits(**raw_response.get('traits', {}))
self.raw = raw_response
def __repr__(self):
return '{module}.{class_name}({data}, {locales})'.format(
module=self.__module__,
class_name=self.__class__.__name__,
data=self.raw,
locales=self._locales)
class City(Country):
"""Model for the GeoIP2 Precision: City and the GeoIP2 City database.
.. attribute:: city
City object for the requested IP address.
:type: :py:class:`geoip2.records.City`
.. attribute:: continent
Continent object for the requested IP address.
:type: :py:class:`geoip2.records.Continent`
.. attribute:: country
Country object for the requested IP address. This record represents the
country where MaxMind believes the IP is located.
:type: :py:class:`geoip2.records.Country`
.. attribute:: location
Location object for the requested IP address.
.. attribute:: maxmind
Information related to your MaxMind account.
:type: :py:class:`geoip2.records.MaxMind`
.. attribute:: registered_country
The registered country object for the requested IP address. This record
represents the country where the ISP has registered a given IP block in
and may differ from the user's country.
:type: :py:class:`geoip2.records.Country`
.. attribute:: represented_country
Object for the country represented by the users of the IP address
when that country is different than the country in ``country``. For
instance, the country represented by an overseas military base.
:type: :py:class:`geoip2.records.RepresentedCountry`
.. attribute:: subdivisions
Object (tuple) representing the subdivisions of the country to which
the location of the requested IP address belongs.
:type: :py:class:`geoip2.records.Subdivisions`
.. attribute:: traits
Object with the traits of the requested IP address.
:type: :py:class:`geoip2.records.Traits`
"""
def __init__(self, raw_response, locales=None):
super(City, self).__init__(raw_response, locales)
self.city = \
geoip2.records.City(locales, **raw_response.get('city', {}))
self.location = \
geoip2.records.Location(**raw_response.get('location', {}))
self.postal = \
geoip2.records.Postal(**raw_response.get('postal', {}))
self.subdivisions = \
geoip2.records.Subdivisions(locales,
*raw_response.get('subdivisions', []))
class Insights(City):
"""Model for the GeoIP2 Precision: Insights web service endpoint.
.. attribute:: city
City object for the requested IP address.
:type: :py:class:`geoip2.records.City`
.. attribute:: continent
Continent object for the requested IP address.
:type: :py:class:`geoip2.records.Continent`
.. attribute:: country
Country object for the requested IP address. This record represents the
country where MaxMind believes the IP is located.
:type: :py:class:`geoip2.records.Country`
.. attribute:: location
Location object for the requested IP address.
.. attribute:: maxmind
Information related to your MaxMind account.
:type: :py:class:`geoip2.records.MaxMind`
.. attribute:: registered_country
The registered country object for the requested IP address. This record
represents the country where the ISP has registered a given IP block in
and may differ from the user's country.
:type: :py:class:`geoip2.records.Country`
.. attribute:: represented_country
Object for the country represented by the users of the IP address
when that country is different than the country in ``country``. For
instance, the country represented by an overseas military base.
:type: :py:class:`geoip2.records.RepresentedCountry`
.. attribute:: subdivisions
Object (tuple) representing the subdivisions of the country to which
the location of the requested IP address belongs.
:type: :py:class:`geoip2.records.Subdivisions`
.. attribute:: traits
Object with the traits of the requested IP address.
:type: :py:class:`geoip2.records.Traits`
"""
class Enterprise(City):
"""Model for the GeoIP2 Enterprise database.
.. attribute:: city
City object for the requested IP address.
:type: :py:class:`geoip2.records.City`
.. attribute:: continent
Continent object for the requested IP address.
:type: :py:class:`geoip2.records.Continent`
.. attribute:: country
Country object for the requested IP address. This record represents the
country where MaxMind believes the IP is located.
:type: :py:class:`geoip2.records.Country`
.. attribute:: location
Location object for the requested IP address.
.. attribute:: maxmind
Information related to your MaxMind account.
:type: :py:class:`geoip2.records.MaxMind`
.. attribute:: registered_country
The registered country object for the requested IP address. This record
represents the country where the ISP has registered a given IP block in
and may differ from the user's country.
:type: :py:class:`geoip2.records.Country`
.. attribute:: represented_country
Object for the country represented by the users of the IP address
when that country is different than the country in ``country``. For
instance, the country represented by an overseas military base.
:type: :py:class:`geoip2.records.RepresentedCountry`
.. attribute:: subdivisions
Object (tuple) representing the subdivisions of the country to which
the location of the requested IP address belongs.
:type: :py:class:`geoip2.records.Subdivisions`
.. attribute:: traits
Object with the traits of the requested IP address.
:type: :py:class:`geoip2.records.Traits`
"""
class SimpleModel(SimpleEquality):
"""Provides basic methods for non-location models"""
__metaclass__ = ABCMeta
def __repr__(self):
# pylint: disable=no-member
return '{module}.{class_name}({data})'.format(
module=self.__module__,
class_name=self.__class__.__name__,
data=str(self.raw))
class AnonymousIP(SimpleModel):
"""Model class for the GeoIP2 Anonymous IP.
This class provides the following attribute:
.. attribute:: is_anonymous
This is true if the IP address belongs to any sort of anonymous network.
:type: bool
.. attribute:: is_anonymous_vpn
This is true if the IP address belongs to an anonymous VPN system.
:type: bool
.. attribute:: is_hosting_provider
This is true if the IP address belongs to a hosting provider.
:type: bool
.. attribute:: is_public_proxy
This is true if the IP address belongs to a public proxy.
:type: bool
.. attribute:: is_tor_exit_node
This is true if the IP address is a Tor exit node.
:type: bool
.. attribute:: ip_address
The IP address used in the lookup.
:type: unicode
"""
def __init__(self, raw):
self.is_anonymous = raw.get('is_anonymous', False)
self.is_anonymous_vpn = raw.get('is_anonymous_vpn', False)
self.is_hosting_provider = raw.get('is_hosting_provider', False)
self.is_public_proxy = raw.get('is_public_proxy', False)
self.is_tor_exit_node = raw.get('is_tor_exit_node', False)
self.ip_address = raw.get('ip_address')
self.raw = raw
class ConnectionType(SimpleModel):
"""Model class for the GeoIP2 Connection-Type.
This class provides the following attribute:
.. attribute:: connection_type
The connection type may take the following values:
- Dialup
- Cable/DSL
- Corporate
- Cellular
Additional values may be added in the future.
:type: unicode
.. attribute:: ip_address
The IP address used in the lookup.
:type: unicode
"""
def __init__(self, raw):
self.connection_type = raw.get('connection_type')
self.ip_address = raw.get('ip_address')
self.raw = raw
class Domain(SimpleModel):
"""Model class for the GeoIP2 Domain.
This class provides the following attribute:
.. attribute:: domain
The domain associated with the IP address.
:type: unicode
.. attribute:: ip_address
The IP address used in the lookup.
:type: unicode
"""
def __init__(self, raw):
self.domain = raw.get('domain')
self.ip_address = raw.get('ip_address')
self.raw = raw
class ISP(SimpleModel):
"""Model class for the GeoIP2 ISP.
This class provides the following attribute:
.. attribute:: autonomous_system_number
The autonomous system number associated with the IP address.
:type: int
.. attribute:: autonomous_system_organization
The organization associated with the registered autonomous system number
for the IP address.
:type: unicode
.. attribute:: isp
The name of the ISP associated with the IP address.
:type: unicode
.. attribute:: organization
The name of the organization associated with the IP address.
:type: unicode
.. attribute:: ip_address
The IP address used in the lookup.
:type: unicode
"""
# pylint:disable=too-many-arguments
def __init__(self, raw):
self.autonomous_system_number = raw.get('autonomous_system_number')
self.autonomous_system_organization = raw.get(
'autonomous_system_organization')
self.isp = raw.get('isp')
self.organization = raw.get('organization')
self.ip_address = raw.get('ip_address')
self.raw = raw

605
lib/geoip2/records.py Normal file
View file

@ -0,0 +1,605 @@
"""
Records
=======
"""
# pylint:disable=R0903
from abc import ABCMeta
from geoip2.mixins import SimpleEquality
class Record(SimpleEquality):
"""All records are subclasses of the abstract class ``Record``."""
__metaclass__ = ABCMeta
_valid_attributes = set()
def __init__(self, **kwargs):
valid_args = dict((k, kwargs.get(k)) for k in self._valid_attributes)
self.__dict__.update(valid_args)
def __setattr__(self, name, value):
raise AttributeError("can't set attribute")
def __repr__(self):
args = ', '.join('%s=%r' % x for x in self.__dict__.items())
return '{module}.{class_name}({data})'.format(
module=self.__module__,
class_name=self.__class__.__name__,
data=args)
class PlaceRecord(Record):
"""All records with :py:attr:`names` subclass :py:class:`PlaceRecord`."""
__metaclass__ = ABCMeta
def __init__(self, locales=None, **kwargs):
if locales is None:
locales = ['en']
if kwargs.get('names') is None:
kwargs['names'] = {}
object.__setattr__(self, '_locales', locales)
super(PlaceRecord, self).__init__(**kwargs)
@property
def name(self):
"""Dict with locale codes as keys and localized name as value."""
# pylint:disable=E1101
return next(
(self.names.get(x) for x in self._locales
if x in self.names), None)
class City(PlaceRecord):
"""Contains data for the city record associated with an IP address.
This class contains the city-level data associated with an IP address.
This record is returned by ``city``, ``enterprise``, and ``insights``.
Attributes:
.. attribute:: confidence
A value from 0-100 indicating MaxMind's
confidence that the city is correct. This attribute is only available
from the Insights end point and the GeoIP2 Enterprise database.
:type: int
.. attribute:: geoname_id
The GeoName ID for the city.
:type: int
.. attribute:: name
The name of the city based on the locales list passed to the
constructor.
:type: unicode
.. attribute:: names
A dictionary where the keys are locale codes
and the values are names.
:type: dict
"""
_valid_attributes = set(['confidence', 'geoname_id', 'names'])
class Continent(PlaceRecord):
"""Contains data for the continent record associated with an IP address.
This class contains the continent-level data associated with an IP
address.
Attributes:
.. attribute:: code
A two character continent code like "NA" (North America)
or "OC" (Oceania).
:type: unicode
.. attribute:: geoname_id
The GeoName ID for the continent.
:type: int
.. attribute:: name
Returns the name of the continent based on the locales list passed to
the constructor.
:type: unicode
.. attribute:: names
A dictionary where the keys are locale codes
and the values are names.
:type: dict
"""
_valid_attributes = set(['code', 'geoname_id', 'names'])
class Country(PlaceRecord):
"""Contains data for the country record associated with an IP address.
This class contains the country-level data associated with an IP address.
Attributes:
.. attribute:: confidence
A value from 0-100 indicating MaxMind's confidence that
the country is correct. This attribute is only available from the
Insights end point and the GeoIP2 Enterprise database.
:type: int
.. attribute:: geoname_id
The GeoName ID for the country.
:type: int
.. attribute:: iso_code
The two-character `ISO 3166-1
<http://en.wikipedia.org/wiki/ISO_3166-1>`_ alpha code for the
country.
:type: unicode
.. attribute:: name
The name of the country based on the locales list passed to the
constructor.
:type: unicode
.. attribute:: names
A dictionary where the keys are locale codes and the values
are names.
:type: dict
"""
_valid_attributes = set(['confidence', 'geoname_id', 'iso_code', 'names'])
class RepresentedCountry(Country):
"""Contains data for the represented country associated with an IP address.
This class contains the country-level data associated with an IP address
for the IP's represented country. The represented country is the country
represented by something like a military base.
Attributes:
.. attribute:: confidence
A value from 0-100 indicating MaxMind's confidence that
the country is correct. This attribute is only available from the
Insights end point and the GeoIP2 Enterprise database.
:type: int
.. attribute:: geoname_id
The GeoName ID for the country.
:type: int
.. attribute:: iso_code
The two-character `ISO 3166-1
<http://en.wikipedia.org/wiki/ISO_3166-1>`_ alpha code for the country.
:type: unicode
.. attribute:: name
The name of the country based on the locales list passed to the
constructor.
:type: unicode
.. attribute:: names
A dictionary where the keys are locale codes and the values
are names.
:type: dict
.. attribute:: type
A string indicating the type of entity that is representing the
country. Currently we only return ``military`` but this could expand to
include other types in the future.
:type: unicode
"""
_valid_attributes = set(['confidence', 'geoname_id', 'iso_code', 'names',
'type'])
class Location(Record):
"""Contains data for the location record associated with an IP address.
This class contains the location data associated with an IP address.
This record is returned by ``city``, ``enterprise``, and ``insights``.
Attributes:
.. attribute:: average_income
The average income in US dollars associated with the requested IP
address. This attribute is only available from the Insights end point.
:type: int
.. attribute:: accuracy_radius
The radius in kilometers around the specified location where the IP
address is likely to be.
:type: int
.. attribute:: latitude
The approximate latitude of the location associated with the IP
address. This value is not precise and should not be used to identify a
particular address or household.
:type: float
.. attribute:: longitude
The approximate longitude of the location associated with the IP
address. This value is not precise and should not be used to identify a
particular address or household.
:type: float
.. attribute:: metro_code
The metro code of the location if the
location is in the US. MaxMind returns the same metro codes as the
`Google AdWords API
<https://developers.google.com/adwords/api/docs/appendix/cities-DMAregions>`_.
:type: int
.. attribute:: population_density
The estimated population per square kilometer associated with the IP
address. This attribute is only available from the Insights end point.
:type: int
.. attribute:: time_zone
The time zone associated with location, as specified by the `IANA Time
Zone Database <http://www.iana.org/time-zones>`_, e.g.,
"America/New_York".
:type: unicode
"""
_valid_attributes = set(['average_income', 'accuracy_radius', 'latitude',
'longitude', 'metro_code', 'population_density',
'postal_code', 'postal_confidence', 'time_zone'])
class MaxMind(Record):
"""Contains data related to your MaxMind account.
Attributes:
.. attribute:: queries_remaining
The number of remaining queries you have
for the end point you are calling.
:type: int
"""
_valid_attributes = set(['queries_remaining'])
class Postal(Record):
"""Contains data for the postal record associated with an IP address.
This class contains the postal data associated with an IP address.
This attribute is returned by ``city``, ``enterprise``, and ``insights``.
Attributes:
.. attribute:: code
The postal code of the location. Postal
codes are not available for all countries. In some countries, this will
only contain part of the postal code.
:type: unicode
.. attribute:: confidence
A value from 0-100 indicating
MaxMind's confidence that the postal code is correct. This attribute is
only available from the Insights end point and the GeoIP2 Enterprise
database.
:type: int
"""
_valid_attributes = set(['code', 'confidence'])
class Subdivision(PlaceRecord):
"""Contains data for the subdivisions associated with an IP address.
This class contains the subdivision data associated with an IP address.
This attribute is returned by ``city``, ``enterprise``, and ``insights``.
Attributes:
.. attribute:: confidence
This is a value from 0-100 indicating MaxMind's
confidence that the subdivision is correct. This attribute is only
available from the Insights end point and the GeoIP2 Enterprise
database.
:type: int
.. attribute:: geoname_id
This is a GeoName ID for the subdivision.
:type: int
.. attribute:: iso_code
This is a string up to three characters long
contain the subdivision portion of the `ISO 3166-2 code
<http://en.wikipedia.org/wiki/ISO_3166-2>`_.
:type: unicode
.. attribute:: name
The name of the subdivision based on the locales list passed to the
constructor.
:type: unicode
.. attribute:: names
A dictionary where the keys are locale codes and the
values are names
:type: dict
"""
_valid_attributes = set(['confidence', 'geoname_id', 'iso_code', 'names'])
class Subdivisions(tuple):
"""A tuple-like collection of subdivisions associated with an IP address.
This class contains the subdivisions of the country associated with the
IP address from largest to smallest.
For instance, the response for Oxford in the United Kingdom would have
England as the first element and Oxfordshire as the second element.
This attribute is returned by ``city``, ``enterprise``, and ``insights``.
"""
def __new__(cls, locales, *subdivisions):
subdivisions = [Subdivision(locales, **x) for x in subdivisions]
obj = super(cls, Subdivisions).__new__(cls, subdivisions)
return obj
def __init__(self, locales, *subdivisions): # pylint:disable=W0613
self._locales = locales
super(Subdivisions, self).__init__()
@property
def most_specific(self):
"""The most specific (smallest) subdivision available.
If there are no :py:class:`Subdivision` objects for the response,
this returns an empty :py:class:`Subdivision`.
:type: :py:class:`Subdivision`
"""
try:
return self[-1]
except IndexError:
return Subdivision(self._locales)
class Traits(Record):
"""Contains data for the traits record associated with an IP address.
This class contains the traits data associated with an IP address.
This class has the following attributes:
.. attribute:: autonomous_system_number
The `autonomous system
number <http://en.wikipedia.org/wiki/Autonomous_system_(Internet)>`_
associated with the IP address. This attribute is only available from
the City and Insights web service end points and the GeoIP2 Enterprise
database.
:type: int
.. attribute:: autonomous_system_organization
The organization associated with the registered `autonomous system
number <http://en.wikipedia.org/wiki/Autonomous_system_(Internet)>`_ for
the IP address. This attribute is only available from the City and
Insights web service end points and the GeoIP2 Enterprise database.
:type: unicode
.. attribute:: connection_type
The connection type may take the following values:
- Dialup
- Cable/DSL
- Corporate
- Cellular
Additional values may be added in the future.
This attribute is only available in the GeoIP2 Enterprise database.
:type: unicode
.. attribute:: domain
The second level domain associated with the
IP address. This will be something like "example.com" or
"example.co.uk", not "foo.example.com". This attribute is only available
from the City and Insights web service end points and the GeoIP2
Enterprise database.
:type: unicode
.. attribute:: ip_address
The IP address that the data in the model
is for. If you performed a "me" lookup against the web service, this
will be the externally routable IP address for the system the code is
running on. If the system is behind a NAT, this may differ from the IP
address locally assigned to it.
:type: unicode
.. attribute:: is_anonymous_proxy
This is true if the IP is an anonymous
proxy. See http://dev.maxmind.com/faq/geoip#anonproxy for further
details.
:type: bool
.. deprecated:: 2.2.0
Use our our `GeoIP2 Anonymous IP database
<https://www.maxmind.com/en/geoip2-anonymous-ip-database GeoIP2>`_
instead.
.. attribute:: is_legitimate_proxy
This attribute is true if MaxMind believes this IP address to be a
legitimate proxy, such as an internal VPN used by a corporation. This
attribute is only available in the GeoIP2 Enterprise database.
:type: bool
.. attribute:: is_satellite_provider
This is true if the IP address is from a satellite provider that
provides service to multiple countries.
:type: bool
.. deprecated:: 2.2.0
Due to the increased coverage by mobile carriers, very few
satellite providers now serve multiple countries. As a result, the
output does not provide sufficiently relevant data for us to maintain
it.
.. attribute:: isp
The name of the ISP associated with the IP address. This attribute is
only available from the City and Insights web service end points and the
GeoIP2 Enterprise database.
:type: unicode
.. attribute:: organization
The name of the organization associated with the IP address. This
attribute is only available from the City and Insights web service end
points and the GeoIP2 Enterprise database.
:type: unicode
.. attribute:: user_type
The user type associated with the IP
address. This can be one of the following values:
* business
* cafe
* cellular
* college
* content_delivery_network
* dialup
* government
* hosting
* library
* military
* residential
* router
* school
* search_engine_spider
* traveler
This attribute is only available from the Insights end point and the
GeoIP2 Enterprise database.
:type: unicode
"""
_valid_attributes = set(
['autonomous_system_number', 'autonomous_system_organization',
'connection_type', 'domain', 'is_anonymous_proxy',
'is_legitimate_proxy', 'is_satellite_provider', 'isp', 'ip_address',
'organization', 'user_type'])
def __init__(self, **kwargs):
for k in ['is_anonymous_proxy', 'is_legitimate_proxy',
'is_satellite_provider']:
kwargs[k] = bool(kwargs.get(k, False))
super(Traits, self).__init__(**kwargs)

219
lib/geoip2/webservice.py Normal file
View file

@ -0,0 +1,219 @@
"""
============================
WebServices Client API
============================
This class provides a client API for all the GeoIP2 Precision web service end
points. The end points are Country, City, and Insights. Each end point returns
a different set of data about an IP address, with Country returning the least
data and Insights the most.
Each web service end point is represented by a different model class, and
these model classes in turn contain multiple record classes. The record
classes have attributes which contain data about the IP address.
If the web service does not return a particular piece of data for an IP
address, the associated attribute is not populated.
The web service may not return any information for an entire record, in which
case all of the attributes for that record class will be empty.
SSL
---
Requests to the GeoIP2 Precision web service are always made with SSL.
"""
import requests
from requests.utils import default_user_agent
import geoip2
import geoip2.models
from .compat import compat_ip_address
from .errors import (AddressNotFoundError, AuthenticationError, GeoIP2Error,
HTTPError, InvalidRequestError, OutOfQueriesError,
PermissionRequiredError)
class Client(object):
"""Creates a new client object.
It accepts the following required arguments:
:param user_id: Your MaxMind User ID.
:param license_key: Your MaxMind license key.
Go to https://www.maxmind.com/en/my_license_key to see your MaxMind
User ID and license key.
The following keyword arguments are also accepted:
:param host: The hostname to make a request against. This defaults to
"geoip.maxmind.com". In most cases, you should not need to set this
explicitly.
:param locales: This is list of locale codes. This argument will be
passed on to record classes to use when their name properties are
called. The default value is ['en'].
The order of the locales is significant. When a record class has
multiple names (country, city, etc.), its name property will return
the name in the first locale that has one.
Note that the only locale which is always present in the GeoIP2
data is "en". If you do not include this locale, the name property
may end up returning None even when the record has an English name.
Currently, the valid locale codes are:
* de -- German
* en -- English names may still include accented characters if that is
the accepted spelling in English. In other words, English does not
mean ASCII.
* es -- Spanish
* fr -- French
* ja -- Japanese
* pt-BR -- Brazilian Portuguese
* ru -- Russian
* zh-CN -- Simplified Chinese.
"""
def __init__(self,
user_id,
license_key,
host='geoip.maxmind.com',
locales=None,
timeout=None):
"""Construct a Client."""
# pylint: disable=too-many-arguments
if locales is None:
locales = ['en']
self._locales = locales
self._user_id = user_id
self._license_key = license_key
self._base_uri = 'https://%s/geoip/v2.1' % host
self._timeout = timeout
def city(self, ip_address='me'):
"""Call GeoIP2 Precision City endpoint with the specified IP.
:param ip_address: IPv4 or IPv6 address as a string. If no
address is provided, the address that the web service is
called from will be used.
:returns: :py:class:`geoip2.models.City` object
"""
return self._response_for('city', geoip2.models.City, ip_address)
def country(self, ip_address='me'):
"""Call the GeoIP2 Country endpoint with the specified IP.
:param ip_address: IPv4 or IPv6 address as a string. If no address
is provided, the address that the web service is called from will
be used.
:returns: :py:class:`geoip2.models.Country` object
"""
return self._response_for('country', geoip2.models.Country, ip_address)
def insights(self, ip_address='me'):
"""Call the GeoIP2 Precision: Insights endpoint with the specified IP.
:param ip_address: IPv4 or IPv6 address as a string. If no address
is provided, the address that the web service is called from will
be used.
:returns: :py:class:`geoip2.models.Insights` object
"""
return self._response_for('insights', geoip2.models.Insights,
ip_address)
def _response_for(self, path, model_class, ip_address):
if ip_address != 'me':
ip_address = str(compat_ip_address(ip_address))
uri = '/'.join([self._base_uri, path, ip_address])
response = requests.get(uri,
auth=(self._user_id, self._license_key),
headers={'Accept': 'application/json',
'User-Agent': self._user_agent()},
timeout=self._timeout)
if response.status_code == 200:
body = self._handle_success(response, uri)
return model_class(body, locales=self._locales)
else:
self._handle_error(response, uri)
def _user_agent(self):
return 'GeoIP2 Python Client v%s (%s)' % (geoip2.__version__,
default_user_agent())
def _handle_success(self, response, uri):
try:
return response.json()
except ValueError as ex:
raise GeoIP2Error('Received a 200 response for %(uri)s'
' but could not decode the response as '
'JSON: ' % locals() + ', '.join(ex.args), 200,
uri)
def _handle_error(self, response, uri):
status = response.status_code
if 400 <= status < 500:
self._handle_4xx_status(response, status, uri)
elif 500 <= status < 600:
self._handle_5xx_status(status, uri)
else:
self._handle_non_200_status(status, uri)
def _handle_4xx_status(self, response, status, uri):
if not response.content:
raise HTTPError('Received a %(status)i error for %(uri)s '
'with no body.' % locals(), status, uri)
elif response.headers['Content-Type'].find('json') == -1:
raise HTTPError('Received a %i for %s with the following '
'body: %s' % (status, uri, response.content),
status, uri)
try:
body = response.json()
except ValueError as ex:
raise HTTPError(
'Received a %(status)i error for %(uri)s but it did'
' not include the expected JSON body: ' % locals() +
', '.join(ex.args), status, uri)
else:
if 'code' in body and 'error' in body:
self._handle_web_service_error(
body.get('error'), body.get('code'), status, uri)
else:
raise HTTPError(
'Response contains JSON but it does not specify '
'code or error keys', status, uri)
def _handle_web_service_error(self, message, code, status, uri):
if code in ('IP_ADDRESS_NOT_FOUND', 'IP_ADDRESS_RESERVED'):
raise AddressNotFoundError(message)
elif code in ('AUTHORIZATION_INVALID', 'LICENSE_KEY_REQUIRED',
'USER_ID_REQUIRED', 'USER_ID_UNKNOWN'):
raise AuthenticationError(message)
elif code in ('INSUFFICIENT_FUNDS', 'OUT_OF_QUERIES'):
raise OutOfQueriesError(message)
elif code == 'PERMISSION_REQUIRED':
raise PermissionRequiredError(message)
raise InvalidRequestError(message, code, status, uri)
def _handle_5xx_status(self, status, uri):
raise HTTPError('Received a server error (%(status)i) for '
'%(uri)s' % locals(), status, uri)
def _handle_non_200_status(self, status, uri):
raise HTTPError('Received a very surprising HTTP status '
'(%(status)i) for %(uri)s' % locals(), status, uri)

46
lib/maxminddb/__init__.py Normal file
View file

@ -0,0 +1,46 @@
# pylint:disable=C0111
import os
import maxminddb.reader
try:
import maxminddb.extension
except ImportError:
maxminddb.extension = None
from maxminddb.const import (MODE_AUTO, MODE_MMAP, MODE_MMAP_EXT, MODE_FILE,
MODE_MEMORY)
from maxminddb.decoder import InvalidDatabaseError
def open_database(database, mode=MODE_AUTO):
"""Open a Maxmind DB database
Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2
database file.
mode -- mode to open the database with. Valid mode are:
* MODE_MMAP_EXT - use the C extension with memory map.
* MODE_MMAP - read from memory map. Pure Python.
* MODE_FILE - read database as standard file. Pure Python.
* MODE_MEMORY - load database into memory. Pure Python.
* MODE_AUTO - tries MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that
order. Default mode.
"""
if (mode == MODE_AUTO and maxminddb.extension and
hasattr(maxminddb.extension, 'Reader')) or mode == MODE_MMAP_EXT:
return maxminddb.extension.Reader(database)
elif mode in (MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY):
return maxminddb.reader.Reader(database, mode)
raise ValueError('Unsupported open mode: {0}'.format(mode))
def Reader(database): # pylint: disable=invalid-name
"""This exists for backwards compatibility. Use open_database instead"""
return open_database(database)
__title__ = 'maxminddb'
__version__ = '1.2.1'
__author__ = 'Gregory Oschwald'
__license__ = 'Apache License, Version 2.0'
__copyright__ = 'Copyright 2014 Maxmind, Inc.'

33
lib/maxminddb/compat.py Normal file
View file

@ -0,0 +1,33 @@
import sys
import ipaddress
# pylint: skip-file
if sys.version_info[0] == 2:
def compat_ip_address(address):
if isinstance(address, bytes):
address = address.decode()
return ipaddress.ip_address(address)
int_from_byte = ord
FileNotFoundError = IOError
def int_from_bytes(b):
if b:
return int(b.encode("hex"), 16)
return 0
byte_from_int = chr
else:
def compat_ip_address(address):
return ipaddress.ip_address(address)
int_from_byte = lambda x: x
FileNotFoundError = FileNotFoundError
int_from_bytes = lambda x: int.from_bytes(x, 'big')
byte_from_int = lambda x: bytes([x])

7
lib/maxminddb/const.py Normal file
View file

@ -0,0 +1,7 @@
"""Constants used in the API"""
MODE_AUTO = 0
MODE_MMAP_EXT = 1
MODE_MMAP = 2
MODE_FILE = 4
MODE_MEMORY = 8

173
lib/maxminddb/decoder.py Normal file
View file

@ -0,0 +1,173 @@
"""
maxminddb.decoder
~~~~~~~~~~~~~~~~~
This package contains code for decoding the MaxMind DB data section.
"""
from __future__ import unicode_literals
import struct
from maxminddb.compat import byte_from_int, int_from_bytes
from maxminddb.errors import InvalidDatabaseError
class Decoder(object): # pylint: disable=too-few-public-methods
"""Decoder for the data section of the MaxMind DB"""
def __init__(self, database_buffer, pointer_base=0, pointer_test=False):
"""Created a Decoder for a MaxMind DB
Arguments:
database_buffer -- an mmap'd MaxMind DB file.
pointer_base -- the base number to use when decoding a pointer
pointer_test -- used for internal unit testing of pointer code
"""
self._pointer_test = pointer_test
self._buffer = database_buffer
self._pointer_base = pointer_base
def _decode_array(self, size, offset):
array = []
for _ in range(size):
(value, offset) = self.decode(offset)
array.append(value)
return array, offset
def _decode_boolean(self, size, offset):
return size != 0, offset
def _decode_bytes(self, size, offset):
new_offset = offset + size
return self._buffer[offset:new_offset], new_offset
# pylint: disable=no-self-argument
# |-> I am open to better ways of doing this as long as it doesn't involve
# lots of code duplication.
def _decode_packed_type(type_code, type_size, pad=False):
# pylint: disable=protected-access, missing-docstring
def unpack_type(self, size, offset):
if not pad:
self._verify_size(size, type_size)
new_offset = offset + type_size
packed_bytes = self._buffer[offset:new_offset]
if pad:
packed_bytes = packed_bytes.rjust(type_size, b'\x00')
(value,) = struct.unpack(type_code, packed_bytes)
return value, new_offset
return unpack_type
def _decode_map(self, size, offset):
container = {}
for _ in range(size):
(key, offset) = self.decode(offset)
(value, offset) = self.decode(offset)
container[key] = value
return container, offset
_pointer_value_offset = {
1: 0,
2: 2048,
3: 526336,
4: 0,
}
def _decode_pointer(self, size, offset):
pointer_size = ((size >> 3) & 0x3) + 1
new_offset = offset + pointer_size
pointer_bytes = self._buffer[offset:new_offset]
packed = pointer_bytes if pointer_size == 4 else struct.pack(
b'!c', byte_from_int(size & 0x7)) + pointer_bytes
unpacked = int_from_bytes(packed)
pointer = unpacked + self._pointer_base + \
self._pointer_value_offset[pointer_size]
if self._pointer_test:
return pointer, new_offset
(value, _) = self.decode(pointer)
return value, new_offset
def _decode_uint(self, size, offset):
new_offset = offset + size
uint_bytes = self._buffer[offset:new_offset]
return int_from_bytes(uint_bytes), new_offset
def _decode_utf8_string(self, size, offset):
new_offset = offset + size
return self._buffer[offset:new_offset].decode('utf-8'), new_offset
_type_decoder = {
1: _decode_pointer,
2: _decode_utf8_string,
3: _decode_packed_type(b'!d', 8), # double,
4: _decode_bytes,
5: _decode_uint, # uint16
6: _decode_uint, # uint32
7: _decode_map,
8: _decode_packed_type(b'!i', 4, pad=True), # int32
9: _decode_uint, # uint64
10: _decode_uint, # uint128
11: _decode_array,
14: _decode_boolean,
15: _decode_packed_type(b'!f', 4), # float,
}
def decode(self, offset):
"""Decode a section of the data section starting at offset
Arguments:
offset -- the location of the data structure to decode
"""
new_offset = offset + 1
(ctrl_byte,) = struct.unpack(b'!B', self._buffer[offset:new_offset])
type_num = ctrl_byte >> 5
# Extended type
if not type_num:
(type_num, new_offset) = self._read_extended(new_offset)
if type_num not in self._type_decoder:
raise InvalidDatabaseError('Unexpected type number ({type}) '
'encountered'.format(type=type_num))
(size, new_offset) = self._size_from_ctrl_byte(
ctrl_byte, new_offset, type_num)
return self._type_decoder[type_num](self, size, new_offset)
def _read_extended(self, offset):
(next_byte,) = struct.unpack(b'!B', self._buffer[offset:offset + 1])
type_num = next_byte + 7
if type_num < 7:
raise InvalidDatabaseError(
'Something went horribly wrong in the decoder. An '
'extended type resolved to a type number < 8 '
'({type})'.format(type=type_num))
return type_num, offset + 1
def _verify_size(self, expected, actual):
if expected != actual:
raise InvalidDatabaseError(
'The MaxMind DB file\'s data section contains bad data '
'(unknown data type or corrupt data)'
)
def _size_from_ctrl_byte(self, ctrl_byte, offset, type_num):
size = ctrl_byte & 0x1f
if type_num == 1:
return size, offset
bytes_to_read = 0 if size < 29 else size - 28
new_offset = offset + bytes_to_read
size_bytes = self._buffer[offset:new_offset]
# Using unpack rather than int_from_bytes as it is about 200 lookups
# per second faster here.
if size == 29:
size = 29 + struct.unpack(b'!B', size_bytes)[0]
elif size == 30:
size = 285 + struct.unpack(b'!H', size_bytes)[0]
elif size > 30:
size = struct.unpack(
b'!I', size_bytes.rjust(4, b'\x00'))[0] + 65821
return size, new_offset

11
lib/maxminddb/errors.py Normal file
View file

@ -0,0 +1,11 @@
"""
maxminddb.errors
~~~~~~~~~~~~~~~~
This module contains custom errors for the MaxMind DB reader
"""
class InvalidDatabaseError(RuntimeError):
"""This error is thrown when unexpected data is found in the database."""

570
lib/maxminddb/extension/maxminddb.c Normal file
View file

@ -0,0 +1,570 @@
#include <Python.h>
#include <maxminddb.h>
#include "structmember.h"
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
static PyTypeObject Reader_Type;
static PyTypeObject Metadata_Type;
static PyObject *MaxMindDB_error;
typedef struct {
PyObject_HEAD /* no semicolon */
MMDB_s *mmdb;
} Reader_obj;
typedef struct {
PyObject_HEAD /* no semicolon */
PyObject *binary_format_major_version;
PyObject *binary_format_minor_version;
PyObject *build_epoch;
PyObject *database_type;
PyObject *description;
PyObject *ip_version;
PyObject *languages;
PyObject *node_count;
PyObject *record_size;
} Metadata_obj;
static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list);
static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list);
static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list);
static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list);
#if PY_MAJOR_VERSION >= 3
#define MOD_INIT(name) PyMODINIT_FUNC PyInit_ ## name(void)
#define RETURN_MOD_INIT(m) return (m)
#define FILE_NOT_FOUND_ERROR PyExc_FileNotFoundError
#else
#define MOD_INIT(name) PyMODINIT_FUNC init ## name(void)
#define RETURN_MOD_INIT(m) return
#define PyInt_FromLong PyLong_FromLong
#define FILE_NOT_FOUND_ERROR PyExc_IOError
#endif
#ifdef __GNUC__
# define UNUSED(x) UNUSED_ ## x __attribute__((__unused__))
#else
# define UNUSED(x) UNUSED_ ## x
#endif
static int Reader_init(PyObject *self, PyObject *args, PyObject *kwds)
{
char *filename;
int mode = 0;
static char *kwlist[] = {"database", "mode", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", kwlist, &filename, &mode)) {
return -1;
}
if (mode != 0 && mode != 1) {
PyErr_Format(PyExc_ValueError, "Unsupported open mode (%i). Only "
"MODE_AUTO and MODE_MMAP_EXT are supported by this extension.",
mode);
return -1;
}
if (0 != access(filename, R_OK)) {
PyErr_Format(FILE_NOT_FOUND_ERROR,
"No such file or directory: '%s'",
filename);
return -1;
}
MMDB_s *mmdb = (MMDB_s *)malloc(sizeof(MMDB_s));
if (NULL == mmdb) {
PyErr_NoMemory();
return -1;
}
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (!mmdb_obj) {
free(mmdb);
PyErr_NoMemory();
return -1;
}
uint16_t status = MMDB_open(filename, MMDB_MODE_MMAP, mmdb);
if (MMDB_SUCCESS != status) {
free(mmdb);
PyErr_Format(
MaxMindDB_error,
"Error opening database file (%s). Is this a valid MaxMind DB file?",
filename
);
return -1;
}
mmdb_obj->mmdb = mmdb;
return 0;
}
static PyObject *Reader_get(PyObject *self, PyObject *args)
{
char *ip_address = NULL;
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (!PyArg_ParseTuple(args, "s", &ip_address)) {
return NULL;
}
MMDB_s *mmdb = mmdb_obj->mmdb;
if (NULL == mmdb) {
PyErr_SetString(PyExc_ValueError,
"Attempt to read from a closed MaxMind DB.");
return NULL;
}
int gai_error = 0;
int mmdb_error = MMDB_SUCCESS;
MMDB_lookup_result_s result =
MMDB_lookup_string(mmdb, ip_address, &gai_error,
&mmdb_error);
if (0 != gai_error) {
PyErr_Format(PyExc_ValueError,
"'%s' does not appear to be an IPv4 or IPv6 address.",
ip_address);
return NULL;
}
if (MMDB_SUCCESS != mmdb_error) {
PyObject *exception;
if (MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR == mmdb_error) {
exception = PyExc_ValueError;
} else {
exception = MaxMindDB_error;
}
PyErr_Format(exception, "Error looking up %s. %s",
ip_address, MMDB_strerror(mmdb_error));
return NULL;
}
if (!result.found_entry) {
Py_RETURN_NONE;
}
MMDB_entry_data_list_s *entry_data_list = NULL;
int status = MMDB_get_entry_data_list(&result.entry, &entry_data_list);
if (MMDB_SUCCESS != status) {
PyErr_Format(MaxMindDB_error,
"Error while looking up data for %s. %s",
ip_address, MMDB_strerror(status));
MMDB_free_entry_data_list(entry_data_list);
return NULL;
}
MMDB_entry_data_list_s *original_entry_data_list = entry_data_list;
PyObject *py_obj = from_entry_data_list(&entry_data_list);
MMDB_free_entry_data_list(original_entry_data_list);
return py_obj;
}
static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args))
{
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (NULL == mmdb_obj->mmdb) {
PyErr_SetString(PyExc_IOError,
"Attempt to read from a closed MaxMind DB.");
return NULL;
}
MMDB_entry_data_list_s *entry_data_list;
MMDB_get_metadata_as_entry_data_list(mmdb_obj->mmdb, &entry_data_list);
MMDB_entry_data_list_s *original_entry_data_list = entry_data_list;
PyObject *metadata_dict = from_entry_data_list(&entry_data_list);
MMDB_free_entry_data_list(original_entry_data_list);
if (NULL == metadata_dict || !PyDict_Check(metadata_dict)) {
PyErr_SetString(MaxMindDB_error,
"Error decoding metadata.");
return NULL;
}
PyObject *args = PyTuple_New(0);
if (NULL == args) {
Py_DECREF(metadata_dict);
return NULL;
}
PyObject *metadata = PyObject_Call((PyObject *)&Metadata_Type, args,
metadata_dict);
Py_DECREF(metadata_dict);
return metadata;
}
static PyObject *Reader_close(PyObject *self, PyObject *UNUSED(args))
{
Reader_obj *mmdb_obj = (Reader_obj *)self;
if (NULL != mmdb_obj->mmdb) {
MMDB_close(mmdb_obj->mmdb);
free(mmdb_obj->mmdb);
mmdb_obj->mmdb = NULL;
}
Py_RETURN_NONE;
}
static void Reader_dealloc(PyObject *self)
{
Reader_obj *obj = (Reader_obj *)self;
if (NULL != obj->mmdb) {
Reader_close(self, NULL);
}
PyObject_Del(self);
}
static int Metadata_init(PyObject *self, PyObject *args, PyObject *kwds)
{
PyObject
*binary_format_major_version,
*binary_format_minor_version,
*build_epoch,
*database_type,
*description,
*ip_version,
*languages,
*node_count,
*record_size;
static char *kwlist[] = {
"binary_format_major_version",
"binary_format_minor_version",
"build_epoch",
"database_type",
"description",
"ip_version",
"languages",
"node_count",
"record_size",
NULL
};
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOOOOOOOO", kwlist,
&binary_format_major_version,
&binary_format_minor_version,
&build_epoch,
&database_type,
&description,
&ip_version,
&languages,
&node_count,
&record_size)) {
return -1;
}
Metadata_obj *obj = (Metadata_obj *)self;
obj->binary_format_major_version = binary_format_major_version;
obj->binary_format_minor_version = binary_format_minor_version;
obj->build_epoch = build_epoch;
obj->database_type = database_type;
obj->description = description;
obj->ip_version = ip_version;
obj->languages = languages;
obj->node_count = node_count;
obj->record_size = record_size;
Py_INCREF(obj->binary_format_major_version);
Py_INCREF(obj->binary_format_minor_version);
Py_INCREF(obj->build_epoch);
Py_INCREF(obj->database_type);
Py_INCREF(obj->description);
Py_INCREF(obj->ip_version);
Py_INCREF(obj->languages);
Py_INCREF(obj->node_count);
Py_INCREF(obj->record_size);
return 0;
}
static void Metadata_dealloc(PyObject *self)
{
Metadata_obj *obj = (Metadata_obj *)self;
Py_DECREF(obj->binary_format_major_version);
Py_DECREF(obj->binary_format_minor_version);
Py_DECREF(obj->build_epoch);
Py_DECREF(obj->database_type);
Py_DECREF(obj->description);
Py_DECREF(obj->ip_version);
Py_DECREF(obj->languages);
Py_DECREF(obj->node_count);
Py_DECREF(obj->record_size);
PyObject_Del(self);
}
static PyObject *from_entry_data_list(MMDB_entry_data_list_s **entry_data_list)
{
if (NULL == entry_data_list || NULL == *entry_data_list) {
PyErr_SetString(
MaxMindDB_error,
"Error while looking up data. Your database may be corrupt or you have found a bug in libmaxminddb."
);
return NULL;
}
switch ((*entry_data_list)->entry_data.type) {
case MMDB_DATA_TYPE_MAP:
return from_map(entry_data_list);
case MMDB_DATA_TYPE_ARRAY:
return from_array(entry_data_list);
case MMDB_DATA_TYPE_UTF8_STRING:
return PyUnicode_FromStringAndSize(
(*entry_data_list)->entry_data.utf8_string,
(*entry_data_list)->entry_data.data_size
);
case MMDB_DATA_TYPE_BYTES:
return PyByteArray_FromStringAndSize(
(const char *)(*entry_data_list)->entry_data.bytes,
(Py_ssize_t)(*entry_data_list)->entry_data.data_size);
case MMDB_DATA_TYPE_DOUBLE:
return PyFloat_FromDouble((*entry_data_list)->entry_data.double_value);
case MMDB_DATA_TYPE_FLOAT:
return PyFloat_FromDouble((*entry_data_list)->entry_data.float_value);
case MMDB_DATA_TYPE_UINT16:
return PyLong_FromLong( (*entry_data_list)->entry_data.uint16);
case MMDB_DATA_TYPE_UINT32:
return PyLong_FromLong((*entry_data_list)->entry_data.uint32);
case MMDB_DATA_TYPE_BOOLEAN:
return PyBool_FromLong((*entry_data_list)->entry_data.boolean);
case MMDB_DATA_TYPE_UINT64:
return PyLong_FromUnsignedLongLong(
(*entry_data_list)->entry_data.uint64);
case MMDB_DATA_TYPE_UINT128:
return from_uint128(*entry_data_list);
case MMDB_DATA_TYPE_INT32:
return PyLong_FromLong((*entry_data_list)->entry_data.int32);
default:
PyErr_Format(MaxMindDB_error,
"Invalid data type arguments: %d",
(*entry_data_list)->entry_data.type);
return NULL;
}
return NULL;
}
static PyObject *from_map(MMDB_entry_data_list_s **entry_data_list)
{
PyObject *py_obj = PyDict_New();
if (NULL == py_obj) {
PyErr_NoMemory();
return NULL;
}
const uint32_t map_size = (*entry_data_list)->entry_data.data_size;
uint i;
// entry_data_list cannot start out NULL (see from_entry_data_list). We
// check it in the loop because it may become NULL.
// coverity[check_after_deref]
for (i = 0; i < map_size && entry_data_list; i++) {
*entry_data_list = (*entry_data_list)->next;
PyObject *key = PyUnicode_FromStringAndSize(
(char *)(*entry_data_list)->entry_data.utf8_string,
(*entry_data_list)->entry_data.data_size
);
*entry_data_list = (*entry_data_list)->next;
PyObject *value = from_entry_data_list(entry_data_list);
if (NULL == value) {
Py_DECREF(key);
Py_DECREF(py_obj);
return NULL;
}
PyDict_SetItem(py_obj, key, value);
Py_DECREF(value);
Py_DECREF(key);
}
return py_obj;
}
static PyObject *from_array(MMDB_entry_data_list_s **entry_data_list)
{
const uint32_t size = (*entry_data_list)->entry_data.data_size;
PyObject *py_obj = PyList_New(size);
if (NULL == py_obj) {
PyErr_NoMemory();
return NULL;
}
uint i;
// entry_data_list cannot start out NULL (see from_entry_data_list). We
// check it in the loop because it may become NULL.
// coverity[check_after_deref]
for (i = 0; i < size && entry_data_list; i++) {
*entry_data_list = (*entry_data_list)->next;
PyObject *value = from_entry_data_list(entry_data_list);
if (NULL == value) {
Py_DECREF(py_obj);
return NULL;
}
// PyList_SetItem 'steals' the reference
PyList_SetItem(py_obj, i, value);
}
return py_obj;
}
static PyObject *from_uint128(const MMDB_entry_data_list_s *entry_data_list)
{
uint64_t high = 0;
uint64_t low = 0;
#if MMDB_UINT128_IS_BYTE_ARRAY
int i;
for (i = 0; i < 8; i++) {
high = (high << 8) | entry_data_list->entry_data.uint128[i];
}
for (i = 8; i < 16; i++) {
low = (low << 8) | entry_data_list->entry_data.uint128[i];
}
#else
high = entry_data_list->entry_data.uint128 >> 64;
low = (uint64_t)entry_data_list->entry_data.uint128;
#endif
char *num_str = malloc(33);
if (NULL == num_str) {
PyErr_NoMemory();
return NULL;
}
snprintf(num_str, 33, "%016" PRIX64 "%016" PRIX64, high, low);
PyObject *py_obj = PyLong_FromString(num_str, NULL, 16);
free(num_str);
return py_obj;
}
static PyMethodDef Reader_methods[] = {
{ "get", Reader_get, METH_VARARGS,
"Get record for IP address" },
{ "metadata", Reader_metadata, METH_NOARGS,
"Returns metadata object for database" },
{ "close", Reader_close, METH_NOARGS, "Closes database"},
{ NULL, NULL, 0, NULL }
};
static PyTypeObject Reader_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_basicsize = sizeof(Reader_obj),
.tp_dealloc = Reader_dealloc,
.tp_doc = "Reader object",
.tp_flags = Py_TPFLAGS_DEFAULT,
.tp_methods = Reader_methods,
.tp_name = "Reader",
.tp_init = Reader_init,
};
static PyMethodDef Metadata_methods[] = {
{ NULL, NULL, 0, NULL }
};
/* *INDENT-OFF* */
static PyMemberDef Metadata_members[] = {
{ "binary_format_major_version", T_OBJECT, offsetof(
Metadata_obj, binary_format_major_version), READONLY, NULL },
{ "binary_format_minor_version", T_OBJECT, offsetof(
Metadata_obj, binary_format_minor_version), READONLY, NULL },
{ "build_epoch", T_OBJECT, offsetof(Metadata_obj, build_epoch),
READONLY, NULL },
{ "database_type", T_OBJECT, offsetof(Metadata_obj, database_type),
READONLY, NULL },
{ "description", T_OBJECT, offsetof(Metadata_obj, description),
READONLY, NULL },
{ "ip_version", T_OBJECT, offsetof(Metadata_obj, ip_version),
READONLY, NULL },
{ "languages", T_OBJECT, offsetof(Metadata_obj, languages), READONLY,
NULL },
{ "node_count", T_OBJECT, offsetof(Metadata_obj, node_count),
READONLY, NULL },
{ "record_size", T_OBJECT, offsetof(Metadata_obj, record_size),
READONLY, NULL },
{ NULL, 0, 0, 0, NULL }
};
/* *INDENT-ON* */
static PyTypeObject Metadata_Type = {
PyVarObject_HEAD_INIT(NULL, 0)
.tp_basicsize = sizeof(Metadata_obj),
.tp_dealloc = Metadata_dealloc,
.tp_doc = "Metadata object",
.tp_flags = Py_TPFLAGS_DEFAULT,
.tp_members = Metadata_members,
.tp_methods = Metadata_methods,
.tp_name = "Metadata",
.tp_init = Metadata_init
};
static PyMethodDef MaxMindDB_methods[] = {
{ NULL, NULL, 0, NULL }
};
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef MaxMindDB_module = {
PyModuleDef_HEAD_INIT,
.m_name = "extension",
.m_doc = "This is a C extension to read MaxMind DB file format",
.m_methods = MaxMindDB_methods,
};
#endif
MOD_INIT(extension){
PyObject *m;
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&MaxMindDB_module);
#else
m = Py_InitModule("extension", MaxMindDB_methods);
#endif
if (!m) {
RETURN_MOD_INIT(NULL);
}
Reader_Type.tp_new = PyType_GenericNew;
if (PyType_Ready(&Reader_Type)) {
RETURN_MOD_INIT(NULL);
}
Py_INCREF(&Reader_Type);
PyModule_AddObject(m, "Reader", (PyObject *)&Reader_Type);
Metadata_Type.tp_new = PyType_GenericNew;
if (PyType_Ready(&Metadata_Type)) {
RETURN_MOD_INIT(NULL);
}
PyModule_AddObject(m, "extension", (PyObject *)&Metadata_Type);
PyObject* error_mod = PyImport_ImportModule("maxminddb.errors");
if (error_mod == NULL) {
RETURN_MOD_INIT(NULL);
}
MaxMindDB_error = PyObject_GetAttrString(error_mod, "InvalidDatabaseError");
Py_DECREF(error_mod);
if (MaxMindDB_error == NULL) {
RETURN_MOD_INIT(NULL);
}
Py_INCREF(MaxMindDB_error);
/* We primarily add it to the module for backwards compatibility */
PyModule_AddObject(m, "InvalidDatabaseError", MaxMindDB_error);
RETURN_MOD_INIT(m);
}

66
lib/maxminddb/file.py Normal file
View file

@ -0,0 +1,66 @@
"""For internal use only. It provides a slice-like file reader."""
import os
try:
# pylint: disable=no-name-in-module
from multiprocessing import Lock
except ImportError:
from threading import Lock
class FileBuffer(object):
"""A slice-able file reader"""
def __init__(self, database):
self._handle = open(database, 'rb')
self._size = os.fstat(self._handle.fileno()).st_size
if not hasattr(os, 'pread'):
self._lock = Lock()
def __getitem__(self, key):
if isinstance(key, slice):
return self._read(key.stop - key.start, key.start)
elif isinstance(key, int):
return self._read(1, key)
else:
raise TypeError("Invalid argument type.")
def rfind(self, needle, start):
"""Reverse find needle from start"""
pos = self._read(self._size - start - 1, start).rfind(needle)
if pos == -1:
return pos
return start + pos
def size(self):
"""Size of file"""
return self._size
def close(self):
"""Close file"""
self._handle.close()
if hasattr(os, 'pread'):
def _read(self, buffersize, offset):
"""read that uses pread"""
# pylint: disable=no-member
return os.pread(self._handle.fileno(), buffersize, offset)
else:
def _read(self, buffersize, offset):
"""read with a lock
This lock is necessary as after a fork, the different processes
will share the same file table entry, even if we dup the fd, and
as such the same offsets. There does not appear to be a way to
duplicate the file table entry and we cannot re-open based on the
original path as that file may have replaced with another or
unlinked.
"""
with self._lock:
self._handle.seek(offset)
return self._handle.read(buffersize)

223
lib/maxminddb/reader.py Normal file
View file

@ -0,0 +1,223 @@
"""
maxminddb.reader
~~~~~~~~~~~~~~~~
This module contains the pure Python database reader and related classes.
"""
from __future__ import unicode_literals
try:
import mmap
except ImportError:
# pylint: disable=invalid-name
mmap = None
import struct
from maxminddb.compat import byte_from_int, int_from_byte, compat_ip_address
from maxminddb.const import MODE_AUTO, MODE_MMAP, MODE_FILE, MODE_MEMORY
from maxminddb.decoder import Decoder
from maxminddb.errors import InvalidDatabaseError
from maxminddb.file import FileBuffer
class Reader(object):
"""
Instances of this class provide a reader for the MaxMind DB format. IP
addresses can be looked up using the ``get`` method.
"""
_DATA_SECTION_SEPARATOR_SIZE = 16
_METADATA_START_MARKER = b"\xAB\xCD\xEFMaxMind.com"
_ipv4_start = None
def __init__(self, database, mode=MODE_AUTO):
"""Reader for the MaxMind DB file format
Arguments:
database -- A path to a valid MaxMind DB file such as a GeoIP2
database file.
mode -- mode to open the database with. Valid mode are:
* MODE_MMAP - read from memory map.
* MODE_FILE - read database as standard file.
* MODE_MEMORY - load database into memory.
* MODE_AUTO - tries MODE_MMAP and then MODE_FILE. Default.
"""
# pylint: disable=redefined-variable-type
if (mode == MODE_AUTO and mmap) or mode == MODE_MMAP:
with open(database, 'rb') as db_file:
self._buffer = mmap.mmap(
db_file.fileno(), 0, access=mmap.ACCESS_READ)
self._buffer_size = self._buffer.size()
elif mode in (MODE_AUTO, MODE_FILE):
self._buffer = FileBuffer(database)
self._buffer_size = self._buffer.size()
elif mode == MODE_MEMORY:
with open(database, 'rb') as db_file:
self._buffer = db_file.read()
self._buffer_size = len(self._buffer)
else:
raise ValueError('Unsupported open mode ({0}). Only MODE_AUTO, '
' MODE_FILE, and MODE_MEMORY are support by the pure Python '
'Reader'.format(mode))
metadata_start = self._buffer.rfind(self._METADATA_START_MARKER,
max(0, self._buffer_size
- 128 * 1024))
if metadata_start == -1:
self.close()
raise InvalidDatabaseError('Error opening database file ({0}). '
'Is this a valid MaxMind DB file?'
''.format(database))
metadata_start += len(self._METADATA_START_MARKER)
metadata_decoder = Decoder(self._buffer, metadata_start)
(metadata, _) = metadata_decoder.decode(metadata_start)
self._metadata = Metadata(
**metadata) # pylint: disable=bad-option-value
self._decoder = Decoder(self._buffer, self._metadata.search_tree_size
+ self._DATA_SECTION_SEPARATOR_SIZE)
def metadata(self):
"""Return the metadata associated with the MaxMind DB file"""
return self._metadata
def get(self, ip_address):
"""Return the record for the ip_address in the MaxMind DB
Arguments:
ip_address -- an IP address in the standard string notation
"""
address = compat_ip_address(ip_address)
if address.version == 6 and self._metadata.ip_version == 4:
raise ValueError('Error looking up {0}. You attempted to look up '
'an IPv6 address in an IPv4-only database.'.format(
ip_address))
pointer = self._find_address_in_tree(address)
return self._resolve_data_pointer(pointer) if pointer else None
def _find_address_in_tree(self, ip_address):
packed = ip_address.packed
bit_count = len(packed) * 8
node = self._start_node(bit_count)
for i in range(bit_count):
if node >= self._metadata.node_count:
break
bit = 1 & (int_from_byte(packed[i >> 3]) >> 7 - (i % 8))
node = self._read_node(node, bit)
if node == self._metadata.node_count:
# Record is empty
return 0
elif node > self._metadata.node_count:
return node
raise InvalidDatabaseError('Invalid node in search tree')
def _start_node(self, length):
if self._metadata.ip_version != 6 or length == 128:
return 0
# We are looking up an IPv4 address in an IPv6 tree. Skip over the
# first 96 nodes.
if self._ipv4_start:
return self._ipv4_start
node = 0
for _ in range(96):
if node >= self._metadata.node_count:
break
node = self._read_node(node, 0)
self._ipv4_start = node
return node
def _read_node(self, node_number, index):
base_offset = node_number * self._metadata.node_byte_size
record_size = self._metadata.record_size
if record_size == 24:
offset = base_offset + index * 3
node_bytes = b'\x00' + self._buffer[offset:offset + 3]
elif record_size == 28:
(middle,) = struct.unpack(
b'!B', self._buffer[base_offset + 3:base_offset + 4])
if index:
middle &= 0x0F
else:
middle = (0xF0 & middle) >> 4
offset = base_offset + index * 4
node_bytes = byte_from_int(
middle) + self._buffer[offset:offset + 3]
elif record_size == 32:
offset = base_offset + index * 4
node_bytes = self._buffer[offset:offset + 4]
else:
raise InvalidDatabaseError(
'Unknown record size: {0}'.format(record_size))
return struct.unpack(b'!I', node_bytes)[0]
def _resolve_data_pointer(self, pointer):
resolved = pointer - self._metadata.node_count + \
self._metadata.search_tree_size
if resolved > self._buffer_size:
raise InvalidDatabaseError(
"The MaxMind DB file's search tree is corrupt")
(data, _) = self._decoder.decode(resolved)
return data
def close(self):
"""Closes the MaxMind DB file and returns the resources to the system"""
# pylint: disable=unidiomatic-typecheck
if type(self._buffer) not in (str, bytes):
self._buffer.close()
class Metadata(object):
"""Metadata for the MaxMind DB reader"""
# pylint: disable=too-many-instance-attributes
def __init__(self, **kwargs):
"""Creates new Metadata object. kwargs are key/value pairs from spec"""
# Although I could just update __dict__, that is less obvious and it
# doesn't work well with static analysis tools and some IDEs
self.node_count = kwargs['node_count']
self.record_size = kwargs['record_size']
self.ip_version = kwargs['ip_version']
self.database_type = kwargs['database_type']
self.languages = kwargs['languages']
self.binary_format_major_version = kwargs[
'binary_format_major_version']
self.binary_format_minor_version = kwargs[
'binary_format_minor_version']
self.build_epoch = kwargs['build_epoch']
self.description = kwargs['description']
@property
def node_byte_size(self):
"""The size of a node in bytes"""
return self.record_size // 4
@property
def search_tree_size(self):
"""The size of the search tree"""
return self.node_count * self.node_byte_size
def __repr__(self):
args = ', '.join('%s=%r' % x for x in self.__dict__.items())
return '{module}.{class_name}({data})'.format(
module=self.__module__,
class_name=self.__class__.__name__,
data=args)

1
plexpy/config.py
View file

@ -154,6 +154,7 @@ _CONFIG_DEFINITIONS = {
'FACEBOOK_ON_PMSUPDATE': (int, 'Facebook', 0),
'FIRST_RUN_COMPLETE': (int, 'General', 0),
'FREEZE_DB': (int, 'General', 0),
'GEOIP_DB': (str, 'General', ''),
'GET_FILE_SIZES': (int, 'General', 0),
'GET_FILE_SIZES_HOLD': (dict, 'General', {'section_ids': [], 'rating_keys': []}),
'GIT_BRANCH': (str, 'General', 'master'),

95
plexpy/helpers.py
View file

@ -16,11 +16,14 @@
import base64
import datetime
from functools import wraps
import geoip2.database, geoip2.errors
import gzip
import hashlib
import imghdr
from IPy import IP
import json
import math
import maxminddb
from operator import itemgetter
import os
import re
@ -514,6 +517,98 @@ def get_ip(host):
return ip_address
def install_geoip_db():
maxmind_url = 'http://geolite.maxmind.com/download/geoip/database/'
geolite2_gz = 'GeoLite2-City.mmdb.gz'
geolite2_md5 = 'GeoLite2-City.md5'
geolite2_db = geolite2_gz[:-3]
md5_checksum = ''
temp_gz = os.path.join(plexpy.CONFIG.CACHE_DIR, geolite2_gz)
geolite2_db = os.path.join(plexpy.DATA_DIR, geolite2_db)
# Retrieve the GeoLite2 gzip file
logger.debug(u"PlexPy Helpers :: Downloading GeoLite2 gzip file from MaxMind...")
try:
maxmind = urllib.URLopener()
maxmind.retrieve(maxmind_url + geolite2_gz, temp_gz)
md5_checksum = urllib2.urlopen(maxmind_url + geolite2_md5).read()
except Exception as e:
logger.error(u"PlexPy Helpers :: Failed to download GeoLite2 gzip file from MaxMind: %s" % e)
return False
# Extract the GeoLite2 database file
logger.debug(u"PlexPy Helpers :: Extracting GeoLite2 database...")
try:
with gzip.open(temp_gz, 'rb') as gz:
with open(geolite2_db, 'wb') as db:
db.write(gz.read())
except Exception as e:
logger.error(u"PlexPy Helpers :: Failed to extract the GeoLite2 database: %s" % e)
return False
# Check MD5 hash for GeoLite2 database file
logger.debug(u"PlexPy Helpers :: Checking MD5 checksum for GeoLite2 database...")
try:
hash_md5 = hashlib.md5()
with open(geolite2_db, 'rb') as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
md5_hash = hash_md5.hexdigest()
if md5_hash != md5_checksum:
logger.error(u"PlexPy Helpers :: MD5 checksum doesn't match for GeoLite2 database. "
"Checksum: %s, file hash: %s" % (md5_checksum, md5_hash))
return False
except Exception as e:
logger.error(u"PlexPy Helpers :: Failed to generate MD5 checksum for GeoLite2 database: %s" % e)
return False
# Delete temportary GeoLite2 gzip file
logger.debug(u"PlexPy Helpers :: Deleting temporary GeoLite2 gzip file...")
try:
os.remove(temp_gz)
except Exception as e:
logger.warn(u"PlexPy Helpers :: Failed to remove temporary GeoLite2 gzip file: %s" % e)
logger.debug(u"PlexPy Helpers :: GeoLite2 database installed successfully.")
plexpy.CONFIG.__setattr__('GEOIP_DB', geolite2_db)
plexpy.CONFIG.write()
return True
def geoip_lookup(ip_address):
if not plexpy.CONFIG.GEOIP_DB:
return 'GeoLite2 database not installed. Please install from the Settings page.'
if not ip_address:
return 'No IP address provided.'
try:
reader = geoip2.database.Reader(plexpy.CONFIG.GEOIP_DB)
geo = reader.city(ip_address)
reader.close()
except IOError as e:
return 'Missing GeoLite2 database. Please reinstall from the Settings page.'
except ValueError as e:
return 'Unable to read GeoLite2 database: %s' % e
except maxminddb.InvalidDatabaseError as e:
return 'Invalid GeoLite2 database.'
except geoip2.errors.AddressNotFoundError as e:
return '%s' % e
except Exception as e:
return 'Error: %s' % e
geo_info = {'country': geo.country.name,
'region': geo.subdivisions.most_specific.name,
'city': geo.city.name,
'timezone': geo.location.time_zone,
'latitude': geo.location.latitude,
'longitude': geo.location.longitude
}
return geo_info
# Taken from SickRage
def anon_url(*url):
"""

23
plexpy/webserve.py
View file

@ -2608,6 +2608,7 @@ class WebInterface(object):
"imgur_client_id": plexpy.CONFIG.IMGUR_CLIENT_ID,
"cache_images": checked(plexpy.CONFIG.CACHE_IMAGES),
"pms_version": plexpy.CONFIG.PMS_VERSION,
"geoip_db": plexpy.CONFIG.GEOIP_DB
}
return serve_template(templatename="settings.html", title="Settings", config=config)
@ -2795,6 +2796,19 @@ class WebInterface(object):
else:
return {'result': 'error', 'message': 'Database backup failed.'}
@cherrypy.expose
@cherrypy.tools.json_out()
@requireAuth(member_of("admin"))
def install_geoip_db(self):
""" Downloads and installs the GeoIP database """
result = helpers.install_geoip_db()
if result:
return {'result': 'success', 'message': 'GeoIP database installed successful.'}
else:
return {'result': 'error', 'message': 'GeoIP database install failed.'}
@cherrypy.expose
@requireAuth(member_of("admin"))
def get_notification_agent_config(self, agent_id, **kwargs):
@ -4244,3 +4258,12 @@ class WebInterface(object):
pms_connect = pmsconnect.PmsConnect()
result = pms_connect.get_update_staus()
return result
@cherrypy.expose
@cherrypy.tools.json_out()
@requireAuth()
def get_geoip_lookup(self, ip_address='', **kwargs):
geo_info = helpers.geoip_lookup(ip_address)
if isinstance(geo_info, basestring):
return {'error': geo_info}
return geo_info